From 897e08f83ec64dd833a85a93691f556c0f51f9af Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:25:15 -0500 Subject: [PATCH 01/14] Update Python SDK Version to 1.22.0b2 (#1589) Co-authored-by: AzureFunctionsPython --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 244c3cb2..aeb411d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ "Intended Audience :: Developers" ] dependencies = [ - "azure-functions==1.22.0b1", + "azure-functions==1.22.0b2", "python-dateutil ~=2.9.0", "protobuf~=3.19.3; python_version == '3.7'", "protobuf~=4.25.3; python_version >= '3.8'", From dd28a181c56349bf0222f125015dad056cba3cc1 Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Wed, 9 Oct 2024 15:22:23 -0500 Subject: [PATCH 02/14] Update Python SDK Version to 1.22.0b3 (#1592) Co-authored-by: AzureFunctionsPython --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index aeb411d0..cdba09cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ "Intended Audience :: Developers" ] dependencies = [ - "azure-functions==1.22.0b2", + "azure-functions==1.22.0b3", "python-dateutil ~=2.9.0", "protobuf~=3.19.3; python_version == '3.7'", "protobuf~=4.25.3; python_version >= '3.8'", From 0bfcc7a23e5ff1f1e82c19bc80a1f90ff6d5335f Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:45:13 -0500 Subject: [PATCH 03/14] Update Python SDK Version to 1.22.0b4 (#1601) Co-authored-by: AzureFunctionsPython --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cdba09cc..dfde4254 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ "Intended Audience :: Developers" ] dependencies = [ - "azure-functions==1.22.0b3", + "azure-functions==1.22.0b4", "python-dateutil ~=2.9.0", "protobuf~=3.19.3; python_version == '3.7'", "protobuf~=4.25.3; python_version >= '3.8'", From 7e6acf36e43335ad69742923af9ff3d62b99242a Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:43:52 -0500 Subject: [PATCH 04/14] build: update Python Worker Version to 4.34.0 (#1602) Co-authored-by: AzureFunctionsPython --- azure_functions_worker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_functions_worker/version.py b/azure_functions_worker/version.py index 3a14957c..adb42153 100644 --- a/azure_functions_worker/version.py +++ b/azure_functions_worker/version.py @@ -1,4 +1,4 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -VERSION = '4.33.0' +VERSION = '4.34.0' From 139af01c5ca4dd6a504a6e88d93ec21529c186f6 Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:16:59 -0600 Subject: [PATCH 05/14] fix: codeql fixes (#1606) * codeql fixes * lint * removed redundant pycryptodome definition --------- Co-authored-by: Victoria Hall --- .../blob_functions_stein/function_app.py | 40 ++++++------- .../generic/function_app.py | 40 ++++++------- .../main.py | 4 +- .../main.py | 4 +- .../main.py | 4 +- .../main.py | 4 +- .../main.py | 4 +- .../main.py | 20 +++---- .../test_mock_blob_shared_memory_functions.py | 60 +++++++++---------- .../test_third_party_http_functions.py | 9 ++- .../stein/asgi_function/function_app.py | 17 +++++- .../stein/wsgi_function/function_app.py | 4 +- 12 files changed, 113 insertions(+), 97 deletions(-) diff --git a/tests/endtoend/blob_functions/blob_functions_stein/function_app.py b/tests/endtoend/blob_functions/blob_functions_stein/function_app.py index 2621a336..2edc1384 100644 --- a/tests/endtoend/blob_functions/blob_functions_stein/function_app.py +++ b/tests/endtoend/blob_functions/blob_functions_stein/function_app.py @@ -52,11 +52,11 @@ def get_blob_as_bytes_return_http_response(req: func.HttpRequest, file: bytes) \ assert isinstance(file, bytes) content_size = len(file) - content_md5 = hashlib.md5(file).hexdigest() + content_sha256 = hashlib.sha256(file).hexdigest() response_dict = { 'content_size': content_size, - 'content_md5': content_md5 + 'content_sha256': content_sha256 } response_body = json.dumps(response_dict, indent=2) @@ -84,11 +84,11 @@ def get_blob_as_bytes_stream_return_http_response(req: func.HttpRequest, file_bytes = file.read() content_size = len(file_bytes) - content_md5 = hashlib.md5(file_bytes).hexdigest() + content_sha256 = hashlib.sha256(file_bytes).hexdigest() response_dict = { 'content_size': content_size, - 'content_md5': content_md5 + 'content_sha256': content_sha256 } response_body = json.dumps(response_dict, indent=2) @@ -127,11 +127,11 @@ def get_blob_as_str_return_http_response(req: func.HttpRequest, num_chars = len(file) content_bytes = file.encode('utf-8') - content_md5 = hashlib.md5(content_bytes).hexdigest() + content_sha256 = hashlib.sha256(content_bytes).hexdigest() response_dict = { 'num_chars': num_chars, - 'content_md5': content_md5 + 'content_sha256': content_sha256 } response_body = json.dumps(response_dict, indent=2) @@ -212,13 +212,13 @@ def put_blob_as_bytes_return_http_response(req: func.HttpRequest, content = b'\x01' * content_size else: content = bytearray(random.getrandbits(8) for _ in range(content_size)) - content_md5 = hashlib.md5(content).hexdigest() + content_sha256 = hashlib.sha256(content).hexdigest() file.set(content) response_dict = { 'content_size': content_size, - 'content_md5': content_md5 + 'content_sha256': content_sha256 } response_body = json.dumps(response_dict, indent=2) @@ -249,14 +249,14 @@ def put_blob_as_str_return_http_response(req: func.HttpRequest, file: func.Out[ k=num_chars)) content_bytes = content.encode('utf-8') content_size = len(content_bytes) - content_md5 = hashlib.md5(content_bytes).hexdigest() + content_sha256 = hashlib.sha256(content_bytes).hexdigest() file.set(content) response_dict = { 'num_chars': num_chars, 'content_size': content_size, - 'content_md5': content_md5 + 'content_sha256': content_sha256 } response_body = json.dumps(response_dict, indent=2) @@ -321,8 +321,8 @@ def put_blob_trigger(req: func.HttpRequest, file: func.Out[str]) -> str: def _generate_content_and_digest(content_size): content = bytearray(random.getrandbits(8) for _ in range(content_size)) - content_md5 = hashlib.md5(content).hexdigest() - return content, content_md5 + content_sha256 = hashlib.sha256(content).hexdigest() + return content, content_sha256 @app.function_name(name="put_get_multiple_blobs_as_bytes_return_http_response") @@ -359,15 +359,15 @@ def put_get_multiple_blobs_as_bytes_return_http_response( input_content_size_1 = len(inputfile1) input_content_size_2 = len(inputfile2) - input_content_md5_1 = hashlib.md5(inputfile1).hexdigest() - input_content_md5_2 = hashlib.md5(inputfile2).hexdigest() + input_content_sha256_1 = hashlib.sha256(inputfile1).hexdigest() + input_content_sha256_2 = hashlib.sha256(inputfile2).hexdigest() output_content_size_1 = int(req.params['output_content_size_1']) output_content_size_2 = int(req.params['output_content_size_2']) - output_content_1, output_content_md5_1 = \ + output_content_1, output_content_sha256_1 = \ _generate_content_and_digest(output_content_size_1) - output_content_2, output_content_md5_2 = \ + output_content_2, output_content_sha256_2 = \ _generate_content_and_digest(output_content_size_2) outputfile1.set(output_content_1) @@ -376,12 +376,12 @@ def put_get_multiple_blobs_as_bytes_return_http_response( response_dict = { 'input_content_size_1': input_content_size_1, 'input_content_size_2': input_content_size_2, - 'input_content_md5_1': input_content_md5_1, - 'input_content_md5_2': input_content_md5_2, + 'input_content_sha256_1': input_content_sha256_1, + 'input_content_sha256_2': input_content_sha256_2, 'output_content_size_1': output_content_size_1, 'output_content_size_2': output_content_size_2, - 'output_content_md5_1': output_content_md5_1, - 'output_content_md5_2': output_content_md5_2 + 'output_content_sha256_1': output_content_sha256_1, + 'output_content_sha256_2': output_content_sha256_2 } response_body = json.dumps(response_dict, indent=2) diff --git a/tests/endtoend/blob_functions/blob_functions_stein/generic/function_app.py b/tests/endtoend/blob_functions/blob_functions_stein/generic/function_app.py index d902a09e..77e9dc59 100644 --- a/tests/endtoend/blob_functions/blob_functions_stein/generic/function_app.py +++ b/tests/endtoend/blob_functions/blob_functions_stein/generic/function_app.py @@ -64,11 +64,11 @@ def get_blob_as_bytes_return_http_response(req: func.HttpRequest, file: bytes) \ assert isinstance(file, bytes) content_size = len(file) - content_md5 = hashlib.md5(file).hexdigest() + content_sha256 = hashlib.sha256(file).hexdigest() response_dict = { 'content_size': content_size, - 'content_md5': content_md5 + 'content_sha256': content_sha256 } response_body = json.dumps(response_dict, indent=2) @@ -100,11 +100,11 @@ def get_blob_as_bytes_stream_return_http_response(req: func.HttpRequest, file_bytes = file.read() content_size = len(file_bytes) - content_md5 = hashlib.md5(file_bytes).hexdigest() + content_sha256 = hashlib.sha256(file_bytes).hexdigest() response_dict = { 'content_size': content_size, - 'content_md5': content_md5 + 'content_sha256': content_sha256 } response_body = json.dumps(response_dict, indent=2) @@ -151,11 +151,11 @@ def get_blob_as_str_return_http_response(req: func.HttpRequest, num_chars = len(file) content_bytes = file.encode('utf-8') - content_md5 = hashlib.md5(content_bytes).hexdigest() + content_sha256 = hashlib.sha256(content_bytes).hexdigest() response_dict = { 'num_chars': num_chars, - 'content_md5': content_md5 + 'content_sha256': content_sha256 } response_body = json.dumps(response_dict, indent=2) @@ -259,13 +259,13 @@ def put_blob_as_bytes_return_http_response(req: func.HttpRequest, content = b'\x01' * content_size else: content = bytearray(random.getrandbits(8) for _ in range(content_size)) - content_md5 = hashlib.md5(content).hexdigest() + content_sha256 = hashlib.sha256(content).hexdigest() file.set(content) response_dict = { 'content_size': content_size, - 'content_md5': content_md5 + 'content_sha256': content_sha256 } response_body = json.dumps(response_dict, indent=2) @@ -299,14 +299,14 @@ def put_blob_as_str_return_http_response( k=num_chars)) content_bytes = content.encode('utf-8') content_size = len(content_bytes) - content_md5 = hashlib.md5(content_bytes).hexdigest() + content_sha256 = hashlib.sha256(content_bytes).hexdigest() file.set(content) response_dict = { 'num_chars': num_chars, 'content_size': content_size, - 'content_md5': content_md5 + 'content_sha256': content_sha256 } response_body = json.dumps(response_dict, indent=2) @@ -389,8 +389,8 @@ def put_blob_trigger(req: func.HttpRequest, file: func.Out[str]) -> str: def _generate_content_and_digest(content_size): content = bytearray(random.getrandbits(8) for _ in range(content_size)) - content_md5 = hashlib.md5(content).hexdigest() - return content, content_md5 + content_sha256 = hashlib.sha256(content).hexdigest() + return content, content_sha256 @app.function_name(name="put_get_multiple_blobs_as_bytes_return_http_response") @@ -437,15 +437,15 @@ def put_get_multiple_blobs_as_bytes_return_http_response( input_content_size_1 = len(inputfile1) input_content_size_2 = len(inputfile2) - input_content_md5_1 = hashlib.md5(inputfile1).hexdigest() - input_content_md5_2 = hashlib.md5(inputfile2).hexdigest() + input_content_sha256_1 = hashlib.sha256(inputfile1).hexdigest() + input_content_sha256_2 = hashlib.sha256(inputfile2).hexdigest() output_content_size_1 = int(req.params['output_content_size_1']) output_content_size_2 = int(req.params['output_content_size_2']) - output_content_1, output_content_md5_1 = \ + output_content_1, output_content_sha256_1 = \ _generate_content_and_digest(output_content_size_1) - output_content_2, output_content_md5_2 = \ + output_content_2, output_content_sha256_2 = \ _generate_content_and_digest(output_content_size_2) outputfile1.set(output_content_1) @@ -454,12 +454,12 @@ def put_get_multiple_blobs_as_bytes_return_http_response( response_dict = { 'input_content_size_1': input_content_size_1, 'input_content_size_2': input_content_size_2, - 'input_content_md5_1': input_content_md5_1, - 'input_content_md5_2': input_content_md5_2, + 'input_content_sha256_1': input_content_sha256_1, + 'input_content_sha256_2': input_content_sha256_2, 'output_content_size_1': output_content_size_1, 'output_content_size_2': output_content_size_2, - 'output_content_md5_1': output_content_md5_1, - 'output_content_md5_2': output_content_md5_2 + 'output_content_sha256_1': output_content_sha256_1, + 'output_content_sha256_2': output_content_sha256_2 } response_body = json.dumps(response_dict, indent=2) diff --git a/tests/endtoend/blob_functions/get_blob_as_bytes_return_http_response/main.py b/tests/endtoend/blob_functions/get_blob_as_bytes_return_http_response/main.py index 636ac90d..f7069cab 100644 --- a/tests/endtoend/blob_functions/get_blob_as_bytes_return_http_response/main.py +++ b/tests/endtoend/blob_functions/get_blob_as_bytes_return_http_response/main.py @@ -15,11 +15,11 @@ def main(req: azf.HttpRequest, file: bytes) -> azf.HttpResponse: assert isinstance(file, bytes) content_size = len(file) - content_md5 = hashlib.md5(file).hexdigest() + content_sha256 = hashlib.sha256(file).hexdigest() response_dict = { 'content_size': content_size, - 'content_md5': content_md5 + 'content_sha256': content_sha256 } response_body = json.dumps(response_dict, indent=2) diff --git a/tests/endtoend/blob_functions/get_blob_as_bytes_stream_return_http_response/main.py b/tests/endtoend/blob_functions/get_blob_as_bytes_stream_return_http_response/main.py index eb8986c0..bd65835b 100644 --- a/tests/endtoend/blob_functions/get_blob_as_bytes_stream_return_http_response/main.py +++ b/tests/endtoend/blob_functions/get_blob_as_bytes_stream_return_http_response/main.py @@ -15,11 +15,11 @@ def main(req: azf.HttpRequest, file: azf.InputStream) -> azf.HttpResponse: file_bytes = file.read() content_size = len(file_bytes) - content_md5 = hashlib.md5(file_bytes).hexdigest() + content_sha256 = hashlib.sha256(file_bytes).hexdigest() response_dict = { 'content_size': content_size, - 'content_md5': content_md5 + 'content_sha256': content_sha256 } response_body = json.dumps(response_dict, indent=2) diff --git a/tests/endtoend/blob_functions/get_blob_as_str_return_http_response/main.py b/tests/endtoend/blob_functions/get_blob_as_str_return_http_response/main.py index 8d8bf533..16f98375 100644 --- a/tests/endtoend/blob_functions/get_blob_as_str_return_http_response/main.py +++ b/tests/endtoend/blob_functions/get_blob_as_str_return_http_response/main.py @@ -16,11 +16,11 @@ def main(req: azf.HttpRequest, file: str) -> azf.HttpResponse: num_chars = len(file) content_bytes = file.encode('utf-8') - content_md5 = hashlib.md5(content_bytes).hexdigest() + content_sha256 = hashlib.sha256(content_bytes).hexdigest() response_dict = { 'num_chars': num_chars, - 'content_md5': content_md5 + 'content_sha256': content_sha256 } response_body = json.dumps(response_dict, indent=2) diff --git a/tests/endtoend/blob_functions/put_blob_as_bytes_return_http_response/main.py b/tests/endtoend/blob_functions/put_blob_as_bytes_return_http_response/main.py index 5e461cf9..58325882 100644 --- a/tests/endtoend/blob_functions/put_blob_as_bytes_return_http_response/main.py +++ b/tests/endtoend/blob_functions/put_blob_as_bytes_return_http_response/main.py @@ -24,13 +24,13 @@ def main(req: azf.HttpRequest, file: azf.Out[bytes]) -> azf.HttpResponse: content = b'\x01' * content_size else: content = bytearray(random.getrandbits(8) for _ in range(content_size)) - content_md5 = hashlib.md5(content).hexdigest() + content_sha256 = hashlib.sha256(content).hexdigest() file.set(content) response_dict = { 'content_size': content_size, - 'content_md5': content_md5 + 'content_sha256': content_sha256 } response_body = json.dumps(response_dict, indent=2) diff --git a/tests/endtoend/blob_functions/put_blob_as_str_return_http_response/main.py b/tests/endtoend/blob_functions/put_blob_as_str_return_http_response/main.py index 4e16c84a..3174d3cf 100644 --- a/tests/endtoend/blob_functions/put_blob_as_str_return_http_response/main.py +++ b/tests/endtoend/blob_functions/put_blob_as_str_return_http_response/main.py @@ -21,14 +21,14 @@ def main(req: azf.HttpRequest, file: azf.Out[str]) -> azf.HttpResponse: k=num_chars)) content_bytes = content.encode('utf-8') content_size = len(content_bytes) - content_md5 = hashlib.md5(content_bytes).hexdigest() + content_sha256 = hashlib.sha256(content_bytes).hexdigest() file.set(content) response_dict = { 'num_chars': num_chars, 'content_size': content_size, - 'content_md5': content_md5 + 'content_sha256': content_sha256 } response_body = json.dumps(response_dict, indent=2) diff --git a/tests/endtoend/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/main.py b/tests/endtoend/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/main.py index 9d2811e3..95710c9c 100644 --- a/tests/endtoend/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/main.py +++ b/tests/endtoend/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/main.py @@ -10,8 +10,8 @@ def _generate_content_and_digest(content_size): content = bytearray(random.getrandbits(8) for _ in range(content_size)) - content_md5 = hashlib.md5(content).hexdigest() - return content, content_md5 + content_sha256 = hashlib.sha256(content).hexdigest() + return content, content_sha256 def main( @@ -30,15 +30,15 @@ def main( input_content_size_1 = len(inputfile1) input_content_size_2 = len(inputfile2) - input_content_md5_1 = hashlib.md5(inputfile1).hexdigest() - input_content_md5_2 = hashlib.md5(inputfile2).hexdigest() + input_content_sha256_1 = hashlib.sha256(inputfile1).hexdigest() + input_content_sha256_2 = hashlib.sha256(inputfile2).hexdigest() output_content_size_1 = int(req.params['output_content_size_1']) output_content_size_2 = int(req.params['output_content_size_2']) - output_content_1, output_content_md5_1 = \ + output_content_1, output_content_sha256_1 = \ _generate_content_and_digest(output_content_size_1) - output_content_2, output_content_md5_2 = \ + output_content_2, output_content_sha256_2 = \ _generate_content_and_digest(output_content_size_2) outputfile1.set(output_content_1) @@ -47,12 +47,12 @@ def main( response_dict = { 'input_content_size_1': input_content_size_1, 'input_content_size_2': input_content_size_2, - 'input_content_md5_1': input_content_md5_1, - 'input_content_md5_2': input_content_md5_2, + 'input_content_sha256_1': input_content_sha256_1, + 'input_content_sha256_2': input_content_sha256_2, 'output_content_size_1': output_content_size_1, 'output_content_size_2': output_content_size_2, - 'output_content_md5_1': output_content_md5_1, - 'output_content_md5_2': output_content_md5_2 + 'output_content_sha256_1': output_content_sha256_1, + 'output_content_sha256_2': output_content_sha256_2 } response_body = json.dumps(response_dict, indent=2) diff --git a/tests/unittests/test_mock_blob_shared_memory_functions.py b/tests/unittests/test_mock_blob_shared_memory_functions.py index cdbbfc92..f1154542 100644 --- a/tests/unittests/test_mock_blob_shared_memory_functions.py +++ b/tests/unittests/test_mock_blob_shared_memory_functions.py @@ -78,12 +78,12 @@ async def test_binary_blob_write_function(self): self.assertEqual(protos.StatusResult.Success, response_msg.response.result.status) - # The function responds back in the HTTP body with the md5 digest of + # The function responds back in the HTTP body with the sha256 digest of # the output it created along with its size response_bytes = response_msg.response.return_value.http.body.bytes json_response = json.loads(response_bytes) func_created_content_size = json_response['content_size'] - func_created_content_md5 = json_response['content_md5'] + func_created_content_sha256 = json_response['content_sha256'] # Verify if the worker produced an output blob which was written # in shared memory @@ -122,8 +122,8 @@ async def test_binary_blob_write_function(self): # Verify if we were able to read the correct output that the # function has produced - read_content_md5 = hashlib.md5(read_content).hexdigest() - self.assertEqual(func_created_content_md5, read_content_md5) + read_content_sha256 = hashlib.sha256(read_content).hexdigest() + self.assertEqual(func_created_content_sha256, read_content_sha256) self.assertEqual(len(read_content), func_created_content_size) async def test_str_blob_read_function(self): @@ -144,7 +144,7 @@ async def test_str_blob_read_function(self): num_chars = int(content_size / consts.SIZE_OF_CHAR_BYTES) content = self.get_random_string(num_chars) content_bytes = content.encode('utf-8') - content_md5 = hashlib.md5(content_bytes).hexdigest() + content_sha256 = hashlib.sha256(content_bytes).hexdigest() mem_map_size = consts.CONTENT_HEADER_TOTAL_BYTES + content_size mem_map = self.file_accessor.create_mem_map(mem_map_name, mem_map_size) @@ -187,12 +187,12 @@ async def test_str_blob_read_function(self): response_bytes = response_msg.response.return_value.http.body.bytes json_response = json.loads(response_bytes) func_received_num_chars = json_response['num_chars'] - func_received_content_md5 = json_response['content_md5'] + func_received_content_sha256 = json_response['content_sha256'] # Check the function response to ensure that it read the complete - # input that we provided and the md5 matches + # input that we provided and the sha256 matches self.assertEqual(num_chars, func_received_num_chars) - self.assertEqual(content_md5, func_received_content_md5) + self.assertEqual(content_sha256, func_received_content_sha256) async def test_str_blob_write_function(self): """ @@ -226,12 +226,12 @@ async def test_str_blob_write_function(self): self.assertEqual(protos.StatusResult.Success, response_msg.response.result.status) - # The function responds back in the HTTP body with the md5 digest of + # The function responds back in the HTTP body with the sha256 digest of # the output it created along with its size response_bytes = response_msg.response.return_value.http.body.bytes json_response = json.loads(response_bytes) func_created_num_chars = json_response['num_chars'] - func_created_content_md5 = json_response['content_md5'] + func_created_content_sha256 = json_response['content_sha256'] # Verify if the worker produced an output blob which was written # in shared memory @@ -270,8 +270,8 @@ async def test_str_blob_write_function(self): # Verify if we were able to read the correct output that the # function has produced - read_content_md5 = hashlib.md5(read_content_bytes).hexdigest() - self.assertEqual(func_created_content_md5, read_content_md5) + read_content_sha256 = hashlib.sha256(read_content_bytes).hexdigest() + self.assertEqual(func_created_content_sha256, read_content_sha256) read_content = read_content_bytes.decode('utf-8') self.assertEqual(len(read_content), func_created_num_chars) @@ -386,7 +386,7 @@ async def test_multiple_input_output_blobs(self): mem_map_name_1 = self.get_new_mem_map_name() input_content_size_1 = consts.MIN_BYTES_FOR_SHARED_MEM_TRANSFER + 10 input_content_1 = self.get_random_bytes(input_content_size_1) - input_content_md5_1 = hashlib.md5(input_content_1).hexdigest() + input_content_sha256_1 = hashlib.sha256(input_content_1).hexdigest() input_mem_map_size_1 = \ consts.CONTENT_HEADER_TOTAL_BYTES + input_content_size_1 input_mem_map_1 = \ @@ -412,7 +412,7 @@ async def test_multiple_input_output_blobs(self): mem_map_name_2 = self.get_new_mem_map_name() input_content_size_2 = consts.MIN_BYTES_FOR_SHARED_MEM_TRANSFER + 20 input_content_2 = self.get_random_bytes(input_content_size_2) - input_content_md5_2 = hashlib.md5(input_content_2).hexdigest() + input_content_sha256_2 = hashlib.sha256(input_content_2).hexdigest() input_mem_map_size_2 = \ consts.CONTENT_HEADER_TOTAL_BYTES + input_content_size_2 input_mem_map_2 = \ @@ -476,20 +476,20 @@ async def test_multiple_input_output_blobs(self): json_response = json.loads(response_bytes) func_received_content_size_1 = json_response['input_content_size_1'] - func_received_content_md5_1 = json_response['input_content_md5_1'] + func_received_content_sha256_1 = json_response['input_content_sha256_1'] func_received_content_size_2 = json_response['input_content_size_2'] - func_received_content_md5_2 = json_response['input_content_md5_2'] + func_received_content_sha256_2 = json_response['input_content_sha256_2'] func_created_content_size_1 = json_response['output_content_size_1'] func_created_content_size_2 = json_response['output_content_size_2'] - func_created_content_md5_1 = json_response['output_content_md5_1'] - func_created_content_md5_2 = json_response['output_content_md5_2'] + func_created_content_sha256_1 = json_response['output_content_sha256_1'] + func_created_content_sha256_2 = json_response['output_content_sha256_2'] # Check the function response to ensure that it read the complete - # input that we provided and the md5 matches + # input that we provided and the sha256 matches self.assertEqual(input_content_size_1, func_received_content_size_1) - self.assertEqual(input_content_md5_1, func_received_content_md5_1) + self.assertEqual(input_content_sha256_1, func_received_content_sha256_1) self.assertEqual(input_content_size_2, func_received_content_size_2) - self.assertEqual(input_content_md5_2, func_received_content_md5_2) + self.assertEqual(input_content_sha256_2, func_received_content_sha256_2) # Verify if the worker produced two output blobs which were written # in shared memory @@ -503,7 +503,7 @@ async def test_multiple_input_output_blobs(self): shmem_1 = output_binding_1.rpc_shared_memory self._verify_function_output(shmem_1, func_created_content_size_1, - func_created_content_md5_1) + func_created_content_sha256_1) # Output 2 output_binding_2 = output_data[1] @@ -512,7 +512,7 @@ async def test_multiple_input_output_blobs(self): shmem_2 = output_binding_2.rpc_shared_memory self._verify_function_output(shmem_2, func_created_content_size_2, - func_created_content_md5_2) + func_created_content_sha256_2) async def _test_binary_blob_read_function(self, func_name): """ @@ -528,7 +528,7 @@ async def _test_binary_blob_read_function(self, func_name): mem_map_name = self.get_new_mem_map_name() content_size = consts.MIN_BYTES_FOR_SHARED_MEM_TRANSFER + 10 content = self.get_random_bytes(content_size) - content_md5 = hashlib.md5(content).hexdigest() + content_sha256 = hashlib.sha256(content).hexdigest() mem_map_size = consts.CONTENT_HEADER_TOTAL_BYTES + content_size mem_map = self.file_accessor.create_mem_map(mem_map_name, mem_map_size) @@ -571,18 +571,18 @@ async def _test_binary_blob_read_function(self, func_name): response_bytes = response_msg.response.return_value.http.body.bytes json_response = json.loads(response_bytes) func_received_content_size = json_response['content_size'] - func_received_content_md5 = json_response['content_md5'] + func_received_content_sha256 = json_response['content_sha256'] # Check the function response to ensure that it read the complete - # input that we provided and the md5 matches + # input that we provided and the sha256 matches self.assertEqual(content_size, func_received_content_size) - self.assertEqual(content_md5, func_received_content_md5) + self.assertEqual(content_sha256, func_received_content_sha256) def _verify_function_output( self, shmem: protos.RpcSharedMemory, expected_size: int, - expected_md5: str): + expected_sha256: str): """ Verify if the output produced by the worker is what we expect it to be based on the size and MD5 digest. @@ -615,6 +615,6 @@ def _verify_function_output( # Verify if we were able to read the correct output that the # function has produced - output_read_content_md5 = hashlib.md5(output_read_content).hexdigest() - self.assertEqual(expected_md5, output_read_content_md5) + output_read_content_sha256 = hashlib.sha256(output_read_content).hexdigest() + self.assertEqual(expected_sha256, output_read_content_sha256) self.assertEqual(len(output_read_content), expected_size) diff --git a/tests/unittests/test_third_party_http_functions.py b/tests/unittests/test_third_party_http_functions.py index ba1e44f2..8d97af67 100644 --- a/tests/unittests/test_third_party_http_functions.py +++ b/tests/unittests/test_third_party_http_functions.py @@ -5,6 +5,8 @@ import pathlib import re import typing +import urllib.parse + from unittest.mock import patch from tests.utils import testutils @@ -131,14 +133,15 @@ def test_raw_body_bytes(self): image_file = parent_dir / 'unittests/resources/functions.png' with open(image_file, 'rb') as image: img = image.read() - img_len = len(img) + sanitized_image = urllib.parse.quote(img) + sanitized_img_len = len(sanitized_image) r = self.webhost.request('POST', 'raw_body_bytes', data=img, no_prefix=True) received_body_len = int(r.headers['body-len']) - self.assertEqual(received_body_len, img_len) + self.assertEqual(received_body_len, sanitized_img_len) - body = r.content + body = urllib.parse.unquote_to_bytes(r.content) try: received_img_file = parent_dir / 'received_img.png' with open(received_img_file, 'wb') as received_img: diff --git a/tests/unittests/third_party_http_functions/stein/asgi_function/function_app.py b/tests/unittests/third_party_http_functions/stein/asgi_function/function_app.py index 3248f25f..ab7e060d 100644 --- a/tests/unittests/third_party_http_functions/stein/asgi_function/function_app.py +++ b/tests/unittests/third_party_http_functions/stein/asgi_function/function_app.py @@ -1,7 +1,9 @@ import asyncio import logging +import re import sys from urllib.request import urlopen +import urllib.parse import azure.functions as func from fastapi import FastAPI, Request, Response @@ -131,7 +133,9 @@ async def print_logging(message: str = "", flush: str = 'false', @fast_app.post("/raw_body_bytes") async def raw_body_bytes(request: Request): raw_body = await request.body() - return Response(content=raw_body, headers={'body-len': str(len(raw_body))}) + sanitized_body = urllib.parse.quote(raw_body) + return Response(content=sanitized_body, + headers={'body-len': str(len(sanitized_body))}) @fast_app.get("/return_http_no_body") @@ -146,10 +150,17 @@ async def return_http(request: Request): @fast_app.get("/return_http_redirect") async def return_http_redirect(request: Request, code: str = ''): + allowed_url_pattern = r"^http://.+" + location = 'return_http?code={}'.format(code) + redirect_url = f"http://{request.url.components[1]}/{location}" + if re.match(allowed_url_pattern, redirect_url): + # Redirect URL is in the expected format + return RedirectResponse(status_code=302, + url=redirect_url) + # Redirect URL was not in the expected format return RedirectResponse(status_code=302, - url=f"http://{request.url.components[1]}/" - f"{location}") + url='/') @fast_app.get("/unhandled_error") diff --git a/tests/unittests/third_party_http_functions/stein/wsgi_function/function_app.py b/tests/unittests/third_party_http_functions/stein/wsgi_function/function_app.py index 7605b4b1..5f1ec8e0 100644 --- a/tests/unittests/third_party_http_functions/stein/wsgi_function/function_app.py +++ b/tests/unittests/third_party_http_functions/stein/wsgi_function/function_app.py @@ -1,6 +1,7 @@ import logging import sys from urllib.request import urlopen +import urllib.parse import azure.functions as func from flask import Flask, Response, redirect, request, url_for @@ -61,7 +62,8 @@ def print_logging(): def raw_body_bytes(): body = request.get_data() - return Response(body, headers={'body-len': str(len(body))}) + sanitized_body = urllib.parse.quote(body) + return Response(sanitized_body, headers={'body-len': str(len(sanitized_body))}) @flask_app.get("/return_http_no_body") From ac11e210f56343e4e530f9be6b19e433b69cbe90 Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Mon, 2 Dec 2024 16:50:31 -0600 Subject: [PATCH 06/14] Update Python SDK Version to 1.23.0b1 (#1616) Co-authored-by: AzureFunctionsPython --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index dfde4254..6acc3598 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ "Intended Audience :: Developers" ] dependencies = [ - "azure-functions==1.22.0b4", + "azure-functions==1.23.0b1", "python-dateutil ~=2.9.0", "protobuf~=3.19.3; python_version == '3.7'", "protobuf~=4.25.3; python_version >= '3.8'", From 114be16ab5191dbd797ad3207206680c6c3abd50 Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Tue, 3 Dec 2024 11:26:53 -0600 Subject: [PATCH 07/14] fix: remaining CodeQL fixes (#1612) * CodeQL fixes * asgi function app fixes * revert cryptography changes * comment out edited tests * re-add changes * fix console logging tests * fix tests --------- Co-authored-by: Victoria Hall Co-authored-by: Gavin Aguiar <80794152+gavin-aguiar@users.noreply.github.com> --- pyproject.toml | 4 +- tests/unittests/test_http_functions.py | 14 ++++--- tests/unittests/test_http_functions_v2.py | 10 +++-- .../unittests/test_log_filtering_functions.py | 16 ++++---- .../test_third_party_http_functions.py | 29 ++++++++----- .../stein/asgi_function/function_app.py | 41 +++++++++++++------ .../stein/wsgi_function/function_app.py | 9 ++-- tests/utils/testutils_lc.py | 36 ++++++++++++---- 8 files changed, 103 insertions(+), 56 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6acc3598..e5de9624 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,6 @@ dev = [ "flask", "fastapi~=0.103.2", "pydantic", - "pycryptodome==3.*", "flake8==5.*; python_version == '3.7'", "flake8==6.*; python_version >= '3.8'", "mypy", @@ -69,7 +68,8 @@ dev = [ "pandas", "numpy", "pre-commit", - "invoke" + "invoke", + "cryptography" ] test-http-v2 = [ "azurefunctions-extensions-http-fastapi", diff --git a/tests/unittests/test_http_functions.py b/tests/unittests/test_http_functions.py index 03e0b580..cfa46be5 100644 --- a/tests/unittests/test_http_functions.py +++ b/tests/unittests/test_http_functions.py @@ -415,9 +415,11 @@ def test_response_cookie_header_nullable_double_err(self): self.assertFalse("Set-Cookie" in r.headers) def check_log_print_to_console_stdout(self, host_out: typing.List[str]): - # System logs stdout should not exist in host_out - self.assertNotIn('Secret42', host_out) + # System logs stdout should exist in host_out + self.assertIn('Secret42', host_out) + @skipIf(sys.version_info < (3, 9, 0), + "Skip the tests for Python 3.8 and below") def test_print_to_console_stderr(self): r = self.webhost.request('GET', 'print_logging?console=true' '&message=Secret42&is_stderr=true') @@ -425,8 +427,8 @@ def test_print_to_console_stderr(self): self.assertEqual(r.text, 'OK-print-logging') def check_log_print_to_console_stderr(self, host_out: typing.List[str], ): - # System logs stderr should not exist in host_out - self.assertNotIn('Secret42', host_out) + # System logs stderr should exist in host_out + self.assertIn('Secret42', host_out) def test_hijack_current_event_loop(self): r = self.webhost.request('GET', 'hijack_current_event_loop/') @@ -443,8 +445,8 @@ def check_log_hijack_current_event_loop(self, host_out: typing.List[str]): self.assertIn('parallelly_log_custom at custom_logger', host_out) self.assertIn('callsoon_log', host_out) - # System logs should not exist in host_out - self.assertNotIn('parallelly_log_system at disguised_logger', host_out) + # System logs should exist in host_out + self.assertIn('parallelly_log_system at disguised_logger', host_out) @skipIf(sys.version_info.minor < 11, "The context param is only available for 3.11+") diff --git a/tests/unittests/test_http_functions_v2.py b/tests/unittests/test_http_functions_v2.py index 17af822c..6aebad6a 100644 --- a/tests/unittests/test_http_functions_v2.py +++ b/tests/unittests/test_http_functions_v2.py @@ -392,6 +392,8 @@ def test_response_cookie_header_nullable_bool_err(self): self.assertEqual(r.status_code, 200) self.assertTrue("Set-Cookie" in r.headers) + @skipIf(sys.version_info < (3, 9, 0), + "Skip the tests for Python 3.8 and below") def test_print_to_console_stderr(self): r = self.webhost.request('GET', 'print_logging?console=true' '&message=Secret42&is_stderr=true') @@ -399,8 +401,8 @@ def test_print_to_console_stderr(self): self.assertEqual(r.text, '"OK-print-logging"') def check_log_print_to_console_stderr(self, host_out: typing.List[str], ): - # System logs stderr should not exist in host_out - self.assertNotIn('Secret42', host_out) + # System logs stderr now exist in host_out + self.assertIn('Secret42', host_out) def test_hijack_current_event_loop(self): r = self.webhost.request('GET', 'hijack_current_event_loop/') @@ -417,8 +419,8 @@ def check_log_hijack_current_event_loop(self, host_out: typing.List[str]): self.assertIn('parallelly_log_custom at custom_logger', host_out) self.assertIn('callsoon_log', host_out) - # System logs should not exist in host_out - self.assertNotIn('parallelly_log_system at disguised_logger', host_out) + # System logs now exist in host_out + self.assertIn('parallelly_log_system at disguised_logger', host_out) def test_no_type_hint(self): r = self.webhost.request('GET', 'no_type_hint') diff --git a/tests/unittests/test_log_filtering_functions.py b/tests/unittests/test_log_filtering_functions.py index 70579473..3d074316 100644 --- a/tests/unittests/test_log_filtering_functions.py +++ b/tests/unittests/test_log_filtering_functions.py @@ -83,10 +83,10 @@ def test_info_with_sdk_logging(self): def check_log_info_with_sdk_logging(self, host_out: typing.List[str]): # See TestLogFilteringFunctions docstring - # System log should not be captured in console - self.assertNotIn('sdk_logger info', host_out) - self.assertNotIn('sdk_logger warning', host_out) - self.assertNotIn('sdk_logger error', host_out) + # System log should be captured in console + self.assertIn('sdk_logger info', host_out) + self.assertIn('sdk_logger warning', host_out) + self.assertIn('sdk_logger error', host_out) self.assertNotIn('sdk_logger debug', host_out) def test_info_with_sdk_submodule_logging(self): @@ -101,8 +101,8 @@ def test_info_with_sdk_submodule_logging(self): def check_log_info_with_sdk_submodule_logging(self, host_out: typing.List[str]): # See TestLogFilteringFunctions docstring - # System log should not be captured in console - self.assertNotIn('sdk_submodule_logger info', host_out) - self.assertNotIn('sdk_submodule_logger warning', host_out) - self.assertNotIn('sdk_submodule_logger error', host_out) + # System log should be captured in console + self.assertIn('sdk_submodule_logger info', host_out) + self.assertIn('sdk_submodule_logger warning', host_out) + self.assertIn('sdk_submodule_logger error', host_out) self.assertNotIn('sdk_submodule_logger debug', host_out) diff --git a/tests/unittests/test_third_party_http_functions.py b/tests/unittests/test_third_party_http_functions.py index 8d97af67..73aca898 100644 --- a/tests/unittests/test_third_party_http_functions.py +++ b/tests/unittests/test_third_party_http_functions.py @@ -5,8 +5,10 @@ import pathlib import re import typing -import urllib.parse +import base64 +import sys +from unittest import skipIf from unittest.mock import patch from tests.utils import testutils @@ -113,9 +115,11 @@ def test_print_to_console_stdout(self): def check_log_print_to_console_stdout(self, host_out: typing.List[str]): - # System logs stdout should not exist in host_out - self.assertNotIn('Secret42', host_out) + # System logs stdout now exist in host_out + self.assertIn('Secret42', host_out) + @skipIf(sys.version_info < (3, 9, 0), + "Skip the tests for Python 3.8 and below") def test_print_to_console_stderr(self): r = self.webhost.request('GET', 'print_logging?console=true' '&message=Secret42&is_stderr=true', @@ -125,23 +129,26 @@ def test_print_to_console_stderr(self): def check_log_print_to_console_stderr(self, host_out: typing.List[str], ): - # System logs stderr should not exist in host_out - self.assertNotIn('Secret42', host_out) + # System logs stderr now exist in host_out + self.assertIn('Secret42', host_out) def test_raw_body_bytes(self): parent_dir = pathlib.Path(__file__).parent.parent image_file = parent_dir / 'unittests/resources/functions.png' with open(image_file, 'rb') as image: img = image.read() - sanitized_image = urllib.parse.quote(img) - sanitized_img_len = len(sanitized_image) + encoded_image = base64.b64encode(img).decode('utf-8') + html_img_tag = \ + f'PNG Image' # noqa + sanitized_img_len = len(html_img_tag) r = self.webhost.request('POST', 'raw_body_bytes', data=img, no_prefix=True) received_body_len = int(r.headers['body-len']) self.assertEqual(received_body_len, sanitized_img_len) - body = urllib.parse.unquote_to_bytes(r.content) + encoded_image_data = encoded_image.split(",")[0] + body = base64.b64decode(encoded_image_data) try: received_img_file = parent_dir / 'received_img.png' with open(received_img_file, 'wb') as received_img: @@ -217,9 +224,9 @@ def check_log_hijack_current_event_loop(self, self.assertIn('parallelly_log_custom at custom_logger', host_out) self.assertIn('callsoon_log', host_out) - # System logs should not exist in host_out - self.assertNotIn('parallelly_log_system at disguised_logger', - host_out) + # System logs now exist in host_out + self.assertIn('parallelly_log_system at disguised_logger', + host_out) class TestWsgiHttpFunctions( diff --git a/tests/unittests/third_party_http_functions/stein/asgi_function/function_app.py b/tests/unittests/third_party_http_functions/stein/asgi_function/function_app.py index ab7e060d..916b5d86 100644 --- a/tests/unittests/third_party_http_functions/stein/asgi_function/function_app.py +++ b/tests/unittests/third_party_http_functions/stein/asgi_function/function_app.py @@ -3,7 +3,7 @@ import re import sys from urllib.request import urlopen -import urllib.parse +import base64 import azure.functions as func from fastapi import FastAPI, Request, Response @@ -132,10 +132,13 @@ async def print_logging(message: str = "", flush: str = 'false', @fast_app.post("/raw_body_bytes") async def raw_body_bytes(request: Request): - raw_body = await request.body() - sanitized_body = urllib.parse.quote(raw_body) - return Response(content=sanitized_body, - headers={'body-len': str(len(sanitized_body))}) + body = await request.body() + + base64_encoded = base64.b64encode(body).decode('utf-8') + html_img_tag = \ + f'PNG Image' + + return Response(html_img_tag, headers={'body-len': str(len(html_img_tag))}) @fast_app.get("/return_http_no_body") @@ -150,17 +153,29 @@ async def return_http(request: Request): @fast_app.get("/return_http_redirect") async def return_http_redirect(request: Request, code: str = ''): - allowed_url_pattern = r"^http://.+" + # Expected format: 127.0.0.1: + host_and_port = request.url.components[1] + + # Validate to ensure it's a valid host and port structure + match = re.match(r'^127\.0\.0\.1:(\d+)$', host_and_port) + if not match: + return Response("Invalid request", status_code=400) + + # Validate port is within specific range + port = int(match.group(1)) + if port < 50000 or port > 65999: + return Response("Invalid port", status_code=400) + + # Validate the code param + allowed_codes = ['', 'testFunctionKey'] + if code not in allowed_codes: + return Response("Invalid code", status_code=400) + # Return after all validation succeeds location = 'return_http?code={}'.format(code) - redirect_url = f"http://{request.url.components[1]}/{location}" - if re.match(allowed_url_pattern, redirect_url): - # Redirect URL is in the expected format - return RedirectResponse(status_code=302, - url=redirect_url) - # Redirect URL was not in the expected format return RedirectResponse(status_code=302, - url='/') + url=f"http://{host_and_port}/" + f"{location}") @fast_app.get("/unhandled_error") diff --git a/tests/unittests/third_party_http_functions/stein/wsgi_function/function_app.py b/tests/unittests/third_party_http_functions/stein/wsgi_function/function_app.py index 5f1ec8e0..e717f395 100644 --- a/tests/unittests/third_party_http_functions/stein/wsgi_function/function_app.py +++ b/tests/unittests/third_party_http_functions/stein/wsgi_function/function_app.py @@ -1,7 +1,7 @@ import logging import sys from urllib.request import urlopen -import urllib.parse +import base64 import azure.functions as func from flask import Flask, Response, redirect, request, url_for @@ -62,8 +62,11 @@ def print_logging(): def raw_body_bytes(): body = request.get_data() - sanitized_body = urllib.parse.quote(body) - return Response(sanitized_body, headers={'body-len': str(len(sanitized_body))}) + base64_encoded = base64.b64encode(body).decode('utf-8') + html_img_tag = \ + f'PNG Image' + + return Response(html_img_tag, headers={'body-len': str(len(html_img_tag))}) @flask_app.get("/return_http_no_body") diff --git a/tests/utils/testutils_lc.py b/tests/utils/testutils_lc.py index d0065d97..43665a65 100644 --- a/tests/utils/testutils_lc.py +++ b/tests/utils/testutils_lc.py @@ -16,9 +16,11 @@ from zipfile import ZipFile import requests -from Crypto.Cipher import AES -from Crypto.Hash.SHA256 import SHA256Hash -from Crypto.Util.Padding import pad +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives import padding + from tests.utils.constants import PROJECT_ROOT # Linux Consumption Testing Constants @@ -287,19 +289,35 @@ def _encrypt_context(cls, encryption_key: str, plain_text: str) -> str: """Encrypt plain text context into a encrypted message which can be accepted by the host """ + # Decode the encryption key encryption_key_bytes = base64.b64decode(encryption_key.encode()) - plain_text_bytes = pad(plain_text.encode(), 16) + + # Pad the plaintext to be a multiple of the AES block size + padder = padding.PKCS7(algorithms.AES.block_size).padder() + plain_text_bytes = padder.update(plain_text.encode()) + padder.finalize() + + # Initialization vector (IV) (fixed value for simplicity) iv_bytes = '0123456789abcedf'.encode() - # Start encryption - cipher = AES.new(encryption_key_bytes, AES.MODE_CBC, iv=iv_bytes) - encrypted_bytes = cipher.encrypt(plain_text_bytes) + # Create AES cipher with CBC mode + cipher = Cipher(algorithms.AES(encryption_key_bytes), + modes.CBC(iv_bytes), backend=default_backend()) - # Prepare final result + # Perform encryption + encryptor = cipher.encryptor() + encrypted_bytes = encryptor.update(plain_text_bytes) + encryptor.finalize() + + # Compute SHA256 hash of the encryption key + digest = hashes.Hash(hashes.SHA256(), backend=default_backend()) + digest.update(encryption_key_bytes) + key_sha256 = digest.finalize() + + # Encode IV, encrypted message, and SHA256 hash in base64 iv_base64 = base64.b64encode(iv_bytes).decode() encrypted_base64 = base64.b64encode(encrypted_bytes).decode() - key_sha256 = SHA256Hash(encryption_key_bytes).digest() key_sha256_base64 = base64.b64encode(key_sha256).decode() + + # Return the final result return f'{iv_base64}.{encrypted_base64}.{key_sha256_base64}' def __enter__(self): From 9151ecde47eefa95bd4ec64b5c55692c3235a3b3 Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:54:22 -0600 Subject: [PATCH 08/14] test: test fixes & vulnerability scan (#1594) * pin to extension versions when testing * add eg sourced blob functions * skipping dependency isolation test when using sdk artifact * reference before assignment * run vulnerability scan * feedback --------- Co-authored-by: Victoria Hall --- README.md | 33 ++++++------ eng/templates/jobs/build.yml | 6 ++- eng/templates/official/jobs/ci-e2e-tests.yml | 21 +++++++- pack/templates/macos_64_env_gen.yml | 5 ++ pack/templates/nix_env_gen.yml | 5 ++ pack/templates/win_env_gen.yml | 5 ++ pyproject.toml | 4 +- requirements.txt | 3 +- .../blob_functions_stein/function_app.py | 52 +++++++++++++++++++ .../test_dependency_isolation_functions.py | 2 +- 10 files changed, 112 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index a889aef3..08baeb6d 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,34 @@ # Functions Header Image - Lightning Logo Azure Functions Python Worker -| Branch | Status | CodeCov | Unittests | E2E tests | -|--------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------| -| main | [![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=main)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=main) | [![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/main/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker) | ![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=main) | ![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=main) | -| dev | [![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=dev)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=dev) | [![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/dev/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker) | ![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=dev) | ![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=dev) | +| Branch | Build Status | CodeCov | Test Status | +|--------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| dev | [![Build Status](https://img.shields.io/azure-devops/build/azfunc/public/658/dev)](https://azfunc.visualstudio.com/public/_build/latest?definitionId=658&branchName=dev) | [![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/dev/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker) | [![Test Status](https://img.shields.io/azure-devops/build/azfunc/public/658/dev)](https://azfunc.visualstudio.com/public/_build/latest?definitionId=658&branchName=dev) | -Python support for Azure Functions is based on Python 3.6, 3.7, 3.8, 3.9, and 3.10 serverless hosting on Linux and the Functions 2.0, 3.0 and 4.0 runtime. +Python support for Azure Functions is based on Python 3.8, 3.9, 3.10, 3.11, and 3.12 serverless hosting on Linux and the Functions 4.0 runtime. Here is the current status of Python in Azure Functions: What are the supported Python versions? -| Azure Functions Runtime | Python 3.6 | Python 3.7 | Python 3.8 | Python 3.9 | Python 3.10 | Python 3.11 | -|----------------------------------|------------|------------|------------|------------|-------------|-------------| -| Azure Functions 2.0 (deprecated) | ✔ | ✔ | - | - | - | - | -| Azure Functions 3.0 (deprecated) | ✔ | ✔ | ✔ | ✔ | - | - | -| Azure Functions 4.0 | - | - | ✔ | ✔ | ✔ | ✔ | +| Azure Functions Runtime | Python 3.8 | Python 3.9 | Python 3.10 | Python 3.11 | Python 3.12 | +|----------------------------------|------------|------------|-------------|-------------|-------------| +| Azure Functions 3.0 (deprecated) | ✔ | ✔ | - | - | - | +| Azure Functions 4.0 | ✔ | ✔ | ✔ | ✔ | ✔ | For information about Azure Functions Runtime, please refer to [Azure Functions runtime versions overview](https://docs.microsoft.com/en-us/azure/azure-functions/functions-versions) page. ### What's available? -- Build, test, debug and publish using Azure Functions Core Tools (CLI) or Visual Studio Code -- Deploy Python Function project onto consumption, dedicated, or elastic premium plan. -- Deploy Python Function project in a custom docker image onto dedicated, or elastic premium plan. -- Triggers / Bindings : HTTP, Blob, Queue, Timer, Cosmos DB, Event Grid, Event Hubs and Service Bus +- Build, test, debug, and publish using Azure Functions Core Tools (CLI) or Visual Studio Code +- Deploy Python Function project onto consumption, dedicated, elastic premium, or flex consumption plan. +- Deploy Python Function project in a custom docker image onto dedicated or elastic premium plan. +- Triggers / Bindings : Blob, Cosmos DB, Event Grid, Event Hub, HTTP, Kafka, MySQL, Queue, ServiceBus, SQL, Timer, and Warmup - Triggers / Bindings : Custom binding support -What's coming? +### What's new? -- [Durable Functions For Python](https://github.com/Azure/azure-functions-durable-python) +- [SDK Type Bindings for Blob](https://techcommunity.microsoft.com/t5/azure-compute-blog/azure-functions-sdk-type-bindings-for-azure-blob-storage-with/ba-p/4146744) +- [HTTP Streaming](https://techcommunity.microsoft.com/t5/azure-compute-blog/azure-functions-support-for-http-streams-in-python-is-now-in/ba-p/4146697) ### Get Started @@ -72,4 +71,4 @@ provided by the bot. You will only need to do this once across all repos using o This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or -contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. \ No newline at end of file diff --git a/eng/templates/jobs/build.yml b/eng/templates/jobs/build.yml index 3b0500df..dd422f4f 100644 --- a/eng/templates/jobs/build.yml +++ b/eng/templates/jobs/build.yml @@ -21,4 +21,8 @@ jobs: python -m pip install . displayName: 'Build python worker' # Skip the build stage for SDK and Extensions release branches. This stage will fail because pyproject.toml contains the updated (and unreleased) library version - condition: and(eq(variables.isSdkRelease, false), eq(variables.isExtensionsRelease, false), eq(variables['USETESTPYTHONSDK'], false), eq(variables['USETESTPYTHONEXTENSIONS'], false)) \ No newline at end of file + condition: and(eq(variables.isSdkRelease, false), eq(variables.isExtensionsRelease, false), eq(variables['USETESTPYTHONSDK'], false), eq(variables['USETESTPYTHONEXTENSIONS'], false)) + - bash: | + pip install pip-audit + pip-audit -r requirements.txt + displayName: 'Run vulnerability scan' \ No newline at end of file diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml index b69d9090..b3ff4c57 100644 --- a/eng/templates/official/jobs/ci-e2e-tests.yml +++ b/eng/templates/official/jobs/ci-e2e-tests.yml @@ -115,6 +115,25 @@ jobs: eng/scripts/test-setup.sh displayName: 'Install test python extension, dependencies and the worker' condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) + - powershell: | + $pipelineVarSet = "$(USETESTPYTHONSDK)" + Write-Host "pipelineVarSet: $pipelineVarSet" + $branch = "$(Build.SourceBranch)" + Write-Host "Branch: $branch" + if($branch.StartsWith("refs/heads/sdk/") -or $pipelineVarSet -eq "true") + { + Write-Host "##vso[task.setvariable variable=skipTest;]true" + } + else + { + Write-Host "##vso[task.setvariable variable=skipTest;]false" + } + displayName: 'Set skipTest variable' + condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) + - powershell: | + Write-Host "skipTest: $(skipTest)" + displayName: 'Display skipTest variable' + condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) - bash: | python -m pytest -q -n auto --dist loadfile --reruns 4 --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend tests/extension_tests/deferred_bindings_tests tests/extension_tests/http_v2_tests env: @@ -125,5 +144,5 @@ jobs: AzureWebJobsSqlConnectionString: $(SQL_CONNECTION) AzureWebJobsEventGridTopicUri: $(EVENTGRID_URI) AzureWebJobsEventGridConnectionKey: $(EVENTGRID_CONNECTION) - USETESTPYTHONSDK: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) + skipTest: $(skipTest) displayName: "Running $(PYTHON_VERSION) Python E2E Tests" diff --git a/pack/templates/macos_64_env_gen.yml b/pack/templates/macos_64_env_gen.yml index 8fb49f73..9bf2027a 100644 --- a/pack/templates/macos_64_env_gen.yml +++ b/pack/templates/macos_64_env_gen.yml @@ -12,6 +12,10 @@ steps: inputs: disableAutoCwd: true scriptPath: 'pack/scripts/mac_arm64_deps.sh' +- bash: | + pip install pip-audit + pip-audit -r requirements.txt + displayName: 'Run vulnerability scan' - task: CopyFiles@2 inputs: contents: | @@ -35,4 +39,5 @@ steps: !distutils-precedence.pth !pkg_resources/** !*.dist-info/** + !werkzeug/debug/shared/debugger.js targetFolder: '$(Build.ArtifactStagingDirectory)' diff --git a/pack/templates/nix_env_gen.yml b/pack/templates/nix_env_gen.yml index 7c2b6870..b89d4813 100644 --- a/pack/templates/nix_env_gen.yml +++ b/pack/templates/nix_env_gen.yml @@ -12,6 +12,10 @@ steps: inputs: disableAutoCwd: true scriptPath: 'pack/scripts/nix_deps.sh' +- bash: | + pip install pip-audit + pip-audit -r requirements.txt + displayName: 'Run vulnerability scan' - task: CopyFiles@2 inputs: contents: | @@ -35,4 +39,5 @@ steps: !distutils-precedence.pth !pkg_resources/** !*.dist-info/** + !werkzeug/debug/shared/debugger.js targetFolder: '$(Build.ArtifactStagingDirectory)' diff --git a/pack/templates/win_env_gen.yml b/pack/templates/win_env_gen.yml index 0ae4f70e..8e9b0321 100644 --- a/pack/templates/win_env_gen.yml +++ b/pack/templates/win_env_gen.yml @@ -12,6 +12,10 @@ steps: - task: PowerShell@2 inputs: filePath: 'pack\scripts\win_deps.ps1' +- bash: | + pip install pip-audit + pip-audit -r requirements.txt + displayName: 'Run vulnerability scan' - task: CopyFiles@2 inputs: contents: | @@ -35,4 +39,5 @@ steps: !distutils-precedence.pth !pkg_resources\** !*.dist-info\** + !werkzeug\debug\shared\debugger.js targetFolder: '$(Build.ArtifactStagingDirectory)' diff --git a/pyproject.toml b/pyproject.toml index e5de9624..c7f41970 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,12 +72,12 @@ dev = [ "cryptography" ] test-http-v2 = [ - "azurefunctions-extensions-http-fastapi", + "azurefunctions-extensions-http-fastapi==1.0.0b1", "ujson", "orjson" ] test-deferred-bindings = [ - "azurefunctions-extensions-bindings-blob" + "azurefunctions-extensions-bindings-blob==1.0.0b2" ] [build-system] diff --git a/requirements.txt b/requirements.txt index 6b5f87b7..3fdb69c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -# Please list runtime dependencies in setup.py 'install_requires' and -# 'extras_require'. +# Required dependencies listed in pyproject.toml . diff --git a/tests/endtoend/blob_functions/blob_functions_stein/function_app.py b/tests/endtoend/blob_functions/blob_functions_stein/function_app.py index 2edc1384..1b77b858 100644 --- a/tests/endtoend/blob_functions/blob_functions_stein/function_app.py +++ b/tests/endtoend/blob_functions/blob_functions_stein/function_app.py @@ -391,3 +391,55 @@ def put_get_multiple_blobs_as_bytes_return_http_response( mimetype="application/json", status_code=200 ) + + +@app.function_name(name="blob_trigger_default_source_enum") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source=func.BlobSource.LOGS_AND_CONTAINER_SCAN) +def blob_trigger_default_source_enum(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) + + +@app.function_name(name="blob_trigger_eventgrid_source_enum") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source=func.BlobSource.EVENT_GRID) +def blob_trigger_eventgrid_source_enum(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) + + +@app.function_name(name="blob_trigger_default_source_str") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source="LogsAndContainerScan") +def blob_trigger_default_source_str(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) + + +@app.function_name(name="blob_trigger_eventgrid_source_str") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source="EventGrid") +def blob_trigger_eventgrid_source_str(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) diff --git a/tests/endtoend/test_dependency_isolation_functions.py b/tests/endtoend/test_dependency_isolation_functions.py index ec1cc251..7ada0ae6 100644 --- a/tests/endtoend/test_dependency_isolation_functions.py +++ b/tests/endtoend/test_dependency_isolation_functions.py @@ -121,7 +121,7 @@ def test_paths_resolution(self): ).lower() ) - @skipIf(is_envvar_true('USETESTPYTHONSDK'), + @skipIf(is_envvar_true('skipTest'), 'Running tests using an editable azure-functions package.') def test_loading_libraries_from_customers_package(self): """Since the Python now loaded the customer's dependencies, the From 6f00a89ba7d2d44730d9d92e4ca8665ba861521d Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:56:20 -0600 Subject: [PATCH 09/14] fix: codeql exclusion for third party dependencies (#1617) * codeql exclusion + skip scan for 3.7 * formatting --------- Co-authored-by: Victoria Hall --- eng/ci/official-build.yml | 2 ++ eng/ci/public-build.yml | 1 + pack/templates/macos_64_env_gen.yml | 1 + pack/templates/nix_env_gen.yml | 1 + pack/templates/win_env_gen.yml | 1 + 5 files changed, 6 insertions(+) diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index ab2dc802..52562bc1 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -39,6 +39,8 @@ extends: image: 1es-windows-2022 os: windows sdl: + codeql: + excludePathPatterns: '/deps' codeSignValidation: enabled: true break: true diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index 67559744..618b3a5b 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -41,6 +41,7 @@ extends: compiled: enabled: true # still only runs for default branch runSourceLanguagesInSourceAnalysis: true + excludePathPatterns: '/deps' settings: skipBuildTagsForGitHubPullRequests: ${{ variables['System.PullRequest.IsFork'] }} stages: diff --git a/pack/templates/macos_64_env_gen.yml b/pack/templates/macos_64_env_gen.yml index 9bf2027a..90a3578d 100644 --- a/pack/templates/macos_64_env_gen.yml +++ b/pack/templates/macos_64_env_gen.yml @@ -16,6 +16,7 @@ steps: pip install pip-audit pip-audit -r requirements.txt displayName: 'Run vulnerability scan' + condition: ne(variables['pythonVersion'], '3.7') - task: CopyFiles@2 inputs: contents: | diff --git a/pack/templates/nix_env_gen.yml b/pack/templates/nix_env_gen.yml index b89d4813..ae3cf433 100644 --- a/pack/templates/nix_env_gen.yml +++ b/pack/templates/nix_env_gen.yml @@ -16,6 +16,7 @@ steps: pip install pip-audit pip-audit -r requirements.txt displayName: 'Run vulnerability scan' + condition: ne(variables['pythonVersion'], '3.7') - task: CopyFiles@2 inputs: contents: | diff --git a/pack/templates/win_env_gen.yml b/pack/templates/win_env_gen.yml index 8e9b0321..2eee3411 100644 --- a/pack/templates/win_env_gen.yml +++ b/pack/templates/win_env_gen.yml @@ -16,6 +16,7 @@ steps: pip install pip-audit pip-audit -r requirements.txt displayName: 'Run vulnerability scan' + condition: ne(variables['pythonVersion'], '3.7') - task: CopyFiles@2 inputs: contents: | From 8f730af1a5f6b25bad8b392d6e5cbe12a5bd6cc2 Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:55:36 -0600 Subject: [PATCH 10/14] fix: remove failing codeql tests (#1618) * remove codeql tests, exclude all dependency locations * Fix excludePathPatterns syntax in build filesformatting * Update CodeQL exclude paths in CI files --------- Co-authored-by: Victoria Hall --- eng/ci/official-build.yml | 3 +- eng/ci/public-build.yml | 3 +- .../test_third_party_http_functions.py | 72 +++++++++---------- .../stein/asgi_function/function_app.py | 29 -------- .../stein/wsgi_function/function_app.py | 12 ---- 5 files changed, 40 insertions(+), 79 deletions(-) diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index 52562bc1..cdf7175e 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -40,7 +40,8 @@ extends: os: windows sdl: codeql: - excludePathPatterns: '/deps' + # Exclude dependencies from CodeQL analysis + excludePathPatterns: '/deps,/build' codeSignValidation: enabled: true break: true diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index 618b3a5b..9806a86b 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -41,7 +41,8 @@ extends: compiled: enabled: true # still only runs for default branch runSourceLanguagesInSourceAnalysis: true - excludePathPatterns: '/deps' + # Exclude dependencies from CodeQL analysis + excludePathPatterns: '/deps,/build' settings: skipBuildTagsForGitHubPullRequests: ${{ variables['System.PullRequest.IsFork'] }} stages: diff --git a/tests/unittests/test_third_party_http_functions.py b/tests/unittests/test_third_party_http_functions.py index 73aca898..7dd57e88 100644 --- a/tests/unittests/test_third_party_http_functions.py +++ b/tests/unittests/test_third_party_http_functions.py @@ -132,48 +132,12 @@ def check_log_print_to_console_stderr(self, # System logs stderr now exist in host_out self.assertIn('Secret42', host_out) - def test_raw_body_bytes(self): - parent_dir = pathlib.Path(__file__).parent.parent - image_file = parent_dir / 'unittests/resources/functions.png' - with open(image_file, 'rb') as image: - img = image.read() - encoded_image = base64.b64encode(img).decode('utf-8') - html_img_tag = \ - f'PNG Image' # noqa - sanitized_img_len = len(html_img_tag) - r = self.webhost.request('POST', 'raw_body_bytes', data=img, - no_prefix=True) - - received_body_len = int(r.headers['body-len']) - self.assertEqual(received_body_len, sanitized_img_len) - - encoded_image_data = encoded_image.split(",")[0] - body = base64.b64decode(encoded_image_data) - try: - received_img_file = parent_dir / 'received_img.png' - with open(received_img_file, 'wb') as received_img: - received_img.write(body) - self.assertTrue(filecmp.cmp(received_img_file, image_file)) - finally: - if (os.path.exists(received_img_file)): - os.remove(received_img_file) - def test_return_http_no_body(self): r = self.webhost.request('GET', 'return_http_no_body', no_prefix=True) self.assertEqual(r.text, '') self.assertEqual(r.status_code, 200) - def test_return_http_redirect(self): - r = self.webhost.request('GET', 'return_http_redirect', - no_prefix=True) - self.assertEqual(r.status_code, 200) - self.assertEqual(r.text, '

Hello World™

') - - r = self.webhost.request('GET', 'return_http_redirect', - allow_redirects=False, no_prefix=True) - self.assertEqual(r.status_code, 302) - def test_unhandled_error(self): r = self.webhost.request('GET', 'unhandled_error', no_prefix=True) self.assertEqual(r.status_code, 500) @@ -228,6 +192,32 @@ def check_log_hijack_current_event_loop(self, self.assertIn('parallelly_log_system at disguised_logger', host_out) + def test_raw_body_bytes(self): + parent_dir = pathlib.Path(__file__).parent.parent + image_file = parent_dir / 'unittests/resources/functions.png' + with open(image_file, 'rb') as image: + img = image.read() + encoded_image = base64.b64encode(img).decode('utf-8') + html_img_tag = \ + f'PNG Image' # noqa + sanitized_img_len = len(html_img_tag) + r = self.webhost.request('POST', 'raw_body_bytes', data=img, + no_prefix=True) + + received_body_len = int(r.headers['body-len']) + self.assertEqual(received_body_len, sanitized_img_len) + + encoded_image_data = encoded_image.split(",")[0] + body = base64.b64decode(encoded_image_data) + try: + received_img_file = parent_dir / 'received_img.png' + with open(received_img_file, 'wb') as received_img: + received_img.write(body) + self.assertTrue(filecmp.cmp(received_img_file, image_file)) + finally: + if (os.path.exists(received_img_file)): + os.remove(received_img_file) + class TestWsgiHttpFunctions( ThirdPartyHttpFunctionsTestBase.TestThirdPartyHttpFunctions): @@ -235,3 +225,13 @@ class TestWsgiHttpFunctions( def get_script_dir(cls): return UNIT_TESTS_ROOT / 'third_party_http_functions' / 'stein' / \ 'wsgi_function' + + def test_return_http_redirect(self): + r = self.webhost.request('GET', 'return_http_redirect', + no_prefix=True) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, '

Hello World™

') + + r = self.webhost.request('GET', 'return_http_redirect', + allow_redirects=False, no_prefix=True) + self.assertEqual(r.status_code, 302) diff --git a/tests/unittests/third_party_http_functions/stein/asgi_function/function_app.py b/tests/unittests/third_party_http_functions/stein/asgi_function/function_app.py index 916b5d86..da76f071 100644 --- a/tests/unittests/third_party_http_functions/stein/asgi_function/function_app.py +++ b/tests/unittests/third_party_http_functions/stein/asgi_function/function_app.py @@ -1,13 +1,11 @@ import asyncio import logging -import re import sys from urllib.request import urlopen import base64 import azure.functions as func from fastapi import FastAPI, Request, Response -from fastapi.responses import RedirectResponse fast_app = FastAPI() logger = logging.getLogger("my-function") @@ -151,33 +149,6 @@ async def return_http(request: Request): return Response('

Hello World™

', media_type='text/html') -@fast_app.get("/return_http_redirect") -async def return_http_redirect(request: Request, code: str = ''): - # Expected format: 127.0.0.1: - host_and_port = request.url.components[1] - - # Validate to ensure it's a valid host and port structure - match = re.match(r'^127\.0\.0\.1:(\d+)$', host_and_port) - if not match: - return Response("Invalid request", status_code=400) - - # Validate port is within specific range - port = int(match.group(1)) - if port < 50000 or port > 65999: - return Response("Invalid port", status_code=400) - - # Validate the code param - allowed_codes = ['', 'testFunctionKey'] - if code not in allowed_codes: - return Response("Invalid code", status_code=400) - - # Return after all validation succeeds - location = 'return_http?code={}'.format(code) - return RedirectResponse(status_code=302, - url=f"http://{host_and_port}/" - f"{location}") - - @fast_app.get("/unhandled_error") async def unhandled_error(): 1 / 0 diff --git a/tests/unittests/third_party_http_functions/stein/wsgi_function/function_app.py b/tests/unittests/third_party_http_functions/stein/wsgi_function/function_app.py index e717f395..3d2f63d9 100644 --- a/tests/unittests/third_party_http_functions/stein/wsgi_function/function_app.py +++ b/tests/unittests/third_party_http_functions/stein/wsgi_function/function_app.py @@ -1,7 +1,6 @@ import logging import sys from urllib.request import urlopen -import base64 import azure.functions as func from flask import Flask, Response, redirect, request, url_for @@ -58,17 +57,6 @@ def print_logging(): return 'OK-print-logging' -@flask_app.post("/raw_body_bytes") -def raw_body_bytes(): - body = request.get_data() - - base64_encoded = base64.b64encode(body).decode('utf-8') - html_img_tag = \ - f'PNG Image' - - return Response(html_img_tag, headers={'body-len': str(len(html_img_tag))}) - - @flask_app.get("/return_http_no_body") def return_http_no_body(): return '' From bbb6624035c217b637420d047fb49a6a11684970 Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Fri, 6 Dec 2024 15:40:34 -0600 Subject: [PATCH 11/14] reformat excludePathPatterns (#1619) Co-authored-by: Victoria Hall --- eng/ci/official-build.yml | 5 ++--- eng/ci/public-build.yml | 13 ++++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index cdf7175e..ad332eb2 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -30,6 +30,8 @@ resources: variables: - template: /eng/templates/utils/variables.yml@self - template: /eng/templates/utils/official-variables.yml@self + - name: codeql.excludePathPatterns + value: deps/,build/ extends: template: v1/1ES.Official.PipelineTemplate.yml@1es @@ -39,9 +41,6 @@ extends: image: 1es-windows-2022 os: windows sdl: - codeql: - # Exclude dependencies from CodeQL analysis - excludePathPatterns: '/deps,/build' codeSignValidation: enabled: true break: true diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index 9806a86b..a9854e7f 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -28,6 +28,12 @@ resources: variables: - template: /eng/templates/utils/variables.yml@self + - name: codeql.excludePathPatterns + value: deps/,build/ + - name: codeql.compiled.enabled + value: true + - name: codeql.runSourceLanguagesInSourceAnalysis + value: true extends: template: v1/1ES.Unofficial.PipelineTemplate.yml@1es @@ -36,13 +42,6 @@ extends: name: 1es-pool-azfunc-public image: 1es-windows-2022 os: windows - sdl: - codeql: - compiled: - enabled: true # still only runs for default branch - runSourceLanguagesInSourceAnalysis: true - # Exclude dependencies from CodeQL analysis - excludePathPatterns: '/deps,/build' settings: skipBuildTagsForGitHubPullRequests: ${{ variables['System.PullRequest.IsFork'] }} stages: From 43e5a72e9e61fac463a24eddc84de4cbf291ba7a Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 11 Dec 2024 10:04:13 -0800 Subject: [PATCH 12/14] refactor: Remove custom targets, use 'None' item group (#1472) * Remove custom targets, use 'None' item group * Add runtime specific copy logic * Add underscore to target name --------- Co-authored-by: wangbill <12449837+YunchuWang@users.noreply.github.com> Co-authored-by: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Co-authored-by: Gavin Aguiar <80794152+gavin-aguiar@users.noreply.github.com> --- ...osoft.Azure.Functions.PythonWorker.targets | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/pack/Microsoft.Azure.Functions.PythonWorker.targets b/pack/Microsoft.Azure.Functions.PythonWorker.targets index ba674972..887dc8d5 100644 --- a/pack/Microsoft.Azure.Functions.PythonWorker.targets +++ b/pack/Microsoft.Azure.Functions.PythonWorker.targets @@ -1,25 +1,43 @@ - - - - - - - + + <_PythonWorkerToolsDir>$([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)../tools')) + + + <_PythonSupportedRuntime Include="win-x86" WorkerPath="WINDOWS/X86" /> + <_PythonSupportedRuntime Include="win-x64" WorkerPath="WINDOWS/X64" /> + <_PythonSupportedRuntime Include="linux-x64" WorkerPath="LINUX/X64" /> + <_PythonSupportedRuntime Include="osx-x64" WorkerPath="OSX/X64" /> + <_PythonSupportedRuntime Include="osx-arm64" WorkerPath="OSX/Arm64" /> + + + - + - - + + + + <_PythonWorkerFiles Include="$(_PythonWorkerToolsDir)/**" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" /> + - - + + + + <_PythonWorkersRuntimeFolder>@(_PythonSupportedRuntime->WithMetadataValue('Identity', '$(RuntimeIdentifier)')->Metadata('WorkerPath')) + + + + <_PythonWorkerFiles Include="$(_PythonWorkerToolsDir)/*" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" /> + <_PythonWorkerFiles Include="$(_PythonWorkerToolsDir)/**/$(_PythonWorkersRuntimeFolder)/**" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" /> + + + + + From d7a735abcb651dea02726ed68e294a6c26e3cb36 Mon Sep 17 00:00:00 2001 From: Manvir Kaur <67894494+manvkaur@users.noreply.github.com> Date: Fri, 13 Dec 2024 07:56:58 -0800 Subject: [PATCH 13/14] fix: codeql specify source languages for public build (#1622) * add run source languages var for codeql * revert indentation change --- eng/ci/public-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index a9854e7f..222cdc2d 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -34,6 +34,8 @@ variables: value: true - name: codeql.runSourceLanguagesInSourceAnalysis value: true + - name: codeql.sourceLanguages + value: python, powershell extends: template: v1/1ES.Unofficial.PipelineTemplate.yml@1es From ed26bd064067caa390dc6d32b84112f1dd0a4ff9 Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Fri, 20 Dec 2024 11:47:03 -0600 Subject: [PATCH 14/14] test: run supported tests with emulators (#1564) * run emulators for blob tests * blob, table, queue working. large refactor * directory changes * fix: public build, generic table creation, eh config * docker-compose * emulator support * skip eventhub metadata tests for 37 * run db blob tests in emulator * db blob tests dir change * fix db test directory * merge * run emulator tests for 312 * remove git repo clone * refactor docker-compose, codeowners * retries for flaky tests * update codeowners * run emulator tests on schedule, lowercase config * merge miss * move servicebus tests to emulators * parameter miss * formatting * merge miss * lint * feedback * container name --------- Co-authored-by: Victoria Hall Co-authored-by: wangbill --- CODEOWNERS | 2 +- eng/ci/emulator-tests.yml | 46 + eng/ci/official-build.yml | 6 + eng/ci/public-build.yml | 8 +- eng/templates/jobs/ci-emulator-tests.yml | 100 ++ .../blob_functions_stein/function_app.py | 890 +++++++++--------- .../generic/function_app.py | 0 .../blob_functions/blob_trigger/function.json | 0 .../blob_functions/blob_trigger/main.py | 0 .../get_blob_as_bytes/function.json | 0 .../blob_functions/get_blob_as_bytes/main.py | 0 .../function.json | 0 .../main.py | 0 .../function.json | 0 .../main.py | 0 .../get_blob_as_str/function.json | 0 .../blob_functions/get_blob_as_str/main.py | 0 .../function.json | 0 .../main.py | 0 .../get_blob_bytes/function.json | 0 .../blob_functions/get_blob_bytes/main.py | 0 .../get_blob_filelike/function.json | 0 .../blob_functions/get_blob_filelike/main.py | 0 .../get_blob_return/function.json | 0 .../blob_functions/get_blob_return/main.py | 0 .../blob_functions/get_blob_str/function.json | 0 .../blob_functions/get_blob_str/main.py | 0 .../get_blob_triggered/function.json | 0 .../blob_functions/get_blob_triggered/main.py | 0 .../function.json | 0 .../main.py | 0 .../function.json | 0 .../main.py | 0 .../put_blob_bytes/function.json | 0 .../blob_functions/put_blob_bytes/main.py | 0 .../put_blob_filelike/function.json | 0 .../blob_functions/put_blob_filelike/main.py | 0 .../put_blob_return/function.json | 0 .../blob_functions/put_blob_return/main.py | 0 .../blob_functions/put_blob_str/function.json | 0 .../blob_functions/put_blob_str/main.py | 0 .../put_blob_trigger/function.json | 0 .../blob_functions/put_blob_trigger/main.py | 0 .../function.json | 0 .../main.py | 0 .../function_app.py | 0 .../eventhub_multiple/__init__.py | 0 .../eventhub_multiple/function.json | 0 .../eventhub_output_batch/__init__.py | 0 .../eventhub_output_batch/function.json | 0 .../get_eventhub_batch_triggered/__init__.py | 0 .../function.json | 50 +- .../get_metadata_batch_triggered/__init__.py | 0 .../function.json | 0 .../metadata_multiple/__init__.py | 0 .../metadata_multiple/function.json | 0 .../metadata_output_batch/__init__.py | 0 .../metadata_output_batch/function.json | 0 .../eventhub_functions_stein/function_app.py | 214 ++--- .../generic/function_app.py | 0 .../eventhub_output/__init__.py | 0 .../eventhub_output/function.json | 0 .../eventhub_trigger/__init__.py | 0 .../eventhub_trigger/function.json | 0 .../get_eventhub_triggered/function.json | 0 .../get_eventhub_triggered/main.py | 0 .../get_metadata_triggered/__init__.py | 0 .../get_metadata_triggered/function.json | 0 .../metadata_output/__init__.py | 0 .../metadata_output/function.json | 0 .../metadata_trigger/__init__.py | 0 .../metadata_trigger/function.json | 0 .../generic_functions_stein/function_app.py | 35 +- .../return_bool/function.json | 0 .../generic_functions/return_bool/main.py | 0 .../return_bytes/function.json | 0 .../generic_functions/return_bytes/main.py | 0 .../return_dict/function.json | 0 .../generic_functions/return_dict/main.py | 0 .../return_double/function.json | 0 .../generic_functions/return_double/main.py | 0 .../return_int/function.json | 0 .../generic_functions/return_int/main.py | 0 .../return_list/function.json | 0 .../generic_functions/return_list/main.py | 0 .../return_none/function.json | 0 .../generic_functions/return_none/main.py | 0 .../return_none_no_type_hint/function.json | 0 .../return_none_no_type_hint/main.py | 0 .../return_not_processed_last/__init__.py | 0 .../return_not_processed_last/function.json | 2 +- .../return_processed_last/__init__.py | 0 .../return_processed_last/function.json | 2 +- .../return_string/function.json | 0 .../generic_functions/return_string/main.py | 0 .../table_out_binding/__init__.py | 13 + .../table_out_binding/function.json | 24 + .../get_queue_blob/function.json | 0 .../queue_functions/get_queue_blob/main.py | 0 .../function.json | 0 .../get_queue_blob_message_return/main.py | 0 .../get_queue_blob_return/function.json | 0 .../get_queue_blob_return/main.py | 0 .../function.json | 0 .../get_queue_untyped_blob_return/main.py | 0 .../queue_functions/put_queue/function.json | 0 .../queue_functions/put_queue/main.py | 0 .../put_queue_message_return/function.json | 0 .../put_queue_message_return/main.py | 0 .../put_queue_multiple_out/function.json | 0 .../put_queue_multiple_out/main.py | 0 .../put_queue_return/function.json | 0 .../queue_functions/put_queue_return/main.py | 0 .../put_queue_return_multiple/function.json | 0 .../put_queue_return_multiple/main.py | 0 .../put_queue_untyped_return/function.json | 0 .../put_queue_untyped_return/main.py | 0 .../queue_functions_stein/function_app.py | 370 ++++---- .../generic/function_app.py | 0 .../queue_trigger/function.json | 0 .../queue_functions/queue_trigger/main.py | 0 .../function.json | 0 .../queue_trigger_message_return/main.py | 0 .../queue_trigger_return/function.json | 0 .../queue_trigger_return/main.py | 0 .../function.json | 0 .../queue_trigger_return_multiple/main.py | 0 .../queue_trigger_untyped/function.json | 0 .../queue_trigger_untyped/main.py | 0 .../get_servicebus_triggered/__init__.py | 0 .../get_servicebus_triggered/function.json | 0 .../put_message/__init__.py | 0 .../put_message/function.json | 0 .../function_app.py | 146 +-- .../generic/function_app.py | 162 ++-- .../servicebus_trigger/__init__.py | 0 .../servicebus_trigger/function.json | 0 .../table_functions_stein/function_app.py | 0 .../generic/function_app.py | 0 .../table_in_binding/__init__.py | 0 .../table_in_binding/function.json | 0 .../table_out_binding/__init__.py | 0 .../table_out_binding/function.json | 0 .../test_blob_functions.py | 8 +- .../test_eventhub_batch_functions.py | 12 +- .../test_eventhub_functions.py | 14 +- .../test_generic_functions.py | 10 +- .../test_queue_functions.py | 8 +- .../test_servicebus_functions.py | 11 +- .../test_table_functions.py | 12 +- .../emulator_tests/utils/eventhub/config.json | 51 + .../utils/eventhub/docker-compose.yml | 34 + .../utils/servicebus/config.json | 28 + .../utils/servicebus/docker-compose.yml | 40 + .../test_worker_process_count_functions.py | 19 +- .../http_v2_tests/test_http_v2.py | 5 + tests/unittests/test_http_functions_v2.py | 2 + .../test_mock_blob_shared_memory_functions.py | 2 +- tests/utils/testutils.py | 1 + 159 files changed, 1365 insertions(+), 962 deletions(-) create mode 100644 eng/ci/emulator-tests.yml create mode 100644 eng/templates/jobs/ci-emulator-tests.yml rename tests/{endtoend => emulator_tests}/blob_functions/blob_functions_stein/function_app.py (97%) rename tests/{endtoend => emulator_tests}/blob_functions/blob_functions_stein/generic/function_app.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/blob_trigger/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/blob_trigger/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_bytes/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_bytes/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_bytes_return_http_response/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_bytes_return_http_response/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_bytes_stream_return_http_response/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_bytes_stream_return_http_response/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_str/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_str/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_str_return_http_response/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_str_return_http_response/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_bytes/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_bytes/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_filelike/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_filelike/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_return/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_return/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_str/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_str/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_triggered/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_triggered/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_as_bytes_return_http_response/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_as_bytes_return_http_response/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_as_str_return_http_response/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_as_str_return_http_response/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_bytes/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_bytes/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_filelike/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_filelike/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_return/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_return/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_str/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_str/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_trigger/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_trigger/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/main.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/eventhub_batch_functions_stein/function_app.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/eventhub_multiple/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/eventhub_multiple/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/eventhub_output_batch/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/eventhub_output_batch/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/get_eventhub_batch_triggered/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/get_eventhub_batch_triggered/function.json (95%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/get_metadata_batch_triggered/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/get_metadata_batch_triggered/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/metadata_multiple/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/metadata_multiple/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/metadata_output_batch/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/metadata_output_batch/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/eventhub_functions_stein/function_app.py (97%) rename tests/{endtoend => emulator_tests}/eventhub_functions/eventhub_functions_stein/generic/function_app.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/eventhub_output/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/eventhub_output/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/eventhub_trigger/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/eventhub_trigger/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/get_eventhub_triggered/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/get_eventhub_triggered/main.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/get_metadata_triggered/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/get_metadata_triggered/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/metadata_output/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/metadata_output/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/metadata_trigger/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/metadata_trigger/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/generic_functions_stein/function_app.py (82%) rename tests/{endtoend => emulator_tests}/generic_functions/return_bool/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_bool/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_bytes/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_bytes/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_dict/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_dict/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_double/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_double/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_int/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_int/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_list/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_list/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_none/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_none/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_none_no_type_hint/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_none_no_type_hint/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_not_processed_last/__init__.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_not_processed_last/function.json (91%) rename tests/{endtoend => emulator_tests}/generic_functions/return_processed_last/__init__.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_processed_last/function.json (91%) rename tests/{endtoend => emulator_tests}/generic_functions/return_string/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_string/main.py (100%) create mode 100644 tests/emulator_tests/generic_functions/table_out_binding/__init__.py create mode 100644 tests/emulator_tests/generic_functions/table_out_binding/function.json rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_blob/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_blob/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_blob_message_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_blob_message_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_blob_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_blob_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_untyped_blob_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_untyped_blob_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_message_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_message_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_multiple_out/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_multiple_out/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_return_multiple/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_return_multiple/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_untyped_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_untyped_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_functions_stein/function_app.py (97%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_functions_stein/generic/function_app.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_message_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_message_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_return_multiple/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_return_multiple/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_untyped/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_untyped/main.py (100%) rename tests/{endtoend => emulator_tests}/servicebus_functions/get_servicebus_triggered/__init__.py (100%) rename tests/{endtoend => emulator_tests}/servicebus_functions/get_servicebus_triggered/function.json (100%) rename tests/{endtoend => emulator_tests}/servicebus_functions/put_message/__init__.py (100%) rename tests/{endtoend => emulator_tests}/servicebus_functions/put_message/function.json (100%) rename tests/{endtoend => emulator_tests}/servicebus_functions/servicebus_functions_stein/function_app.py (97%) rename tests/{endtoend => emulator_tests}/servicebus_functions/servicebus_functions_stein/generic/function_app.py (97%) rename tests/{endtoend => emulator_tests}/servicebus_functions/servicebus_trigger/__init__.py (100%) rename tests/{endtoend => emulator_tests}/servicebus_functions/servicebus_trigger/function.json (100%) rename tests/{endtoend => emulator_tests}/table_functions/table_functions_stein/function_app.py (100%) rename tests/{endtoend => emulator_tests}/table_functions/table_functions_stein/generic/function_app.py (100%) rename tests/{endtoend => emulator_tests}/table_functions/table_in_binding/__init__.py (100%) rename tests/{endtoend => emulator_tests}/table_functions/table_in_binding/function.json (100%) rename tests/{endtoend => emulator_tests}/table_functions/table_out_binding/__init__.py (100%) rename tests/{endtoend => emulator_tests}/table_functions/table_out_binding/function.json (100%) rename tests/{endtoend => emulator_tests}/test_blob_functions.py (95%) rename tests/{endtoend => emulator_tests}/test_eventhub_batch_functions.py (94%) rename tests/{endtoend => emulator_tests}/test_eventhub_functions.py (90%) rename tests/{endtoend => emulator_tests}/test_generic_functions.py (83%) rename tests/{endtoend => emulator_tests}/test_queue_functions.py (92%) rename tests/{endtoend => emulator_tests}/test_servicebus_functions.py (84%) rename tests/{endtoend => emulator_tests}/test_table_functions.py (84%) create mode 100644 tests/emulator_tests/utils/eventhub/config.json create mode 100644 tests/emulator_tests/utils/eventhub/docker-compose.yml create mode 100644 tests/emulator_tests/utils/servicebus/config.json create mode 100644 tests/emulator_tests/utils/servicebus/docker-compose.yml diff --git a/CODEOWNERS b/CODEOWNERS index f9350110..e5f28aee 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -10,4 +10,4 @@ # For all file changes, github would automatically # include the following people in the PRs. -* @vrdmr @gavin-aguiar @YunchuWang @pdthummar @hallvictoria +* @vrdmr @gavin-aguiar @hallvictoria diff --git a/eng/ci/emulator-tests.yml b/eng/ci/emulator-tests.yml new file mode 100644 index 00000000..adb1016f --- /dev/null +++ b/eng/ci/emulator-tests.yml @@ -0,0 +1,46 @@ +trigger: none # ensure this is not ran as a CI build + +pr: + branches: + include: + - dev + - release/* + +schedules: + - cron: "0 8 * * 1,2,3,4,5" + displayName: Monday to Friday 3 AM CST build + branches: + include: + - dev + always: true + +resources: + repositories: + - repository: 1es + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + - repository: eng + type: git + name: engineering + ref: refs/tags/release + +variables: + - template: /ci/variables/build.yml@eng + - template: /ci/variables/cfs.yml@eng + - template: /eng/templates/utils/variables.yml@self + +extends: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1es + parameters: + pool: + name: 1es-pool-azfunc + image: 1es-windows-2022 + os: windows + + stages: + - stage: RunEmulatorTests + jobs: + - template: /eng/templates/jobs/ci-emulator-tests.yml@self + parameters: + PoolName: 1es-pool-azfunc \ No newline at end of file diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index ad332eb2..568fdf16 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -54,6 +54,12 @@ extends: dependsOn: Build jobs: - template: /eng/templates/official/jobs/ci-e2e-tests.yml@self + - stage: RunEmulatorTests + dependsOn: Build + jobs: + - template: /eng/templates/jobs/ci-emulator-tests.yml@self + parameters: + PoolName: 1es-pool-azfunc - stage: RunUnitTests dependsOn: Build jobs: diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index 222cdc2d..26f1b662 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -53,4 +53,10 @@ extends: - stage: RunUnitTests dependsOn: Build jobs: - - template: /eng/templates/jobs/ci-unit-tests.yml@self \ No newline at end of file + - template: /eng/templates/jobs/ci-unit-tests.yml@self + - stage: RunEmulatorTests + dependsOn: Build + jobs: + - template: /eng/templates/jobs/ci-emulator-tests.yml@self + parameters: + PoolName: 1es-pool-azfunc-public \ No newline at end of file diff --git a/eng/templates/jobs/ci-emulator-tests.yml b/eng/templates/jobs/ci-emulator-tests.yml new file mode 100644 index 00000000..d2ab3ce8 --- /dev/null +++ b/eng/templates/jobs/ci-emulator-tests.yml @@ -0,0 +1,100 @@ +jobs: + - job: "TestPython" + displayName: "Run Python Emulator Tests" + + pool: + name: ${{ parameters.PoolName }} + image: 1es-ubuntu-22.04 + os: linux + + strategy: + matrix: + Python37: + PYTHON_VERSION: '3.7' + Python38: + PYTHON_VERSION: '3.8' + Python39: + PYTHON_VERSION: '3.9' + Python310: + PYTHON_VERSION: '3.10' + Python311: + PYTHON_VERSION: '3.11' + Python312: + PYTHON_VERSION: '3.12' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: $(PYTHON_VERSION) + - task: UseDotNet@2 + displayName: 'Install .NET 8' + inputs: + version: 8.0.x + - bash: | + chmod +x eng/scripts/install-dependencies.sh + chmod +x eng/scripts/test-setup.sh + + eng/scripts/install-dependencies.sh $(PYTHON_VERSION) + eng/scripts/test-setup.sh + displayName: 'Install dependencies and the worker' + condition: and(eq(variables.isSdkRelease, false), eq(variables.isExtensionsRelease, false), eq(variables['USETESTPYTHONSDK'], false), eq(variables['USETESTPYTHONEXTENSIONS'], false)) + - task: DownloadPipelineArtifact@2 + displayName: 'Download Python SDK Artifact' + inputs: + buildType: specific + artifactName: 'azure-functions' + project: 'internal' + definition: 679 + buildVersionToDownload: latest + targetPath: '$(Pipeline.Workspace)/PythonSdkArtifact' + condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) + - bash: | + chmod +x eng/scripts/test-sdk.sh + chmod +x eng/scripts/test-setup.sh + + eng/scripts/test-sdk.sh $(Pipeline.Workspace) $(PYTHON_VERSION) + eng/scripts/test-setup.sh + displayName: 'Install test python sdk, dependencies and the worker' + condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) + - task: DownloadPipelineArtifact@2 + displayName: 'Download Python Extension Artifact' + inputs: + buildType: specific + artifactName: $(PYTHONEXTENSIONNAME) + project: 'internal' + definition: 798 + buildVersionToDownload: latest + targetPath: '$(Pipeline.Workspace)/PythonExtensionArtifact' + condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) + - bash: | + chmod +x eng/scripts/test-setup.sh + chmod +x eng/scripts/test-extensions.sh + + eng/scripts/test-extensions.sh $(Pipeline.Workspace) $(PYTHON_VERSION) + eng/scripts/test-setup.sh + displayName: 'Install test python extension, dependencies and the worker' + condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) + - bash: | + docker compose -f tests/emulator_tests/utils/eventhub/docker-compose.yml pull + docker compose -f tests/emulator_tests/utils/eventhub/docker-compose.yml up -d + displayName: 'Install Azurite and Start EventHub Emulator' + - bash: | + python -m pytest -q -n auto --dist loadfile --reruns 4 --ignore=tests/emulator_tests/test_servicebus_functions.py tests/emulator_tests + env: + AzureWebJobsStorage: "UseDevelopmentStorage=true" + AzureWebJobsEventHubConnectionString: "Endpoint=sb://localhost;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SAS_KEY_VALUE;UseDevelopmentEmulator=true;" + displayName: "Running $(PYTHON_VERSION) Python Linux Emulator Tests" + - bash: | + # Stop and remove EventHub Emulator container to free up the port + docker stop eventhubs-emulator + docker container rm --force eventhubs-emulator + docker compose -f tests/emulator_tests/utils/servicebus/docker-compose.yml pull + docker compose -f tests/emulator_tests/utils/servicebus/docker-compose.yml up -d + env: + AzureWebJobsSQLPassword: $(AzureWebJobsSQLPassword) + displayName: 'Install Azurite and Start ServiceBus Emulator' + - bash: | + python -m pytest -q -n auto --dist loadfile --reruns 4 tests/emulator_tests/test_servicebus_functions.py + env: + AzureWebJobsStorage: "UseDevelopmentStorage=true" + AzureWebJobsServiceBusConnectionString: "Endpoint=sb://localhost;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SAS_KEY_VALUE;UseDevelopmentEmulator=true;" + displayName: "Running $(PYTHON_VERSION) Python ServiceBus Linux Emulator Tests" diff --git a/tests/endtoend/blob_functions/blob_functions_stein/function_app.py b/tests/emulator_tests/blob_functions/blob_functions_stein/function_app.py similarity index 97% rename from tests/endtoend/blob_functions/blob_functions_stein/function_app.py rename to tests/emulator_tests/blob_functions/blob_functions_stein/function_app.py index 1b77b858..24489b0e 100644 --- a/tests/endtoend/blob_functions/blob_functions_stein/function_app.py +++ b/tests/emulator_tests/blob_functions/blob_functions_stein/function_app.py @@ -1,445 +1,445 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -import hashlib -import io -import json -import random -import string - -import azure.functions as func - -app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) - - -@app.function_name(name="blob_trigger") -@app.blob_trigger(arg_name="file", - path="python-worker-tests/test-blob-trigger.txt", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="$return", - path="python-worker-tests/test-blob-triggered.txt", - connection="AzureWebJobsStorage") -def blob_trigger(file: func.InputStream) -> str: - return json.dumps({ - 'name': file.name, - 'length': file.length, - 'content': file.read().decode('utf-8') - }) - - -@app.function_name(name="get_blob_as_bytes") -@app.route(route="get_blob_as_bytes") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-bytes.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") -def get_blob_as_bytes(req: func.HttpRequest, file: bytes) -> str: - assert isinstance(file, bytes) - return file.decode('utf-8') - - -@app.function_name(name="get_blob_as_bytes_return_http_response") -@app.route(route="get_blob_as_bytes_return_http_response") -@app.blob_input(arg_name="file", - path="python-worker-tests/shmem-test-bytes.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") -def get_blob_as_bytes_return_http_response(req: func.HttpRequest, file: bytes) \ - -> func.HttpResponse: - """ - Read a blob (bytes) and respond back (in HTTP response) with the number of - bytes read and the MD5 digest of the content. - """ - assert isinstance(file, bytes) - - content_size = len(file) - content_sha256 = hashlib.sha256(file).hexdigest() - - response_dict = { - 'content_size': content_size, - 'content_sha256': content_sha256 - } - - response_body = json.dumps(response_dict, indent=2) - - return func.HttpResponse( - body=response_body, - mimetype="application/json", - status_code=200 - ) - - -@app.function_name(name="get_blob_as_bytes_stream_return_http_response") -@app.route(route="get_blob_as_bytes_stream_return_http_response") -@app.blob_input(arg_name="file", - path="python-worker-tests/shmem-test-bytes.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") -def get_blob_as_bytes_stream_return_http_response(req: func.HttpRequest, - file: func.InputStream) \ - -> func.HttpResponse: - """ - Read a blob (as azf.InputStream) and respond back (in HTTP response) with - the number of bytes read and the MD5 digest of the content. - """ - file_bytes = file.read() - - content_size = len(file_bytes) - content_sha256 = hashlib.sha256(file_bytes).hexdigest() - - response_dict = { - 'content_size': content_size, - 'content_sha256': content_sha256 - } - - response_body = json.dumps(response_dict, indent=2) - - return func.HttpResponse( - body=response_body, - mimetype="application/json", - status_code=200 - ) - - -@app.function_name(name="get_blob_as_str") -@app.route(route="get_blob_as_str") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-str.txt", - data_type="STRING", - connection="AzureWebJobsStorage") -def get_blob_as_str(req: func.HttpRequest, file: str) -> str: - assert isinstance(file, str) - return file - - -@app.function_name(name="get_blob_as_str_return_http_response") -@app.route(route="get_blob_as_str_return_http_response") -@app.blob_input(arg_name="file", - path="python-worker-tests/shmem-test-bytes.txt", - data_type="STRING", - connection="AzureWebJobsStorage") -def get_blob_as_str_return_http_response(req: func.HttpRequest, - file: str) -> func.HttpResponse: - """ - Read a blob (string) and respond back (in HTTP response) with the number of - characters read and the MD5 digest of the utf-8 encoded content. - """ - assert isinstance(file, str) - - num_chars = len(file) - content_bytes = file.encode('utf-8') - content_sha256 = hashlib.sha256(content_bytes).hexdigest() - - response_dict = { - 'num_chars': num_chars, - 'content_sha256': content_sha256 - } - - response_body = json.dumps(response_dict, indent=2) - - return func.HttpResponse( - body=response_body, - mimetype="application/json", - status_code=200 - ) - - -@app.function_name(name="get_blob_bytes") -@app.route(route="get_blob_bytes") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-bytes.txt", - connection="AzureWebJobsStorage") -def get_blob_bytes(req: func.HttpRequest, file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="get_blob_filelike") -@app.route(route="get_blob_filelike") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-filelike.txt", - connection="AzureWebJobsStorage") -def get_blob_filelike(req: func.HttpRequest, file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="get_blob_return") -@app.route(route="get_blob_return") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-return.txt", - connection="AzureWebJobsStorage") -def get_blob_return(req: func.HttpRequest, file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="get_blob_str") -@app.route(route="get_blob_str") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-str.txt", - connection="AzureWebJobsStorage") -def get_blob_str(req: func.HttpRequest, file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="get_blob_triggered") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-blob-triggered.txt", - connection="AzureWebJobsStorage") -@app.route(route="get_blob_triggered") -def get_blob_triggered(req: func.HttpRequest, file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="put_blob_as_bytes_return_http_response") -@app.blob_output(arg_name="file", - path="python-worker-tests/shmem-test-bytes-out.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_as_bytes_return_http_response") -def put_blob_as_bytes_return_http_response(req: func.HttpRequest, - file: func.Out[ - bytes]) -> func.HttpResponse: - """ - Write a blob (bytes) and respond back (in HTTP response) with the number of - bytes written and the MD5 digest of the content. - The number of bytes to write are specified in the input HTTP request. - """ - content_size = int(req.params['content_size']) - - # When this is set, then 0x01 byte is repeated content_size number of - # times to use as input. - # This is to avoid generating random input for large size which can be - # slow. - if 'no_random_input' in req.params: - content = b'\x01' * content_size - else: - content = bytearray(random.getrandbits(8) for _ in range(content_size)) - content_sha256 = hashlib.sha256(content).hexdigest() - - file.set(content) - - response_dict = { - 'content_size': content_size, - 'content_sha256': content_sha256 - } - - response_body = json.dumps(response_dict, indent=2) - - return func.HttpResponse( - body=response_body, - mimetype="application/json", - status_code=200 - ) - - -@app.function_name(name="put_blob_as_str_return_http_response") -@app.blob_output(arg_name="file", - path="python-worker-tests/shmem-test-str-out.txt", - data_type="STRING", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_as_str_return_http_response") -def put_blob_as_str_return_http_response(req: func.HttpRequest, file: func.Out[ - str]) -> func.HttpResponse: - """ - Write a blob (string) and respond back (in HTTP response) with the number of - characters written and the MD5 digest of the utf-8 encoded content. - The number of characters to write are specified in the input HTTP request. - """ - num_chars = int(req.params['num_chars']) - - content = ''.join(random.choices(string.ascii_uppercase + string.digits, - k=num_chars)) - content_bytes = content.encode('utf-8') - content_size = len(content_bytes) - content_sha256 = hashlib.sha256(content_bytes).hexdigest() - - file.set(content) - - response_dict = { - 'num_chars': num_chars, - 'content_size': content_size, - 'content_sha256': content_sha256 - } - - response_body = json.dumps(response_dict, indent=2) - - return func.HttpResponse( - body=response_body, - mimetype="application/json", - status_code=200 - ) - - -@app.function_name(name="put_blob_bytes") -@app.blob_output(arg_name="file", - path="python-worker-tests/test-bytes.txt", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_bytes") -def put_blob_bytes(req: func.HttpRequest, file: func.Out[bytes]) -> str: - file.set(req.get_body()) - return 'OK' - - -@app.function_name(name="put_blob_filelike") -@app.blob_output(arg_name="file", - path="python-worker-tests/test-filelike.txt", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_filelike") -def put_blob_filelike(req: func.HttpRequest, - file: func.Out[io.StringIO]) -> str: - file.set(io.StringIO('filelike')) - return 'OK' - - -@app.function_name(name="put_blob_return") -@app.blob_output(arg_name="$return", - path="python-worker-tests/test-return.txt", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_return", binding_arg_name="resp") -def put_blob_return(req: func.HttpRequest, - resp: func.Out[func.HttpResponse]) -> str: - return 'FROM RETURN' - - -@app.function_name(name="put_blob_str") -@app.blob_output(arg_name="file", - path="python-worker-tests/test-str.txt", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_str") -def put_blob_str(req: func.HttpRequest, file: func.Out[str]) -> str: - file.set(req.get_body()) - return 'OK' - - -@app.function_name(name="put_blob_trigger") -@app.blob_output(arg_name="file", - path="python-worker-tests/test-blob-trigger.txt", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_trigger") -def put_blob_trigger(req: func.HttpRequest, file: func.Out[str]) -> str: - file.set(req.get_body()) - return 'OK' - - -def _generate_content_and_digest(content_size): - content = bytearray(random.getrandbits(8) for _ in range(content_size)) - content_sha256 = hashlib.sha256(content).hexdigest() - return content, content_sha256 - - -@app.function_name(name="put_get_multiple_blobs_as_bytes_return_http_response") -@app.blob_input(arg_name="inputfile1", - data_type="BINARY", - path="python-worker-tests/shmem-test-bytes-1.txt", - connection="AzureWebJobsStorage") -@app.blob_input(arg_name="inputfile2", - data_type="BINARY", - path="python-worker-tests/shmem-test-bytes-2.txt", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="outputfile1", - path="python-worker-tests/shmem-test-bytes-out-1.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="outputfile2", - path="python-worker-tests/shmem-test-bytes-out-2.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") -@app.route(route="put_get_multiple_blobs_as_bytes_return_http_response") -def put_get_multiple_blobs_as_bytes_return_http_response( - req: func.HttpRequest, - inputfile1: bytes, - inputfile2: bytes, - outputfile1: func.Out[bytes], - outputfile2: func.Out[bytes]) -> func.HttpResponse: - """ - Read two blobs (bytes) and respond back (in HTTP response) with the number - of bytes read from each blob and the MD5 digest of the content of each. - Write two blobs (bytes) and respond back (in HTTP response) with the number - bytes written in each blob and the MD5 digest of the content of each. - The number of bytes to write are specified in the input HTTP request. - """ - input_content_size_1 = len(inputfile1) - input_content_size_2 = len(inputfile2) - - input_content_sha256_1 = hashlib.sha256(inputfile1).hexdigest() - input_content_sha256_2 = hashlib.sha256(inputfile2).hexdigest() - - output_content_size_1 = int(req.params['output_content_size_1']) - output_content_size_2 = int(req.params['output_content_size_2']) - - output_content_1, output_content_sha256_1 = \ - _generate_content_and_digest(output_content_size_1) - output_content_2, output_content_sha256_2 = \ - _generate_content_and_digest(output_content_size_2) - - outputfile1.set(output_content_1) - outputfile2.set(output_content_2) - - response_dict = { - 'input_content_size_1': input_content_size_1, - 'input_content_size_2': input_content_size_2, - 'input_content_sha256_1': input_content_sha256_1, - 'input_content_sha256_2': input_content_sha256_2, - 'output_content_size_1': output_content_size_1, - 'output_content_size_2': output_content_size_2, - 'output_content_sha256_1': output_content_sha256_1, - 'output_content_sha256_2': output_content_sha256_2 - } - - response_body = json.dumps(response_dict, indent=2) - - return func.HttpResponse( - body=response_body, - mimetype="application/json", - status_code=200 - ) - - -@app.function_name(name="blob_trigger_default_source_enum") -@app.blob_trigger(arg_name="file", - path="python-worker-tests/test-blob-trigger.txt", - connection="AzureWebJobsStorage", - source=func.BlobSource.LOGS_AND_CONTAINER_SCAN) -def blob_trigger_default_source_enum(file: func.InputStream) -> str: - return json.dumps({ - 'name': file.name, - 'length': file.length, - 'content': file.read().decode('utf-8') - }) - - -@app.function_name(name="blob_trigger_eventgrid_source_enum") -@app.blob_trigger(arg_name="file", - path="python-worker-tests/test-blob-trigger.txt", - connection="AzureWebJobsStorage", - source=func.BlobSource.EVENT_GRID) -def blob_trigger_eventgrid_source_enum(file: func.InputStream) -> str: - return json.dumps({ - 'name': file.name, - 'length': file.length, - 'content': file.read().decode('utf-8') - }) - - -@app.function_name(name="blob_trigger_default_source_str") -@app.blob_trigger(arg_name="file", - path="python-worker-tests/test-blob-trigger.txt", - connection="AzureWebJobsStorage", - source="LogsAndContainerScan") -def blob_trigger_default_source_str(file: func.InputStream) -> str: - return json.dumps({ - 'name': file.name, - 'length': file.length, - 'content': file.read().decode('utf-8') - }) - - -@app.function_name(name="blob_trigger_eventgrid_source_str") -@app.blob_trigger(arg_name="file", - path="python-worker-tests/test-blob-trigger.txt", - connection="AzureWebJobsStorage", - source="EventGrid") -def blob_trigger_eventgrid_source_str(file: func.InputStream) -> str: - return json.dumps({ - 'name': file.name, - 'length': file.length, - 'content': file.read().decode('utf-8') - }) +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import hashlib +import io +import json +import random +import string + +import azure.functions as func + +app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) + + +@app.function_name(name="blob_trigger") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="$return", + path="python-worker-tests/test-blob-triggered.txt", + connection="AzureWebJobsStorage") +def blob_trigger(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) + + +@app.function_name(name="get_blob_as_bytes") +@app.route(route="get_blob_as_bytes") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-bytes.txt", + data_type="BINARY", + connection="AzureWebJobsStorage") +def get_blob_as_bytes(req: func.HttpRequest, file: bytes) -> str: + assert isinstance(file, bytes) + return file.decode('utf-8') + + +@app.function_name(name="get_blob_as_bytes_return_http_response") +@app.route(route="get_blob_as_bytes_return_http_response") +@app.blob_input(arg_name="file", + path="python-worker-tests/shmem-test-bytes.txt", + data_type="BINARY", + connection="AzureWebJobsStorage") +def get_blob_as_bytes_return_http_response(req: func.HttpRequest, file: bytes) \ + -> func.HttpResponse: + """ + Read a blob (bytes) and respond back (in HTTP response) with the number of + bytes read and the MD5 digest of the content. + """ + assert isinstance(file, bytes) + + content_size = len(file) + content_sha256 = hashlib.sha256(file).hexdigest() + + response_dict = { + 'content_size': content_size, + 'content_sha256': content_sha256 + } + + response_body = json.dumps(response_dict, indent=2) + + return func.HttpResponse( + body=response_body, + mimetype="application/json", + status_code=200 + ) + + +@app.function_name(name="get_blob_as_bytes_stream_return_http_response") +@app.route(route="get_blob_as_bytes_stream_return_http_response") +@app.blob_input(arg_name="file", + path="python-worker-tests/shmem-test-bytes.txt", + data_type="BINARY", + connection="AzureWebJobsStorage") +def get_blob_as_bytes_stream_return_http_response(req: func.HttpRequest, + file: func.InputStream) \ + -> func.HttpResponse: + """ + Read a blob (as azf.InputStream) and respond back (in HTTP response) with + the number of bytes read and the MD5 digest of the content. + """ + file_bytes = file.read() + + content_size = len(file_bytes) + content_sha256 = hashlib.sha256(file_bytes).hexdigest() + + response_dict = { + 'content_size': content_size, + 'content_sha256': content_sha256 + } + + response_body = json.dumps(response_dict, indent=2) + + return func.HttpResponse( + body=response_body, + mimetype="application/json", + status_code=200 + ) + + +@app.function_name(name="get_blob_as_str") +@app.route(route="get_blob_as_str") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-str.txt", + data_type="STRING", + connection="AzureWebJobsStorage") +def get_blob_as_str(req: func.HttpRequest, file: str) -> str: + assert isinstance(file, str) + return file + + +@app.function_name(name="get_blob_as_str_return_http_response") +@app.route(route="get_blob_as_str_return_http_response") +@app.blob_input(arg_name="file", + path="python-worker-tests/shmem-test-bytes.txt", + data_type="STRING", + connection="AzureWebJobsStorage") +def get_blob_as_str_return_http_response(req: func.HttpRequest, + file: str) -> func.HttpResponse: + """ + Read a blob (string) and respond back (in HTTP response) with the number of + characters read and the MD5 digest of the utf-8 encoded content. + """ + assert isinstance(file, str) + + num_chars = len(file) + content_bytes = file.encode('utf-8') + content_sha256 = hashlib.sha256(content_bytes).hexdigest() + + response_dict = { + 'num_chars': num_chars, + 'content_sha256': content_sha256 + } + + response_body = json.dumps(response_dict, indent=2) + + return func.HttpResponse( + body=response_body, + mimetype="application/json", + status_code=200 + ) + + +@app.function_name(name="get_blob_bytes") +@app.route(route="get_blob_bytes") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-bytes.txt", + connection="AzureWebJobsStorage") +def get_blob_bytes(req: func.HttpRequest, file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="get_blob_filelike") +@app.route(route="get_blob_filelike") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-filelike.txt", + connection="AzureWebJobsStorage") +def get_blob_filelike(req: func.HttpRequest, file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="get_blob_return") +@app.route(route="get_blob_return") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-return.txt", + connection="AzureWebJobsStorage") +def get_blob_return(req: func.HttpRequest, file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="get_blob_str") +@app.route(route="get_blob_str") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-str.txt", + connection="AzureWebJobsStorage") +def get_blob_str(req: func.HttpRequest, file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="get_blob_triggered") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-blob-triggered.txt", + connection="AzureWebJobsStorage") +@app.route(route="get_blob_triggered") +def get_blob_triggered(req: func.HttpRequest, file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="put_blob_as_bytes_return_http_response") +@app.blob_output(arg_name="file", + path="python-worker-tests/shmem-test-bytes-out.txt", + data_type="BINARY", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_as_bytes_return_http_response") +def put_blob_as_bytes_return_http_response(req: func.HttpRequest, + file: func.Out[ + bytes]) -> func.HttpResponse: + """ + Write a blob (bytes) and respond back (in HTTP response) with the number of + bytes written and the MD5 digest of the content. + The number of bytes to write are specified in the input HTTP request. + """ + content_size = int(req.params['content_size']) + + # When this is set, then 0x01 byte is repeated content_size number of + # times to use as input. + # This is to avoid generating random input for large size which can be + # slow. + if 'no_random_input' in req.params: + content = b'\x01' * content_size + else: + content = bytearray(random.getrandbits(8) for _ in range(content_size)) + content_sha256 = hashlib.sha256(content).hexdigest() + + file.set(content) + + response_dict = { + 'content_size': content_size, + 'content_sha256': content_sha256 + } + + response_body = json.dumps(response_dict, indent=2) + + return func.HttpResponse( + body=response_body, + mimetype="application/json", + status_code=200 + ) + + +@app.function_name(name="put_blob_as_str_return_http_response") +@app.blob_output(arg_name="file", + path="python-worker-tests/shmem-test-str-out.txt", + data_type="STRING", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_as_str_return_http_response") +def put_blob_as_str_return_http_response(req: func.HttpRequest, file: func.Out[ + str]) -> func.HttpResponse: + """ + Write a blob (string) and respond back (in HTTP response) with the number of + characters written and the MD5 digest of the utf-8 encoded content. + The number of characters to write are specified in the input HTTP request. + """ + num_chars = int(req.params['num_chars']) + + content = ''.join(random.choices(string.ascii_uppercase + string.digits, + k=num_chars)) + content_bytes = content.encode('utf-8') + content_size = len(content_bytes) + content_sha256 = hashlib.sha256(content_bytes).hexdigest() + + file.set(content) + + response_dict = { + 'num_chars': num_chars, + 'content_size': content_size, + 'content_sha256': content_sha256 + } + + response_body = json.dumps(response_dict, indent=2) + + return func.HttpResponse( + body=response_body, + mimetype="application/json", + status_code=200 + ) + + +@app.function_name(name="put_blob_bytes") +@app.blob_output(arg_name="file", + path="python-worker-tests/test-bytes.txt", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_bytes") +def put_blob_bytes(req: func.HttpRequest, file: func.Out[bytes]) -> str: + file.set(req.get_body()) + return 'OK' + + +@app.function_name(name="put_blob_filelike") +@app.blob_output(arg_name="file", + path="python-worker-tests/test-filelike.txt", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_filelike") +def put_blob_filelike(req: func.HttpRequest, + file: func.Out[io.StringIO]) -> str: + file.set(io.StringIO('filelike')) + return 'OK' + + +@app.function_name(name="put_blob_return") +@app.blob_output(arg_name="$return", + path="python-worker-tests/test-return.txt", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_return", binding_arg_name="resp") +def put_blob_return(req: func.HttpRequest, + resp: func.Out[func.HttpResponse]) -> str: + return 'FROM RETURN' + + +@app.function_name(name="put_blob_str") +@app.blob_output(arg_name="file", + path="python-worker-tests/test-str.txt", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_str") +def put_blob_str(req: func.HttpRequest, file: func.Out[str]) -> str: + file.set(req.get_body()) + return 'OK' + + +@app.function_name(name="put_blob_trigger") +@app.blob_output(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_trigger") +def put_blob_trigger(req: func.HttpRequest, file: func.Out[str]) -> str: + file.set(req.get_body()) + return 'OK' + + +def _generate_content_and_digest(content_size): + content = bytearray(random.getrandbits(8) for _ in range(content_size)) + content_sha256 = hashlib.sha256(content).hexdigest() + return content, content_sha256 + + +@app.function_name(name="put_get_multiple_blobs_as_bytes_return_http_response") +@app.blob_input(arg_name="inputfile1", + data_type="BINARY", + path="python-worker-tests/shmem-test-bytes-1.txt", + connection="AzureWebJobsStorage") +@app.blob_input(arg_name="inputfile2", + data_type="BINARY", + path="python-worker-tests/shmem-test-bytes-2.txt", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="outputfile1", + path="python-worker-tests/shmem-test-bytes-out-1.txt", + data_type="BINARY", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="outputfile2", + path="python-worker-tests/shmem-test-bytes-out-2.txt", + data_type="BINARY", + connection="AzureWebJobsStorage") +@app.route(route="put_get_multiple_blobs_as_bytes_return_http_response") +def put_get_multiple_blobs_as_bytes_return_http_response( + req: func.HttpRequest, + inputfile1: bytes, + inputfile2: bytes, + outputfile1: func.Out[bytes], + outputfile2: func.Out[bytes]) -> func.HttpResponse: + """ + Read two blobs (bytes) and respond back (in HTTP response) with the number + of bytes read from each blob and the MD5 digest of the content of each. + Write two blobs (bytes) and respond back (in HTTP response) with the number + bytes written in each blob and the MD5 digest of the content of each. + The number of bytes to write are specified in the input HTTP request. + """ + input_content_size_1 = len(inputfile1) + input_content_size_2 = len(inputfile2) + + input_content_sha256_1 = hashlib.sha256(inputfile1).hexdigest() + input_content_sha256_2 = hashlib.sha256(inputfile2).hexdigest() + + output_content_size_1 = int(req.params['output_content_size_1']) + output_content_size_2 = int(req.params['output_content_size_2']) + + output_content_1, output_content_sha256_1 = \ + _generate_content_and_digest(output_content_size_1) + output_content_2, output_content_sha256_2 = \ + _generate_content_and_digest(output_content_size_2) + + outputfile1.set(output_content_1) + outputfile2.set(output_content_2) + + response_dict = { + 'input_content_size_1': input_content_size_1, + 'input_content_size_2': input_content_size_2, + 'input_content_sha256_1': input_content_sha256_1, + 'input_content_sha256_2': input_content_sha256_2, + 'output_content_size_1': output_content_size_1, + 'output_content_size_2': output_content_size_2, + 'output_content_sha256_1': output_content_sha256_1, + 'output_content_sha256_2': output_content_sha256_2 + } + + response_body = json.dumps(response_dict, indent=2) + + return func.HttpResponse( + body=response_body, + mimetype="application/json", + status_code=200 + ) + + +@app.function_name(name="blob_trigger_default_source_enum") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source=func.BlobSource.LOGS_AND_CONTAINER_SCAN) +def blob_trigger_default_source_enum(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) + + +@app.function_name(name="blob_trigger_eventgrid_source_enum") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source=func.BlobSource.EVENT_GRID) +def blob_trigger_eventgrid_source_enum(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) + + +@app.function_name(name="blob_trigger_default_source_str") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source="LogsAndContainerScan") +def blob_trigger_default_source_str(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) + + +@app.function_name(name="blob_trigger_eventgrid_source_str") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source="EventGrid") +def blob_trigger_eventgrid_source_str(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) diff --git a/tests/endtoend/blob_functions/blob_functions_stein/generic/function_app.py b/tests/emulator_tests/blob_functions/blob_functions_stein/generic/function_app.py similarity index 100% rename from tests/endtoend/blob_functions/blob_functions_stein/generic/function_app.py rename to tests/emulator_tests/blob_functions/blob_functions_stein/generic/function_app.py diff --git a/tests/endtoend/blob_functions/blob_trigger/function.json b/tests/emulator_tests/blob_functions/blob_trigger/function.json similarity index 100% rename from tests/endtoend/blob_functions/blob_trigger/function.json rename to tests/emulator_tests/blob_functions/blob_trigger/function.json diff --git a/tests/endtoend/blob_functions/blob_trigger/main.py b/tests/emulator_tests/blob_functions/blob_trigger/main.py similarity index 100% rename from tests/endtoend/blob_functions/blob_trigger/main.py rename to tests/emulator_tests/blob_functions/blob_trigger/main.py diff --git a/tests/endtoend/blob_functions/get_blob_as_bytes/function.json b/tests/emulator_tests/blob_functions/get_blob_as_bytes/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_bytes/function.json rename to tests/emulator_tests/blob_functions/get_blob_as_bytes/function.json diff --git a/tests/endtoend/blob_functions/get_blob_as_bytes/main.py b/tests/emulator_tests/blob_functions/get_blob_as_bytes/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_bytes/main.py rename to tests/emulator_tests/blob_functions/get_blob_as_bytes/main.py diff --git a/tests/endtoend/blob_functions/get_blob_as_bytes_return_http_response/function.json b/tests/emulator_tests/blob_functions/get_blob_as_bytes_return_http_response/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_bytes_return_http_response/function.json rename to tests/emulator_tests/blob_functions/get_blob_as_bytes_return_http_response/function.json diff --git a/tests/endtoend/blob_functions/get_blob_as_bytes_return_http_response/main.py b/tests/emulator_tests/blob_functions/get_blob_as_bytes_return_http_response/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_bytes_return_http_response/main.py rename to tests/emulator_tests/blob_functions/get_blob_as_bytes_return_http_response/main.py diff --git a/tests/endtoend/blob_functions/get_blob_as_bytes_stream_return_http_response/function.json b/tests/emulator_tests/blob_functions/get_blob_as_bytes_stream_return_http_response/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_bytes_stream_return_http_response/function.json rename to tests/emulator_tests/blob_functions/get_blob_as_bytes_stream_return_http_response/function.json diff --git a/tests/endtoend/blob_functions/get_blob_as_bytes_stream_return_http_response/main.py b/tests/emulator_tests/blob_functions/get_blob_as_bytes_stream_return_http_response/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_bytes_stream_return_http_response/main.py rename to tests/emulator_tests/blob_functions/get_blob_as_bytes_stream_return_http_response/main.py diff --git a/tests/endtoend/blob_functions/get_blob_as_str/function.json b/tests/emulator_tests/blob_functions/get_blob_as_str/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_str/function.json rename to tests/emulator_tests/blob_functions/get_blob_as_str/function.json diff --git a/tests/endtoend/blob_functions/get_blob_as_str/main.py b/tests/emulator_tests/blob_functions/get_blob_as_str/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_str/main.py rename to tests/emulator_tests/blob_functions/get_blob_as_str/main.py diff --git a/tests/endtoend/blob_functions/get_blob_as_str_return_http_response/function.json b/tests/emulator_tests/blob_functions/get_blob_as_str_return_http_response/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_str_return_http_response/function.json rename to tests/emulator_tests/blob_functions/get_blob_as_str_return_http_response/function.json diff --git a/tests/endtoend/blob_functions/get_blob_as_str_return_http_response/main.py b/tests/emulator_tests/blob_functions/get_blob_as_str_return_http_response/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_str_return_http_response/main.py rename to tests/emulator_tests/blob_functions/get_blob_as_str_return_http_response/main.py diff --git a/tests/endtoend/blob_functions/get_blob_bytes/function.json b/tests/emulator_tests/blob_functions/get_blob_bytes/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_bytes/function.json rename to tests/emulator_tests/blob_functions/get_blob_bytes/function.json diff --git a/tests/endtoend/blob_functions/get_blob_bytes/main.py b/tests/emulator_tests/blob_functions/get_blob_bytes/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_bytes/main.py rename to tests/emulator_tests/blob_functions/get_blob_bytes/main.py diff --git a/tests/endtoend/blob_functions/get_blob_filelike/function.json b/tests/emulator_tests/blob_functions/get_blob_filelike/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_filelike/function.json rename to tests/emulator_tests/blob_functions/get_blob_filelike/function.json diff --git a/tests/endtoend/blob_functions/get_blob_filelike/main.py b/tests/emulator_tests/blob_functions/get_blob_filelike/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_filelike/main.py rename to tests/emulator_tests/blob_functions/get_blob_filelike/main.py diff --git a/tests/endtoend/blob_functions/get_blob_return/function.json b/tests/emulator_tests/blob_functions/get_blob_return/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_return/function.json rename to tests/emulator_tests/blob_functions/get_blob_return/function.json diff --git a/tests/endtoend/blob_functions/get_blob_return/main.py b/tests/emulator_tests/blob_functions/get_blob_return/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_return/main.py rename to tests/emulator_tests/blob_functions/get_blob_return/main.py diff --git a/tests/endtoend/blob_functions/get_blob_str/function.json b/tests/emulator_tests/blob_functions/get_blob_str/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_str/function.json rename to tests/emulator_tests/blob_functions/get_blob_str/function.json diff --git a/tests/endtoend/blob_functions/get_blob_str/main.py b/tests/emulator_tests/blob_functions/get_blob_str/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_str/main.py rename to tests/emulator_tests/blob_functions/get_blob_str/main.py diff --git a/tests/endtoend/blob_functions/get_blob_triggered/function.json b/tests/emulator_tests/blob_functions/get_blob_triggered/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_triggered/function.json rename to tests/emulator_tests/blob_functions/get_blob_triggered/function.json diff --git a/tests/endtoend/blob_functions/get_blob_triggered/main.py b/tests/emulator_tests/blob_functions/get_blob_triggered/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_triggered/main.py rename to tests/emulator_tests/blob_functions/get_blob_triggered/main.py diff --git a/tests/endtoend/blob_functions/put_blob_as_bytes_return_http_response/function.json b/tests/emulator_tests/blob_functions/put_blob_as_bytes_return_http_response/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_blob_as_bytes_return_http_response/function.json rename to tests/emulator_tests/blob_functions/put_blob_as_bytes_return_http_response/function.json diff --git a/tests/endtoend/blob_functions/put_blob_as_bytes_return_http_response/main.py b/tests/emulator_tests/blob_functions/put_blob_as_bytes_return_http_response/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_blob_as_bytes_return_http_response/main.py rename to tests/emulator_tests/blob_functions/put_blob_as_bytes_return_http_response/main.py diff --git a/tests/endtoend/blob_functions/put_blob_as_str_return_http_response/function.json b/tests/emulator_tests/blob_functions/put_blob_as_str_return_http_response/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_blob_as_str_return_http_response/function.json rename to tests/emulator_tests/blob_functions/put_blob_as_str_return_http_response/function.json diff --git a/tests/endtoend/blob_functions/put_blob_as_str_return_http_response/main.py b/tests/emulator_tests/blob_functions/put_blob_as_str_return_http_response/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_blob_as_str_return_http_response/main.py rename to tests/emulator_tests/blob_functions/put_blob_as_str_return_http_response/main.py diff --git a/tests/endtoend/blob_functions/put_blob_bytes/function.json b/tests/emulator_tests/blob_functions/put_blob_bytes/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_blob_bytes/function.json rename to tests/emulator_tests/blob_functions/put_blob_bytes/function.json diff --git a/tests/endtoend/blob_functions/put_blob_bytes/main.py b/tests/emulator_tests/blob_functions/put_blob_bytes/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_blob_bytes/main.py rename to tests/emulator_tests/blob_functions/put_blob_bytes/main.py diff --git a/tests/endtoend/blob_functions/put_blob_filelike/function.json b/tests/emulator_tests/blob_functions/put_blob_filelike/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_blob_filelike/function.json rename to tests/emulator_tests/blob_functions/put_blob_filelike/function.json diff --git a/tests/endtoend/blob_functions/put_blob_filelike/main.py b/tests/emulator_tests/blob_functions/put_blob_filelike/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_blob_filelike/main.py rename to tests/emulator_tests/blob_functions/put_blob_filelike/main.py diff --git a/tests/endtoend/blob_functions/put_blob_return/function.json b/tests/emulator_tests/blob_functions/put_blob_return/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_blob_return/function.json rename to tests/emulator_tests/blob_functions/put_blob_return/function.json diff --git a/tests/endtoend/blob_functions/put_blob_return/main.py b/tests/emulator_tests/blob_functions/put_blob_return/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_blob_return/main.py rename to tests/emulator_tests/blob_functions/put_blob_return/main.py diff --git a/tests/endtoend/blob_functions/put_blob_str/function.json b/tests/emulator_tests/blob_functions/put_blob_str/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_blob_str/function.json rename to tests/emulator_tests/blob_functions/put_blob_str/function.json diff --git a/tests/endtoend/blob_functions/put_blob_str/main.py b/tests/emulator_tests/blob_functions/put_blob_str/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_blob_str/main.py rename to tests/emulator_tests/blob_functions/put_blob_str/main.py diff --git a/tests/endtoend/blob_functions/put_blob_trigger/function.json b/tests/emulator_tests/blob_functions/put_blob_trigger/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_blob_trigger/function.json rename to tests/emulator_tests/blob_functions/put_blob_trigger/function.json diff --git a/tests/endtoend/blob_functions/put_blob_trigger/main.py b/tests/emulator_tests/blob_functions/put_blob_trigger/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_blob_trigger/main.py rename to tests/emulator_tests/blob_functions/put_blob_trigger/main.py diff --git a/tests/endtoend/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/function.json b/tests/emulator_tests/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/function.json rename to tests/emulator_tests/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/function.json diff --git a/tests/endtoend/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/main.py b/tests/emulator_tests/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/main.py rename to tests/emulator_tests/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/main.py diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_batch_functions_stein/function_app.py b/tests/emulator_tests/eventhub_batch_functions/eventhub_batch_functions_stein/function_app.py similarity index 100% rename from tests/endtoend/eventhub_batch_functions/eventhub_batch_functions_stein/function_app.py rename to tests/emulator_tests/eventhub_batch_functions/eventhub_batch_functions_stein/function_app.py diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_multiple/__init__.py b/tests/emulator_tests/eventhub_batch_functions/eventhub_multiple/__init__.py similarity index 100% rename from tests/endtoend/eventhub_batch_functions/eventhub_multiple/__init__.py rename to tests/emulator_tests/eventhub_batch_functions/eventhub_multiple/__init__.py diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_multiple/function.json b/tests/emulator_tests/eventhub_batch_functions/eventhub_multiple/function.json similarity index 100% rename from tests/endtoend/eventhub_batch_functions/eventhub_multiple/function.json rename to tests/emulator_tests/eventhub_batch_functions/eventhub_multiple/function.json diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_output_batch/__init__.py b/tests/emulator_tests/eventhub_batch_functions/eventhub_output_batch/__init__.py similarity index 100% rename from tests/endtoend/eventhub_batch_functions/eventhub_output_batch/__init__.py rename to tests/emulator_tests/eventhub_batch_functions/eventhub_output_batch/__init__.py diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_output_batch/function.json b/tests/emulator_tests/eventhub_batch_functions/eventhub_output_batch/function.json similarity index 100% rename from tests/endtoend/eventhub_batch_functions/eventhub_output_batch/function.json rename to tests/emulator_tests/eventhub_batch_functions/eventhub_output_batch/function.json diff --git a/tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/__init__.py b/tests/emulator_tests/eventhub_batch_functions/get_eventhub_batch_triggered/__init__.py similarity index 100% rename from tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/__init__.py rename to tests/emulator_tests/eventhub_batch_functions/get_eventhub_batch_triggered/__init__.py diff --git a/tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/function.json b/tests/emulator_tests/eventhub_batch_functions/get_eventhub_batch_triggered/function.json similarity index 95% rename from tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/function.json rename to tests/emulator_tests/eventhub_batch_functions/get_eventhub_batch_triggered/function.json index 3e8a6995..8ec2e9d6 100644 --- a/tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/function.json +++ b/tests/emulator_tests/eventhub_batch_functions/get_eventhub_batch_triggered/function.json @@ -1,26 +1,26 @@ -{ - "scriptFile": "__init__.py", - "bindings": [ - { - "type": "httpTrigger", - "direction": "in", - "authLevel": "anonymous", - "methods": [ - "get" - ], - "name": "req" - }, - { - "direction": "in", - "type": "blob", - "name": "testEntities", - "path": "python-worker-tests/test-eventhub-batch-triggered.txt", - "connection": "AzureWebJobsStorage" - }, - { - "type": "http", - "direction": "out", - "name": "$return" - } - ] +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "type": "httpTrigger", + "direction": "in", + "authLevel": "anonymous", + "methods": [ + "get" + ], + "name": "req" + }, + { + "direction": "in", + "type": "blob", + "name": "testEntities", + "path": "python-worker-tests/test-eventhub-batch-triggered.txt", + "connection": "AzureWebJobsStorage" + }, + { + "type": "http", + "direction": "out", + "name": "$return" + } + ] } \ No newline at end of file diff --git a/tests/endtoend/eventhub_batch_functions/get_metadata_batch_triggered/__init__.py b/tests/emulator_tests/eventhub_batch_functions/get_metadata_batch_triggered/__init__.py similarity index 100% rename from tests/endtoend/eventhub_batch_functions/get_metadata_batch_triggered/__init__.py rename to tests/emulator_tests/eventhub_batch_functions/get_metadata_batch_triggered/__init__.py diff --git a/tests/endtoend/eventhub_batch_functions/get_metadata_batch_triggered/function.json b/tests/emulator_tests/eventhub_batch_functions/get_metadata_batch_triggered/function.json similarity index 100% rename from tests/endtoend/eventhub_batch_functions/get_metadata_batch_triggered/function.json rename to tests/emulator_tests/eventhub_batch_functions/get_metadata_batch_triggered/function.json diff --git a/tests/endtoend/eventhub_batch_functions/metadata_multiple/__init__.py b/tests/emulator_tests/eventhub_batch_functions/metadata_multiple/__init__.py similarity index 100% rename from tests/endtoend/eventhub_batch_functions/metadata_multiple/__init__.py rename to tests/emulator_tests/eventhub_batch_functions/metadata_multiple/__init__.py diff --git a/tests/endtoend/eventhub_batch_functions/metadata_multiple/function.json b/tests/emulator_tests/eventhub_batch_functions/metadata_multiple/function.json similarity index 100% rename from tests/endtoend/eventhub_batch_functions/metadata_multiple/function.json rename to tests/emulator_tests/eventhub_batch_functions/metadata_multiple/function.json diff --git a/tests/endtoend/eventhub_batch_functions/metadata_output_batch/__init__.py b/tests/emulator_tests/eventhub_batch_functions/metadata_output_batch/__init__.py similarity index 100% rename from tests/endtoend/eventhub_batch_functions/metadata_output_batch/__init__.py rename to tests/emulator_tests/eventhub_batch_functions/metadata_output_batch/__init__.py diff --git a/tests/endtoend/eventhub_batch_functions/metadata_output_batch/function.json b/tests/emulator_tests/eventhub_batch_functions/metadata_output_batch/function.json similarity index 100% rename from tests/endtoend/eventhub_batch_functions/metadata_output_batch/function.json rename to tests/emulator_tests/eventhub_batch_functions/metadata_output_batch/function.json diff --git a/tests/endtoend/eventhub_functions/eventhub_functions_stein/function_app.py b/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/function_app.py similarity index 97% rename from tests/endtoend/eventhub_functions/eventhub_functions_stein/function_app.py rename to tests/emulator_tests/eventhub_functions/eventhub_functions_stein/function_app.py index 1091f8dd..1481f7b5 100644 --- a/tests/endtoend/eventhub_functions/eventhub_functions_stein/function_app.py +++ b/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/function_app.py @@ -1,107 +1,107 @@ -import json -import os -import typing - -import azure.functions as func -from azure.eventhub import EventData -from azure.eventhub.aio import EventHubProducerClient - -app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) - - -# An HttpTrigger to generating EventHub event from EventHub Output Binding -@app.function_name(name="eventhub_output") -@app.route(route="eventhub_output") -@app.event_hub_output(arg_name="event", - event_hub_name="python-worker-ci-eventhub-one", - connection="AzureWebJobsEventHubConnectionString") -def eventhub_output(req: func.HttpRequest, event: func.Out[str]): - event.set(req.get_body().decode('utf-8')) - return 'OK' - - -# This is an actual EventHub trigger which will convert the event data -# into a storage blob. -@app.function_name(name="eventhub_trigger") -@app.event_hub_message_trigger(arg_name="event", - event_hub_name="python-worker-ci-eventhub-one", - connection="AzureWebJobsEventHubConnectionString" - ) -@app.blob_output(arg_name="$return", - path="python-worker-tests/test-eventhub-triggered.txt", - connection="AzureWebJobsStorage") -def eventhub_trigger(event: func.EventHubEvent) -> bytes: - return event.get_body() - - -# Retrieve the event data from storage blob and return it as Http response -@app.function_name(name="get_eventhub_triggered") -@app.route(route="get_eventhub_triggered") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-eventhub-triggered.txt", - connection="AzureWebJobsStorage") -def get_eventhub_triggered(req: func.HttpRequest, - file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -# Retrieve the event data from storage blob and return it as Http response -@app.function_name(name="get_metadata_triggered") -@app.route(route="get_metadata_triggered") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-metadata-triggered.txt", - connection="AzureWebJobsStorage") -async def get_metadata_triggered(req: func.HttpRequest, - file: func.InputStream) -> str: - return func.HttpResponse(body=file.read().decode('utf-8'), - status_code=200, - mimetype='application/json') - - -# An HttpTrigger to generating EventHub event from azure-eventhub SDK. -# Events generated from azure-eventhub contain the full metadata. -@app.function_name(name="metadata_output") -@app.route(route="metadata_output") -async def metadata_output(req: func.HttpRequest): - # Parse event metadata from http request - json_string = req.get_body().decode('utf-8') - event_dict = json.loads(json_string) - - # Create an EventHub Client and event batch - client = EventHubProducerClient.from_connection_string( - os.getenv('AzureWebJobsEventHubConnectionString'), - eventhub_name='python-worker-ci-eventhub-one-metadata') - - # Generate new event based on http request with full metadata - event_data_batch = await client.create_batch() - event_data_batch.add(EventData(event_dict.get('body'))) - - # Send out event into event hub - try: - await client.send_batch(event_data_batch) - finally: - await client.close() - - return 'OK' - - -@app.function_name(name="metadata_trigger") -@app.event_hub_message_trigger( - arg_name="event", - event_hub_name="python-worker-ci-eventhub-one-metadata", - connection="AzureWebJobsEventHubConnectionString") -@app.blob_output(arg_name="$return", - path="python-worker-tests/test-metadata-triggered.txt", - connection="AzureWebJobsStorage") -async def metadata_trigger(event: func.EventHubEvent) -> bytes: - event_dict: typing.Mapping[str, typing.Any] = { - 'body': event.get_body().decode('utf-8'), - # Uncomment this when the EnqueuedTimeUtc is fixed in azure-functions - # 'enqueued_time': event.enqueued_time.isoformat(), - 'partition_key': event.partition_key, - 'sequence_number': event.sequence_number, - 'offset': event.offset, - 'metadata': event.metadata - } - - return json.dumps(event_dict) +import json +import os +import typing + +import azure.functions as func +from azure.eventhub import EventData +from azure.eventhub.aio import EventHubProducerClient + +app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) + + +# An HttpTrigger to generating EventHub event from EventHub Output Binding +@app.function_name(name="eventhub_output") +@app.route(route="eventhub_output") +@app.event_hub_output(arg_name="event", + event_hub_name="python-worker-ci-eventhub-one", + connection="AzureWebJobsEventHubConnectionString") +def eventhub_output(req: func.HttpRequest, event: func.Out[str]): + event.set(req.get_body().decode('utf-8')) + return 'OK' + + +# This is an actual EventHub trigger which will convert the event data +# into a storage blob. +@app.function_name(name="eventhub_trigger") +@app.event_hub_message_trigger(arg_name="event", + event_hub_name="python-worker-ci-eventhub-one", + connection="AzureWebJobsEventHubConnectionString" + ) +@app.blob_output(arg_name="$return", + path="python-worker-tests/test-eventhub-triggered.txt", + connection="AzureWebJobsStorage") +def eventhub_trigger(event: func.EventHubEvent) -> bytes: + return event.get_body() + + +# Retrieve the event data from storage blob and return it as Http response +@app.function_name(name="get_eventhub_triggered") +@app.route(route="get_eventhub_triggered") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-eventhub-triggered.txt", + connection="AzureWebJobsStorage") +def get_eventhub_triggered(req: func.HttpRequest, + file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +# Retrieve the event data from storage blob and return it as Http response +@app.function_name(name="get_metadata_triggered") +@app.route(route="get_metadata_triggered") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-metadata-triggered.txt", + connection="AzureWebJobsStorage") +async def get_metadata_triggered(req: func.HttpRequest, + file: func.InputStream) -> str: + return func.HttpResponse(body=file.read().decode('utf-8'), + status_code=200, + mimetype='application/json') + + +# An HttpTrigger to generating EventHub event from azure-eventhub SDK. +# Events generated from azure-eventhub contain the full metadata. +@app.function_name(name="metadata_output") +@app.route(route="metadata_output") +async def metadata_output(req: func.HttpRequest): + # Parse event metadata from http request + json_string = req.get_body().decode('utf-8') + event_dict = json.loads(json_string) + + # Create an EventHub Client and event batch + client = EventHubProducerClient.from_connection_string( + os.getenv('AzureWebJobsEventHubConnectionString'), + eventhub_name='python-worker-ci-eventhub-one-metadata') + + # Generate new event based on http request with full metadata + event_data_batch = await client.create_batch() + event_data_batch.add(EventData(event_dict.get('body'))) + + # Send out event into event hub + try: + await client.send_batch(event_data_batch) + finally: + await client.close() + + return 'OK' + + +@app.function_name(name="metadata_trigger") +@app.event_hub_message_trigger( + arg_name="event", + event_hub_name="python-worker-ci-eventhub-one-metadata", + connection="AzureWebJobsEventHubConnectionString") +@app.blob_output(arg_name="$return", + path="python-worker-tests/test-metadata-triggered.txt", + connection="AzureWebJobsStorage") +async def metadata_trigger(event: func.EventHubEvent) -> bytes: + event_dict: typing.Mapping[str, typing.Any] = { + 'body': event.get_body().decode('utf-8'), + # Uncomment this when the EnqueuedTimeUtc is fixed in azure-functions + # 'enqueued_time': event.enqueued_time.isoformat(), + 'partition_key': event.partition_key, + 'sequence_number': event.sequence_number, + 'offset': event.offset, + 'metadata': event.metadata + } + + return json.dumps(event_dict) diff --git a/tests/endtoend/eventhub_functions/eventhub_functions_stein/generic/function_app.py b/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/generic/function_app.py similarity index 100% rename from tests/endtoend/eventhub_functions/eventhub_functions_stein/generic/function_app.py rename to tests/emulator_tests/eventhub_functions/eventhub_functions_stein/generic/function_app.py diff --git a/tests/endtoend/eventhub_functions/eventhub_output/__init__.py b/tests/emulator_tests/eventhub_functions/eventhub_output/__init__.py similarity index 100% rename from tests/endtoend/eventhub_functions/eventhub_output/__init__.py rename to tests/emulator_tests/eventhub_functions/eventhub_output/__init__.py diff --git a/tests/endtoend/eventhub_functions/eventhub_output/function.json b/tests/emulator_tests/eventhub_functions/eventhub_output/function.json similarity index 100% rename from tests/endtoend/eventhub_functions/eventhub_output/function.json rename to tests/emulator_tests/eventhub_functions/eventhub_output/function.json diff --git a/tests/endtoend/eventhub_functions/eventhub_trigger/__init__.py b/tests/emulator_tests/eventhub_functions/eventhub_trigger/__init__.py similarity index 100% rename from tests/endtoend/eventhub_functions/eventhub_trigger/__init__.py rename to tests/emulator_tests/eventhub_functions/eventhub_trigger/__init__.py diff --git a/tests/endtoend/eventhub_functions/eventhub_trigger/function.json b/tests/emulator_tests/eventhub_functions/eventhub_trigger/function.json similarity index 100% rename from tests/endtoend/eventhub_functions/eventhub_trigger/function.json rename to tests/emulator_tests/eventhub_functions/eventhub_trigger/function.json diff --git a/tests/endtoend/eventhub_functions/get_eventhub_triggered/function.json b/tests/emulator_tests/eventhub_functions/get_eventhub_triggered/function.json similarity index 100% rename from tests/endtoend/eventhub_functions/get_eventhub_triggered/function.json rename to tests/emulator_tests/eventhub_functions/get_eventhub_triggered/function.json diff --git a/tests/endtoend/eventhub_functions/get_eventhub_triggered/main.py b/tests/emulator_tests/eventhub_functions/get_eventhub_triggered/main.py similarity index 100% rename from tests/endtoend/eventhub_functions/get_eventhub_triggered/main.py rename to tests/emulator_tests/eventhub_functions/get_eventhub_triggered/main.py diff --git a/tests/endtoend/eventhub_functions/get_metadata_triggered/__init__.py b/tests/emulator_tests/eventhub_functions/get_metadata_triggered/__init__.py similarity index 100% rename from tests/endtoend/eventhub_functions/get_metadata_triggered/__init__.py rename to tests/emulator_tests/eventhub_functions/get_metadata_triggered/__init__.py diff --git a/tests/endtoend/eventhub_functions/get_metadata_triggered/function.json b/tests/emulator_tests/eventhub_functions/get_metadata_triggered/function.json similarity index 100% rename from tests/endtoend/eventhub_functions/get_metadata_triggered/function.json rename to tests/emulator_tests/eventhub_functions/get_metadata_triggered/function.json diff --git a/tests/endtoend/eventhub_functions/metadata_output/__init__.py b/tests/emulator_tests/eventhub_functions/metadata_output/__init__.py similarity index 100% rename from tests/endtoend/eventhub_functions/metadata_output/__init__.py rename to tests/emulator_tests/eventhub_functions/metadata_output/__init__.py diff --git a/tests/endtoend/eventhub_functions/metadata_output/function.json b/tests/emulator_tests/eventhub_functions/metadata_output/function.json similarity index 100% rename from tests/endtoend/eventhub_functions/metadata_output/function.json rename to tests/emulator_tests/eventhub_functions/metadata_output/function.json diff --git a/tests/endtoend/eventhub_functions/metadata_trigger/__init__.py b/tests/emulator_tests/eventhub_functions/metadata_trigger/__init__.py similarity index 100% rename from tests/endtoend/eventhub_functions/metadata_trigger/__init__.py rename to tests/emulator_tests/eventhub_functions/metadata_trigger/__init__.py diff --git a/tests/endtoend/eventhub_functions/metadata_trigger/function.json b/tests/emulator_tests/eventhub_functions/metadata_trigger/function.json similarity index 100% rename from tests/endtoend/eventhub_functions/metadata_trigger/function.json rename to tests/emulator_tests/eventhub_functions/metadata_trigger/function.json diff --git a/tests/endtoend/generic_functions/generic_functions_stein/function_app.py b/tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py similarity index 82% rename from tests/endtoend/generic_functions/generic_functions_stein/function_app.py rename to tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py index 654148c9..2da6d44c 100644 --- a/tests/endtoend/generic_functions/generic_functions_stein/function_app.py +++ b/tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import json import logging +import uuid import azure.functions as func @@ -15,7 +17,7 @@ arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_processed_last(req: func.HttpRequest, testEntity): return func.HttpResponse(status_code=200) @@ -28,7 +30,7 @@ def return_processed_last(req: func.HttpRequest, testEntity): arg_name="testEntities", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_not_processed_last(req: func.HttpRequest, testEntities): return func.HttpResponse(status_code=200) @@ -41,7 +43,7 @@ def return_not_processed_last(req: func.HttpRequest, testEntities): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def mytimer(mytimer: func.TimerRequest, testEntity) -> None: logging.info("This timer trigger function executed successfully") @@ -54,7 +56,7 @@ def mytimer(mytimer: func.TimerRequest, testEntity) -> None: arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_string(mytimer: func.TimerRequest, testEntity): logging.info("Return string") return "hi!" @@ -68,7 +70,7 @@ def return_string(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_bytes(mytimer: func.TimerRequest, testEntity): logging.info("Return bytes") return "test-dată" @@ -82,7 +84,7 @@ def return_bytes(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_dict(mytimer: func.TimerRequest, testEntity): logging.info("Return dict") return {"hello": "world"} @@ -96,7 +98,7 @@ def return_dict(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_list(mytimer: func.TimerRequest, testEntity): logging.info("Return list") return [1, 2, 3] @@ -110,7 +112,7 @@ def return_list(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_int(mytimer: func.TimerRequest, testEntity): logging.info("Return int") return 12 @@ -124,7 +126,7 @@ def return_int(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_double(mytimer: func.TimerRequest, testEntity): logging.info("Return double") return 12.34 @@ -138,7 +140,20 @@ def return_double(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_bool(mytimer: func.TimerRequest, testEntity): logging.info("Return bool") return True + + +@app.function_name(name="table_out_binding") +@app.route(route="table_out_binding", binding_arg_name="resp") +@app.table_output(arg_name="$return", + connection="AzureWebJobsStorage", + table_name="BindingTestTable") +def table_out_binding(req: func.HttpRequest, resp: func.Out[func.HttpResponse]): + row_key_uuid = str(uuid.uuid4()) + table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid} + table_json = json.dumps(table_dict) + resp.set(table_json) + return table_json diff --git a/tests/endtoend/generic_functions/return_bool/function.json b/tests/emulator_tests/generic_functions/return_bool/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_bool/function.json rename to tests/emulator_tests/generic_functions/return_bool/function.json diff --git a/tests/endtoend/generic_functions/return_bool/main.py b/tests/emulator_tests/generic_functions/return_bool/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_bool/main.py rename to tests/emulator_tests/generic_functions/return_bool/main.py diff --git a/tests/endtoend/generic_functions/return_bytes/function.json b/tests/emulator_tests/generic_functions/return_bytes/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_bytes/function.json rename to tests/emulator_tests/generic_functions/return_bytes/function.json diff --git a/tests/endtoend/generic_functions/return_bytes/main.py b/tests/emulator_tests/generic_functions/return_bytes/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_bytes/main.py rename to tests/emulator_tests/generic_functions/return_bytes/main.py diff --git a/tests/endtoend/generic_functions/return_dict/function.json b/tests/emulator_tests/generic_functions/return_dict/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_dict/function.json rename to tests/emulator_tests/generic_functions/return_dict/function.json diff --git a/tests/endtoend/generic_functions/return_dict/main.py b/tests/emulator_tests/generic_functions/return_dict/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_dict/main.py rename to tests/emulator_tests/generic_functions/return_dict/main.py diff --git a/tests/endtoend/generic_functions/return_double/function.json b/tests/emulator_tests/generic_functions/return_double/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_double/function.json rename to tests/emulator_tests/generic_functions/return_double/function.json diff --git a/tests/endtoend/generic_functions/return_double/main.py b/tests/emulator_tests/generic_functions/return_double/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_double/main.py rename to tests/emulator_tests/generic_functions/return_double/main.py diff --git a/tests/endtoend/generic_functions/return_int/function.json b/tests/emulator_tests/generic_functions/return_int/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_int/function.json rename to tests/emulator_tests/generic_functions/return_int/function.json diff --git a/tests/endtoend/generic_functions/return_int/main.py b/tests/emulator_tests/generic_functions/return_int/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_int/main.py rename to tests/emulator_tests/generic_functions/return_int/main.py diff --git a/tests/endtoend/generic_functions/return_list/function.json b/tests/emulator_tests/generic_functions/return_list/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_list/function.json rename to tests/emulator_tests/generic_functions/return_list/function.json diff --git a/tests/endtoend/generic_functions/return_list/main.py b/tests/emulator_tests/generic_functions/return_list/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_list/main.py rename to tests/emulator_tests/generic_functions/return_list/main.py diff --git a/tests/endtoend/generic_functions/return_none/function.json b/tests/emulator_tests/generic_functions/return_none/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_none/function.json rename to tests/emulator_tests/generic_functions/return_none/function.json diff --git a/tests/endtoend/generic_functions/return_none/main.py b/tests/emulator_tests/generic_functions/return_none/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_none/main.py rename to tests/emulator_tests/generic_functions/return_none/main.py diff --git a/tests/endtoend/generic_functions/return_none_no_type_hint/function.json b/tests/emulator_tests/generic_functions/return_none_no_type_hint/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_none_no_type_hint/function.json rename to tests/emulator_tests/generic_functions/return_none_no_type_hint/function.json diff --git a/tests/endtoend/generic_functions/return_none_no_type_hint/main.py b/tests/emulator_tests/generic_functions/return_none_no_type_hint/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_none_no_type_hint/main.py rename to tests/emulator_tests/generic_functions/return_none_no_type_hint/main.py diff --git a/tests/endtoend/generic_functions/return_not_processed_last/__init__.py b/tests/emulator_tests/generic_functions/return_not_processed_last/__init__.py similarity index 100% rename from tests/endtoend/generic_functions/return_not_processed_last/__init__.py rename to tests/emulator_tests/generic_functions/return_not_processed_last/__init__.py diff --git a/tests/endtoend/generic_functions/return_not_processed_last/function.json b/tests/emulator_tests/generic_functions/return_not_processed_last/function.json similarity index 91% rename from tests/endtoend/generic_functions/return_not_processed_last/function.json rename to tests/emulator_tests/generic_functions/return_not_processed_last/function.json index 66d1e80e..e02ae4d1 100644 --- a/tests/endtoend/generic_functions/return_not_processed_last/function.json +++ b/tests/emulator_tests/generic_functions/return_not_processed_last/function.json @@ -14,7 +14,7 @@ "direction": "in", "type": "table", "name": "testEntities", - "tableName": "EventHubBatchTest", + "tableName": "BindingTestTable", "connection": "AzureWebJobsStorage" }, { diff --git a/tests/endtoend/generic_functions/return_processed_last/__init__.py b/tests/emulator_tests/generic_functions/return_processed_last/__init__.py similarity index 100% rename from tests/endtoend/generic_functions/return_processed_last/__init__.py rename to tests/emulator_tests/generic_functions/return_processed_last/__init__.py diff --git a/tests/endtoend/generic_functions/return_processed_last/function.json b/tests/emulator_tests/generic_functions/return_processed_last/function.json similarity index 91% rename from tests/endtoend/generic_functions/return_processed_last/function.json rename to tests/emulator_tests/generic_functions/return_processed_last/function.json index 82ac266a..d23f01a8 100644 --- a/tests/endtoend/generic_functions/return_processed_last/function.json +++ b/tests/emulator_tests/generic_functions/return_processed_last/function.json @@ -14,7 +14,7 @@ "direction": "in", "type": "table", "name": "testEntity", - "tableName": "EventHubBatchTest", + "tableName": "BindingTestTable", "connection": "AzureWebJobsStorage" }, { diff --git a/tests/endtoend/generic_functions/return_string/function.json b/tests/emulator_tests/generic_functions/return_string/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_string/function.json rename to tests/emulator_tests/generic_functions/return_string/function.json diff --git a/tests/endtoend/generic_functions/return_string/main.py b/tests/emulator_tests/generic_functions/return_string/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_string/main.py rename to tests/emulator_tests/generic_functions/return_string/main.py diff --git a/tests/emulator_tests/generic_functions/table_out_binding/__init__.py b/tests/emulator_tests/generic_functions/table_out_binding/__init__.py new file mode 100644 index 00000000..09c7058e --- /dev/null +++ b/tests/emulator_tests/generic_functions/table_out_binding/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import json +import uuid +import azure.functions as func + + +def main(req: func.HttpRequest, resp: func.Out[func.HttpResponse]): + row_key_uuid = str(uuid.uuid4()) + table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid} + table_json = json.dumps(table_dict) + resp.set(table_json) + return table_json diff --git a/tests/emulator_tests/generic_functions/table_out_binding/function.json b/tests/emulator_tests/generic_functions/table_out_binding/function.json new file mode 100644 index 00000000..25537873 --- /dev/null +++ b/tests/emulator_tests/generic_functions/table_out_binding/function.json @@ -0,0 +1,24 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "type": "httpTrigger", + "direction": "in", + "authLevel": "anonymous", + "methods": ["post"], + "name": "req" + }, + { + "direction": "out", + "type": "table", + "name": "$return", + "tableName": "BindingTestTable", + "connection": "AzureWebJobsStorage" + }, + { + "name": "resp", + "type": "http", + "direction": "out" + } + ] + } \ No newline at end of file diff --git a/tests/endtoend/queue_functions/get_queue_blob/function.json b/tests/emulator_tests/queue_functions/get_queue_blob/function.json similarity index 100% rename from tests/endtoend/queue_functions/get_queue_blob/function.json rename to tests/emulator_tests/queue_functions/get_queue_blob/function.json diff --git a/tests/endtoend/queue_functions/get_queue_blob/main.py b/tests/emulator_tests/queue_functions/get_queue_blob/main.py similarity index 100% rename from tests/endtoend/queue_functions/get_queue_blob/main.py rename to tests/emulator_tests/queue_functions/get_queue_blob/main.py diff --git a/tests/endtoend/queue_functions/get_queue_blob_message_return/function.json b/tests/emulator_tests/queue_functions/get_queue_blob_message_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/get_queue_blob_message_return/function.json rename to tests/emulator_tests/queue_functions/get_queue_blob_message_return/function.json diff --git a/tests/endtoend/queue_functions/get_queue_blob_message_return/main.py b/tests/emulator_tests/queue_functions/get_queue_blob_message_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/get_queue_blob_message_return/main.py rename to tests/emulator_tests/queue_functions/get_queue_blob_message_return/main.py diff --git a/tests/endtoend/queue_functions/get_queue_blob_return/function.json b/tests/emulator_tests/queue_functions/get_queue_blob_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/get_queue_blob_return/function.json rename to tests/emulator_tests/queue_functions/get_queue_blob_return/function.json diff --git a/tests/endtoend/queue_functions/get_queue_blob_return/main.py b/tests/emulator_tests/queue_functions/get_queue_blob_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/get_queue_blob_return/main.py rename to tests/emulator_tests/queue_functions/get_queue_blob_return/main.py diff --git a/tests/endtoend/queue_functions/get_queue_untyped_blob_return/function.json b/tests/emulator_tests/queue_functions/get_queue_untyped_blob_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/get_queue_untyped_blob_return/function.json rename to tests/emulator_tests/queue_functions/get_queue_untyped_blob_return/function.json diff --git a/tests/endtoend/queue_functions/get_queue_untyped_blob_return/main.py b/tests/emulator_tests/queue_functions/get_queue_untyped_blob_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/get_queue_untyped_blob_return/main.py rename to tests/emulator_tests/queue_functions/get_queue_untyped_blob_return/main.py diff --git a/tests/endtoend/queue_functions/put_queue/function.json b/tests/emulator_tests/queue_functions/put_queue/function.json similarity index 100% rename from tests/endtoend/queue_functions/put_queue/function.json rename to tests/emulator_tests/queue_functions/put_queue/function.json diff --git a/tests/endtoend/queue_functions/put_queue/main.py b/tests/emulator_tests/queue_functions/put_queue/main.py similarity index 100% rename from tests/endtoend/queue_functions/put_queue/main.py rename to tests/emulator_tests/queue_functions/put_queue/main.py diff --git a/tests/endtoend/queue_functions/put_queue_message_return/function.json b/tests/emulator_tests/queue_functions/put_queue_message_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/put_queue_message_return/function.json rename to tests/emulator_tests/queue_functions/put_queue_message_return/function.json diff --git a/tests/endtoend/queue_functions/put_queue_message_return/main.py b/tests/emulator_tests/queue_functions/put_queue_message_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/put_queue_message_return/main.py rename to tests/emulator_tests/queue_functions/put_queue_message_return/main.py diff --git a/tests/endtoend/queue_functions/put_queue_multiple_out/function.json b/tests/emulator_tests/queue_functions/put_queue_multiple_out/function.json similarity index 100% rename from tests/endtoend/queue_functions/put_queue_multiple_out/function.json rename to tests/emulator_tests/queue_functions/put_queue_multiple_out/function.json diff --git a/tests/endtoend/queue_functions/put_queue_multiple_out/main.py b/tests/emulator_tests/queue_functions/put_queue_multiple_out/main.py similarity index 100% rename from tests/endtoend/queue_functions/put_queue_multiple_out/main.py rename to tests/emulator_tests/queue_functions/put_queue_multiple_out/main.py diff --git a/tests/endtoend/queue_functions/put_queue_return/function.json b/tests/emulator_tests/queue_functions/put_queue_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/put_queue_return/function.json rename to tests/emulator_tests/queue_functions/put_queue_return/function.json diff --git a/tests/endtoend/queue_functions/put_queue_return/main.py b/tests/emulator_tests/queue_functions/put_queue_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/put_queue_return/main.py rename to tests/emulator_tests/queue_functions/put_queue_return/main.py diff --git a/tests/endtoend/queue_functions/put_queue_return_multiple/function.json b/tests/emulator_tests/queue_functions/put_queue_return_multiple/function.json similarity index 100% rename from tests/endtoend/queue_functions/put_queue_return_multiple/function.json rename to tests/emulator_tests/queue_functions/put_queue_return_multiple/function.json diff --git a/tests/endtoend/queue_functions/put_queue_return_multiple/main.py b/tests/emulator_tests/queue_functions/put_queue_return_multiple/main.py similarity index 100% rename from tests/endtoend/queue_functions/put_queue_return_multiple/main.py rename to tests/emulator_tests/queue_functions/put_queue_return_multiple/main.py diff --git a/tests/endtoend/queue_functions/put_queue_untyped_return/function.json b/tests/emulator_tests/queue_functions/put_queue_untyped_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/put_queue_untyped_return/function.json rename to tests/emulator_tests/queue_functions/put_queue_untyped_return/function.json diff --git a/tests/endtoend/queue_functions/put_queue_untyped_return/main.py b/tests/emulator_tests/queue_functions/put_queue_untyped_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/put_queue_untyped_return/main.py rename to tests/emulator_tests/queue_functions/put_queue_untyped_return/main.py diff --git a/tests/endtoend/queue_functions/queue_functions_stein/function_app.py b/tests/emulator_tests/queue_functions/queue_functions_stein/function_app.py similarity index 97% rename from tests/endtoend/queue_functions/queue_functions_stein/function_app.py rename to tests/emulator_tests/queue_functions/queue_functions_stein/function_app.py index 0b883fb1..087cf459 100644 --- a/tests/endtoend/queue_functions/queue_functions_stein/function_app.py +++ b/tests/emulator_tests/queue_functions/queue_functions_stein/function_app.py @@ -1,185 +1,185 @@ -import json -import logging -import typing - -import azure.functions as func - -app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) - - -@app.function_name(name="get_queue_blob") -@app.route(route="get_queue_blob") -@app.blob_input(arg_name="file", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-blob.txt") -def get_queue_blob(req: func.HttpRequest, file: func.InputStream) -> str: - return json.dumps({ - 'queue': json.loads(file.read().decode('utf-8')) - }) - - -@app.function_name(name="get_queue_blob_message_return") -@app.route(route="get_queue_blob_message_return") -@app.blob_input(arg_name="file", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-blob-message-return.txt") -def get_queue_blob_message_return(req: func.HttpRequest, - file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="get_queue_blob_return") -@app.route(route="get_queue_blob_return") -@app.blob_input(arg_name="file", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-blob-return.txt") -def get_queue_blob_return(req: func.HttpRequest, file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="get_queue_untyped_blob_return") -@app.route(route="get_queue_untyped_blob_return") -@app.blob_input(arg_name="file", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-untyped-blob-return.txt") -def get_queue_untyped_blob_return(req: func.HttpRequest, - file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="put_queue") -@app.route(route="put_queue") -@app.queue_output(arg_name="msg", - connection="AzureWebJobsStorage", - queue_name="testqueue") -def put_queue(req: func.HttpRequest, msg: func.Out[str]): - msg.set(req.get_body()) - - return 'OK' - - -@app.function_name(name="put_queue_message_return") -@app.route(route="put_queue_message_return", binding_arg_name="resp") -@app.queue_output(arg_name="$return", - connection="AzureWebJobsStorage", - queue_name="testqueue-message-return") -def main(req: func.HttpRequest, resp: func.Out[str]) -> bytes: - return func.QueueMessage(body=req.get_body()) - - -@app.function_name("put_queue_multiple_out") -@app.route(route="put_queue_multiple_out", binding_arg_name="resp") -@app.queue_output(arg_name="msg", - connection="AzureWebJobsStorage", - queue_name="testqueue-return-multiple-outparam") -def put_queue_multiple_out(req: func.HttpRequest, - resp: func.Out[func.HttpResponse], - msg: func.Out[func.QueueMessage]) -> None: - data = req.get_body().decode() - msg.set(func.QueueMessage(body=data)) - resp.set(func.HttpResponse(body='HTTP response: {}'.format(data))) - - -@app.function_name("put_queue_return") -@app.route(route="put_queue_return", binding_arg_name="resp") -@app.queue_output(arg_name="$return", - connection="AzureWebJobsStorage", - queue_name="testqueue-return") -def put_queue_return(req: func.HttpRequest, resp: func.Out[str]) -> bytes: - return req.get_body() - - -@app.function_name(name="put_queue_multiple_return") -@app.route(route="put_queue_multiple_return") -@app.queue_output(arg_name="msgs", - connection="AzureWebJobsStorage", - queue_name="testqueue-return-multiple") -def put_queue_multiple_return(req: func.HttpRequest, - msgs: func.Out[typing.List[str]]): - msgs.set(['one', 'two']) - - -@app.function_name(name="put_queue_untyped_return") -@app.route(route="put_queue_untyped_return", binding_arg_name="resp") -@app.queue_output(arg_name="$return", - connection="AzureWebJobsStorage", - queue_name="testqueue-untyped-return") -def put_queue_untyped_return(req: func.HttpRequest, - resp: func.Out[str]) -> bytes: - return func.QueueMessage(body=req.get_body()) - - -@app.function_name(name="queue_trigger") -@app.queue_trigger(arg_name="msg", - queue_name="testqueue", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="$return", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-blob.txt") -def queue_trigger(msg: func.QueueMessage) -> str: - result = json.dumps({ - 'id': msg.id, - 'body': msg.get_body().decode('utf-8'), - 'expiration_time': (msg.expiration_time.isoformat() - if msg.expiration_time else None), - 'insertion_time': (msg.insertion_time.isoformat() - if msg.insertion_time else None), - 'time_next_visible': (msg.time_next_visible.isoformat() - if msg.time_next_visible else None), - 'pop_receipt': msg.pop_receipt, - 'dequeue_count': msg.dequeue_count - }) - - return result - - -@app.function_name(name="queue_trigger_message_return") -@app.queue_trigger(arg_name="msg", - queue_name="testqueue-message-return", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="$return", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-blob-message-return.txt") -def queue_trigger_message_return(msg: func.QueueMessage) -> bytes: - return msg.get_body() - - -@app.function_name(name="queue_trigger_return") -@app.queue_trigger(arg_name="msg", - queue_name="testqueue-return", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="$return", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-blob-return.txt") -def queue_trigger_return(msg: func.QueueMessage) -> bytes: - return msg.get_body() - - -@app.function_name(name="queue_trigger_return_multiple") -@app.queue_trigger(arg_name="msg", - queue_name="testqueue-return-multiple", - connection="AzureWebJobsStorage") -def queue_trigger_return_multiple(msg: func.QueueMessage) -> None: - logging.info('trigger on message: %s', msg.get_body().decode('utf-8')) - - -@app.function_name(name="queue_trigger_untyped") -@app.queue_trigger(arg_name="msg", - queue_name="testqueue-untyped-return", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="$return", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-untyped-blob-return.txt") -def queue_trigger_untyped(msg: str) -> str: - return msg - - -@app.function_name(name="put_queue_return_multiple") -@app.route(route="put_queue_return_multiple", binding_arg_name="resp") -@app.queue_output(arg_name="msgs", - connection="AzureWebJobsStorage", - queue_name="testqueue-return-multiple") -def put_queue_return_multiple(req: func.HttpRequest, - resp: func.Out[str], - msgs: func.Out[typing.List[str]]): - msgs.set(['one', 'two']) +import json +import logging +import typing + +import azure.functions as func + +app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) + + +@app.function_name(name="get_queue_blob") +@app.route(route="get_queue_blob") +@app.blob_input(arg_name="file", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-blob.txt") +def get_queue_blob(req: func.HttpRequest, file: func.InputStream) -> str: + return json.dumps({ + 'queue': json.loads(file.read().decode('utf-8')) + }) + + +@app.function_name(name="get_queue_blob_message_return") +@app.route(route="get_queue_blob_message_return") +@app.blob_input(arg_name="file", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-blob-message-return.txt") +def get_queue_blob_message_return(req: func.HttpRequest, + file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="get_queue_blob_return") +@app.route(route="get_queue_blob_return") +@app.blob_input(arg_name="file", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-blob-return.txt") +def get_queue_blob_return(req: func.HttpRequest, file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="get_queue_untyped_blob_return") +@app.route(route="get_queue_untyped_blob_return") +@app.blob_input(arg_name="file", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-untyped-blob-return.txt") +def get_queue_untyped_blob_return(req: func.HttpRequest, + file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="put_queue") +@app.route(route="put_queue") +@app.queue_output(arg_name="msg", + connection="AzureWebJobsStorage", + queue_name="testqueue") +def put_queue(req: func.HttpRequest, msg: func.Out[str]): + msg.set(req.get_body()) + + return 'OK' + + +@app.function_name(name="put_queue_message_return") +@app.route(route="put_queue_message_return", binding_arg_name="resp") +@app.queue_output(arg_name="$return", + connection="AzureWebJobsStorage", + queue_name="testqueue-message-return") +def main(req: func.HttpRequest, resp: func.Out[str]) -> bytes: + return func.QueueMessage(body=req.get_body()) + + +@app.function_name("put_queue_multiple_out") +@app.route(route="put_queue_multiple_out", binding_arg_name="resp") +@app.queue_output(arg_name="msg", + connection="AzureWebJobsStorage", + queue_name="testqueue-return-multiple-outparam") +def put_queue_multiple_out(req: func.HttpRequest, + resp: func.Out[func.HttpResponse], + msg: func.Out[func.QueueMessage]) -> None: + data = req.get_body().decode() + msg.set(func.QueueMessage(body=data)) + resp.set(func.HttpResponse(body='HTTP response: {}'.format(data))) + + +@app.function_name("put_queue_return") +@app.route(route="put_queue_return", binding_arg_name="resp") +@app.queue_output(arg_name="$return", + connection="AzureWebJobsStorage", + queue_name="testqueue-return") +def put_queue_return(req: func.HttpRequest, resp: func.Out[str]) -> bytes: + return req.get_body() + + +@app.function_name(name="put_queue_multiple_return") +@app.route(route="put_queue_multiple_return") +@app.queue_output(arg_name="msgs", + connection="AzureWebJobsStorage", + queue_name="testqueue-return-multiple") +def put_queue_multiple_return(req: func.HttpRequest, + msgs: func.Out[typing.List[str]]): + msgs.set(['one', 'two']) + + +@app.function_name(name="put_queue_untyped_return") +@app.route(route="put_queue_untyped_return", binding_arg_name="resp") +@app.queue_output(arg_name="$return", + connection="AzureWebJobsStorage", + queue_name="testqueue-untyped-return") +def put_queue_untyped_return(req: func.HttpRequest, + resp: func.Out[str]) -> bytes: + return func.QueueMessage(body=req.get_body()) + + +@app.function_name(name="queue_trigger") +@app.queue_trigger(arg_name="msg", + queue_name="testqueue", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="$return", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-blob.txt") +def queue_trigger(msg: func.QueueMessage) -> str: + result = json.dumps({ + 'id': msg.id, + 'body': msg.get_body().decode('utf-8'), + 'expiration_time': (msg.expiration_time.isoformat() + if msg.expiration_time else None), + 'insertion_time': (msg.insertion_time.isoformat() + if msg.insertion_time else None), + 'time_next_visible': (msg.time_next_visible.isoformat() + if msg.time_next_visible else None), + 'pop_receipt': msg.pop_receipt, + 'dequeue_count': msg.dequeue_count + }) + + return result + + +@app.function_name(name="queue_trigger_message_return") +@app.queue_trigger(arg_name="msg", + queue_name="testqueue-message-return", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="$return", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-blob-message-return.txt") +def queue_trigger_message_return(msg: func.QueueMessage) -> bytes: + return msg.get_body() + + +@app.function_name(name="queue_trigger_return") +@app.queue_trigger(arg_name="msg", + queue_name="testqueue-return", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="$return", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-blob-return.txt") +def queue_trigger_return(msg: func.QueueMessage) -> bytes: + return msg.get_body() + + +@app.function_name(name="queue_trigger_return_multiple") +@app.queue_trigger(arg_name="msg", + queue_name="testqueue-return-multiple", + connection="AzureWebJobsStorage") +def queue_trigger_return_multiple(msg: func.QueueMessage) -> None: + logging.info('trigger on message: %s', msg.get_body().decode('utf-8')) + + +@app.function_name(name="queue_trigger_untyped") +@app.queue_trigger(arg_name="msg", + queue_name="testqueue-untyped-return", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="$return", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-untyped-blob-return.txt") +def queue_trigger_untyped(msg: str) -> str: + return msg + + +@app.function_name(name="put_queue_return_multiple") +@app.route(route="put_queue_return_multiple", binding_arg_name="resp") +@app.queue_output(arg_name="msgs", + connection="AzureWebJobsStorage", + queue_name="testqueue-return-multiple") +def put_queue_return_multiple(req: func.HttpRequest, + resp: func.Out[str], + msgs: func.Out[typing.List[str]]): + msgs.set(['one', 'two']) diff --git a/tests/endtoend/queue_functions/queue_functions_stein/generic/function_app.py b/tests/emulator_tests/queue_functions/queue_functions_stein/generic/function_app.py similarity index 100% rename from tests/endtoend/queue_functions/queue_functions_stein/generic/function_app.py rename to tests/emulator_tests/queue_functions/queue_functions_stein/generic/function_app.py diff --git a/tests/endtoend/queue_functions/queue_trigger/function.json b/tests/emulator_tests/queue_functions/queue_trigger/function.json similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger/function.json rename to tests/emulator_tests/queue_functions/queue_trigger/function.json diff --git a/tests/endtoend/queue_functions/queue_trigger/main.py b/tests/emulator_tests/queue_functions/queue_trigger/main.py similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger/main.py rename to tests/emulator_tests/queue_functions/queue_trigger/main.py diff --git a/tests/endtoend/queue_functions/queue_trigger_message_return/function.json b/tests/emulator_tests/queue_functions/queue_trigger_message_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_message_return/function.json rename to tests/emulator_tests/queue_functions/queue_trigger_message_return/function.json diff --git a/tests/endtoend/queue_functions/queue_trigger_message_return/main.py b/tests/emulator_tests/queue_functions/queue_trigger_message_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_message_return/main.py rename to tests/emulator_tests/queue_functions/queue_trigger_message_return/main.py diff --git a/tests/endtoend/queue_functions/queue_trigger_return/function.json b/tests/emulator_tests/queue_functions/queue_trigger_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_return/function.json rename to tests/emulator_tests/queue_functions/queue_trigger_return/function.json diff --git a/tests/endtoend/queue_functions/queue_trigger_return/main.py b/tests/emulator_tests/queue_functions/queue_trigger_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_return/main.py rename to tests/emulator_tests/queue_functions/queue_trigger_return/main.py diff --git a/tests/endtoend/queue_functions/queue_trigger_return_multiple/function.json b/tests/emulator_tests/queue_functions/queue_trigger_return_multiple/function.json similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_return_multiple/function.json rename to tests/emulator_tests/queue_functions/queue_trigger_return_multiple/function.json diff --git a/tests/endtoend/queue_functions/queue_trigger_return_multiple/main.py b/tests/emulator_tests/queue_functions/queue_trigger_return_multiple/main.py similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_return_multiple/main.py rename to tests/emulator_tests/queue_functions/queue_trigger_return_multiple/main.py diff --git a/tests/endtoend/queue_functions/queue_trigger_untyped/function.json b/tests/emulator_tests/queue_functions/queue_trigger_untyped/function.json similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_untyped/function.json rename to tests/emulator_tests/queue_functions/queue_trigger_untyped/function.json diff --git a/tests/endtoend/queue_functions/queue_trigger_untyped/main.py b/tests/emulator_tests/queue_functions/queue_trigger_untyped/main.py similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_untyped/main.py rename to tests/emulator_tests/queue_functions/queue_trigger_untyped/main.py diff --git a/tests/endtoend/servicebus_functions/get_servicebus_triggered/__init__.py b/tests/emulator_tests/servicebus_functions/get_servicebus_triggered/__init__.py similarity index 100% rename from tests/endtoend/servicebus_functions/get_servicebus_triggered/__init__.py rename to tests/emulator_tests/servicebus_functions/get_servicebus_triggered/__init__.py diff --git a/tests/endtoend/servicebus_functions/get_servicebus_triggered/function.json b/tests/emulator_tests/servicebus_functions/get_servicebus_triggered/function.json similarity index 100% rename from tests/endtoend/servicebus_functions/get_servicebus_triggered/function.json rename to tests/emulator_tests/servicebus_functions/get_servicebus_triggered/function.json diff --git a/tests/endtoend/servicebus_functions/put_message/__init__.py b/tests/emulator_tests/servicebus_functions/put_message/__init__.py similarity index 100% rename from tests/endtoend/servicebus_functions/put_message/__init__.py rename to tests/emulator_tests/servicebus_functions/put_message/__init__.py diff --git a/tests/endtoend/servicebus_functions/put_message/function.json b/tests/emulator_tests/servicebus_functions/put_message/function.json similarity index 100% rename from tests/endtoend/servicebus_functions/put_message/function.json rename to tests/emulator_tests/servicebus_functions/put_message/function.json diff --git a/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py b/tests/emulator_tests/servicebus_functions/servicebus_functions_stein/function_app.py similarity index 97% rename from tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py rename to tests/emulator_tests/servicebus_functions/servicebus_functions_stein/function_app.py index 4064085b..9e9d1224 100644 --- a/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py +++ b/tests/emulator_tests/servicebus_functions/servicebus_functions_stein/function_app.py @@ -1,73 +1,73 @@ -import json - -import azure.functions as func - -app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) - - -@app.route(route="put_message") -@app.service_bus_queue_output( - arg_name="msg", - connection="AzureWebJobsServiceBusConnectionString", - queue_name="testqueue") -def put_message(req: func.HttpRequest, msg: func.Out[str]): - msg.set(req.get_body().decode('utf-8')) - return 'OK' - - -@app.route(route="get_servicebus_triggered") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-servicebus-triggered.txt", - connection="AzureWebJobsStorage") -def get_servicebus_triggered(req: func.HttpRequest, - file: func.InputStream) -> str: - return func.HttpResponse( - file.read().decode('utf-8'), mimetype='application/json') - - -@app.service_bus_queue_trigger( - arg_name="msg", - connection="AzureWebJobsServiceBusConnectionString", - queue_name="testqueue") -@app.blob_output(arg_name="$return", - path="python-worker-tests/test-servicebus-triggered.txt", - connection="AzureWebJobsStorage") -def servicebus_trigger(msg: func.ServiceBusMessage) -> str: - result = json.dumps({ - 'message_id': msg.message_id, - 'body': msg.get_body().decode('utf-8'), - 'content_type': msg.content_type, - 'delivery_count': msg.delivery_count, - 'expiration_time': (msg.expiration_time.isoformat() if - msg.expiration_time else None), - 'label': msg.label, - 'partition_key': msg.partition_key, - 'reply_to': msg.reply_to, - 'reply_to_session_id': msg.reply_to_session_id, - 'scheduled_enqueue_time': (msg.scheduled_enqueue_time.isoformat() if - msg.scheduled_enqueue_time else None), - 'session_id': msg.session_id, - 'time_to_live': msg.time_to_live, - 'to': msg.to, - 'user_properties': msg.user_properties, - - 'application_properties': msg.application_properties, - 'correlation_id': msg.correlation_id, - 'dead_letter_error_description': msg.dead_letter_error_description, - 'dead_letter_reason': msg.dead_letter_reason, - 'dead_letter_source': msg.dead_letter_source, - 'enqueued_sequence_number': msg.enqueued_sequence_number, - 'enqueued_time_utc': (msg.enqueued_time_utc.isoformat() if - msg.enqueued_time_utc else None), - 'expires_at_utc': (msg.expires_at_utc.isoformat() if - msg.expires_at_utc else None), - 'locked_until': (msg.locked_until.isoformat() if - msg.locked_until else None), - 'lock_token': msg.lock_token, - 'sequence_number': msg.sequence_number, - 'state': msg.state, - 'subject': msg.subject, - 'transaction_partition_key': msg.transaction_partition_key - }) - - return result +import json + +import azure.functions as func + +app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) + + +@app.route(route="put_message") +@app.service_bus_queue_output( + arg_name="msg", + connection="AzureWebJobsServiceBusConnectionString", + queue_name="testqueue") +def put_message(req: func.HttpRequest, msg: func.Out[str]): + msg.set(req.get_body().decode('utf-8')) + return 'OK' + + +@app.route(route="get_servicebus_triggered") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-servicebus-triggered.txt", + connection="AzureWebJobsStorage") +def get_servicebus_triggered(req: func.HttpRequest, + file: func.InputStream) -> str: + return func.HttpResponse( + file.read().decode('utf-8'), mimetype='application/json') + + +@app.service_bus_queue_trigger( + arg_name="msg", + connection="AzureWebJobsServiceBusConnectionString", + queue_name="testqueue") +@app.blob_output(arg_name="$return", + path="python-worker-tests/test-servicebus-triggered.txt", + connection="AzureWebJobsStorage") +def servicebus_trigger(msg: func.ServiceBusMessage) -> str: + result = json.dumps({ + 'message_id': msg.message_id, + 'body': msg.get_body().decode('utf-8'), + 'content_type': msg.content_type, + 'delivery_count': msg.delivery_count, + 'expiration_time': (msg.expiration_time.isoformat() if + msg.expiration_time else None), + 'label': msg.label, + 'partition_key': msg.partition_key, + 'reply_to': msg.reply_to, + 'reply_to_session_id': msg.reply_to_session_id, + 'scheduled_enqueue_time': (msg.scheduled_enqueue_time.isoformat() if + msg.scheduled_enqueue_time else None), + 'session_id': msg.session_id, + 'time_to_live': msg.time_to_live, + 'to': msg.to, + 'user_properties': msg.user_properties, + + 'application_properties': msg.application_properties, + 'correlation_id': msg.correlation_id, + 'dead_letter_error_description': msg.dead_letter_error_description, + 'dead_letter_reason': msg.dead_letter_reason, + 'dead_letter_source': msg.dead_letter_source, + 'enqueued_sequence_number': msg.enqueued_sequence_number, + 'enqueued_time_utc': (msg.enqueued_time_utc.isoformat() if + msg.enqueued_time_utc else None), + 'expires_at_utc': (msg.expires_at_utc.isoformat() if + msg.expires_at_utc else None), + 'locked_until': (msg.locked_until.isoformat() if + msg.locked_until else None), + 'lock_token': msg.lock_token, + 'sequence_number': msg.sequence_number, + 'state': msg.state, + 'subject': msg.subject, + 'transaction_partition_key': msg.transaction_partition_key + }) + + return result diff --git a/tests/endtoend/servicebus_functions/servicebus_functions_stein/generic/function_app.py b/tests/emulator_tests/servicebus_functions/servicebus_functions_stein/generic/function_app.py similarity index 97% rename from tests/endtoend/servicebus_functions/servicebus_functions_stein/generic/function_app.py rename to tests/emulator_tests/servicebus_functions/servicebus_functions_stein/generic/function_app.py index 3b340153..4fd48785 100644 --- a/tests/endtoend/servicebus_functions/servicebus_functions_stein/generic/function_app.py +++ b/tests/emulator_tests/servicebus_functions/servicebus_functions_stein/generic/function_app.py @@ -1,81 +1,81 @@ -import json - -import azure.functions as func - -app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) - - -@app.function_name(name="put_message") -@app.generic_trigger(arg_name="req", type="httpTrigger", route="put_message") -@app.generic_output_binding(arg_name="msg", - type="serviceBus", - connection="AzureWebJobsServiceBusConnectionString", - queue_name="testqueue") -@app.generic_output_binding(arg_name="$return", type="http") -def put_message(req: func.HttpRequest, msg: func.Out[str]): - msg.set(req.get_body().decode('utf-8')) - return 'OK' - - -@app.function_name(name="get_servicebus_triggered") -@app.generic_trigger(arg_name="req", type="httpTrigger", - route="get_servicebus_triggered") -@app.generic_input_binding(arg_name="file", - type="blob", - path="python-worker-tests/test-servicebus-triggered.txt", # NoQA - connection="AzureWebJobsStorage") -@app.generic_output_binding(arg_name="$return", type="http") -def get_servicebus_triggered(req: func.HttpRequest, - file: func.InputStream) -> str: - return func.HttpResponse( - file.read().decode('utf-8'), mimetype='application/json') - - -@app.generic_trigger( - arg_name="msg", - type="serviceBusTrigger", - connection="AzureWebJobsServiceBusConnectionString", - queue_name="testqueue") -@app.generic_output_binding(arg_name="$return", - path="python-worker-tests/test-servicebus-triggered.txt", # NoQA - type="blob", - connection="AzureWebJobsStorage") -def servicebus_trigger(msg: func.ServiceBusMessage) -> str: - result = json.dumps({ - 'message_id': msg.message_id, - 'body': msg.get_body().decode('utf-8'), - 'content_type': msg.content_type, - 'delivery_count': msg.delivery_count, - 'expiration_time': (msg.expiration_time.isoformat() if - msg.expiration_time else None), - 'label': msg.label, - 'partition_key': msg.partition_key, - 'reply_to': msg.reply_to, - 'reply_to_session_id': msg.reply_to_session_id, - 'scheduled_enqueue_time': (msg.scheduled_enqueue_time.isoformat() if - msg.scheduled_enqueue_time else None), - 'session_id': msg.session_id, - 'time_to_live': msg.time_to_live, - 'to': msg.to, - 'user_properties': msg.user_properties, - - 'application_properties': msg.application_properties, - 'correlation_id': msg.correlation_id, - 'dead_letter_error_description': msg.dead_letter_error_description, - 'dead_letter_reason': msg.dead_letter_reason, - 'dead_letter_source': msg.dead_letter_source, - 'enqueued_sequence_number': msg.enqueued_sequence_number, - 'enqueued_time_utc': (msg.enqueued_time_utc.isoformat() if - msg.enqueued_time_utc else None), - 'expires_at_utc': (msg.expires_at_utc.isoformat() if - msg.expires_at_utc else None), - 'locked_until': (msg.locked_until.isoformat() if - msg.locked_until else None), - 'lock_token': msg.lock_token, - 'sequence_number': msg.sequence_number, - 'state': msg.state, - 'subject': msg.subject, - 'transaction_partition_key': msg.transaction_partition_key - }) - - return result +import json + +import azure.functions as func + +app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) + + +@app.function_name(name="put_message") +@app.generic_trigger(arg_name="req", type="httpTrigger", route="put_message") +@app.generic_output_binding(arg_name="msg", + type="serviceBus", + connection="AzureWebJobsServiceBusConnectionString", + queue_name="testqueue") +@app.generic_output_binding(arg_name="$return", type="http") +def put_message(req: func.HttpRequest, msg: func.Out[str]): + msg.set(req.get_body().decode('utf-8')) + return 'OK' + + +@app.function_name(name="get_servicebus_triggered") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="get_servicebus_triggered") +@app.generic_input_binding(arg_name="file", + type="blob", + path="python-worker-tests/test-servicebus-triggered.txt", # NoQA + connection="AzureWebJobsStorage") +@app.generic_output_binding(arg_name="$return", type="http") +def get_servicebus_triggered(req: func.HttpRequest, + file: func.InputStream) -> str: + return func.HttpResponse( + file.read().decode('utf-8'), mimetype='application/json') + + +@app.generic_trigger( + arg_name="msg", + type="serviceBusTrigger", + connection="AzureWebJobsServiceBusConnectionString", + queue_name="testqueue") +@app.generic_output_binding(arg_name="$return", + path="python-worker-tests/test-servicebus-triggered.txt", # NoQA + type="blob", + connection="AzureWebJobsStorage") +def servicebus_trigger(msg: func.ServiceBusMessage) -> str: + result = json.dumps({ + 'message_id': msg.message_id, + 'body': msg.get_body().decode('utf-8'), + 'content_type': msg.content_type, + 'delivery_count': msg.delivery_count, + 'expiration_time': (msg.expiration_time.isoformat() if + msg.expiration_time else None), + 'label': msg.label, + 'partition_key': msg.partition_key, + 'reply_to': msg.reply_to, + 'reply_to_session_id': msg.reply_to_session_id, + 'scheduled_enqueue_time': (msg.scheduled_enqueue_time.isoformat() if + msg.scheduled_enqueue_time else None), + 'session_id': msg.session_id, + 'time_to_live': msg.time_to_live, + 'to': msg.to, + 'user_properties': msg.user_properties, + + 'application_properties': msg.application_properties, + 'correlation_id': msg.correlation_id, + 'dead_letter_error_description': msg.dead_letter_error_description, + 'dead_letter_reason': msg.dead_letter_reason, + 'dead_letter_source': msg.dead_letter_source, + 'enqueued_sequence_number': msg.enqueued_sequence_number, + 'enqueued_time_utc': (msg.enqueued_time_utc.isoformat() if + msg.enqueued_time_utc else None), + 'expires_at_utc': (msg.expires_at_utc.isoformat() if + msg.expires_at_utc else None), + 'locked_until': (msg.locked_until.isoformat() if + msg.locked_until else None), + 'lock_token': msg.lock_token, + 'sequence_number': msg.sequence_number, + 'state': msg.state, + 'subject': msg.subject, + 'transaction_partition_key': msg.transaction_partition_key + }) + + return result diff --git a/tests/endtoend/servicebus_functions/servicebus_trigger/__init__.py b/tests/emulator_tests/servicebus_functions/servicebus_trigger/__init__.py similarity index 100% rename from tests/endtoend/servicebus_functions/servicebus_trigger/__init__.py rename to tests/emulator_tests/servicebus_functions/servicebus_trigger/__init__.py diff --git a/tests/endtoend/servicebus_functions/servicebus_trigger/function.json b/tests/emulator_tests/servicebus_functions/servicebus_trigger/function.json similarity index 100% rename from tests/endtoend/servicebus_functions/servicebus_trigger/function.json rename to tests/emulator_tests/servicebus_functions/servicebus_trigger/function.json diff --git a/tests/endtoend/table_functions/table_functions_stein/function_app.py b/tests/emulator_tests/table_functions/table_functions_stein/function_app.py similarity index 100% rename from tests/endtoend/table_functions/table_functions_stein/function_app.py rename to tests/emulator_tests/table_functions/table_functions_stein/function_app.py diff --git a/tests/endtoend/table_functions/table_functions_stein/generic/function_app.py b/tests/emulator_tests/table_functions/table_functions_stein/generic/function_app.py similarity index 100% rename from tests/endtoend/table_functions/table_functions_stein/generic/function_app.py rename to tests/emulator_tests/table_functions/table_functions_stein/generic/function_app.py diff --git a/tests/endtoend/table_functions/table_in_binding/__init__.py b/tests/emulator_tests/table_functions/table_in_binding/__init__.py similarity index 100% rename from tests/endtoend/table_functions/table_in_binding/__init__.py rename to tests/emulator_tests/table_functions/table_in_binding/__init__.py diff --git a/tests/endtoend/table_functions/table_in_binding/function.json b/tests/emulator_tests/table_functions/table_in_binding/function.json similarity index 100% rename from tests/endtoend/table_functions/table_in_binding/function.json rename to tests/emulator_tests/table_functions/table_in_binding/function.json diff --git a/tests/endtoend/table_functions/table_out_binding/__init__.py b/tests/emulator_tests/table_functions/table_out_binding/__init__.py similarity index 100% rename from tests/endtoend/table_functions/table_out_binding/__init__.py rename to tests/emulator_tests/table_functions/table_out_binding/__init__.py diff --git a/tests/endtoend/table_functions/table_out_binding/function.json b/tests/emulator_tests/table_functions/table_out_binding/function.json similarity index 100% rename from tests/endtoend/table_functions/table_out_binding/function.json rename to tests/emulator_tests/table_functions/table_out_binding/function.json diff --git a/tests/endtoend/test_blob_functions.py b/tests/emulator_tests/test_blob_functions.py similarity index 95% rename from tests/endtoend/test_blob_functions.py rename to tests/emulator_tests/test_blob_functions.py index 4d51693e..d6a840a3 100644 --- a/tests/endtoend/test_blob_functions.py +++ b/tests/emulator_tests/test_blob_functions.py @@ -10,7 +10,7 @@ class TestBlobFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'blob_functions' + return testutils.EMULATOR_TESTS_FOLDER / 'blob_functions' @testutils.retryable_test(3, 5) def test_blob_io_str(self): @@ -154,13 +154,13 @@ class TestBlobFunctionsStein(TestBlobFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'blob_functions' / \ - 'blob_functions_stein' + return testutils.EMULATOR_TESTS_FOLDER / 'blob_functions' / \ + 'blob_functions_stein' class TestBlobFunctionsSteinGeneric(TestBlobFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'blob_functions' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'blob_functions' / \ 'blob_functions_stein' / 'generic' diff --git a/tests/endtoend/test_eventhub_batch_functions.py b/tests/emulator_tests/test_eventhub_batch_functions.py similarity index 94% rename from tests/endtoend/test_eventhub_batch_functions.py rename to tests/emulator_tests/test_eventhub_batch_functions.py index 11cf4d38..1a8ae2a9 100644 --- a/tests/endtoend/test_eventhub_batch_functions.py +++ b/tests/emulator_tests/test_eventhub_batch_functions.py @@ -1,8 +1,10 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import json +import sys import time from datetime import datetime +from unittest.case import skipIf from dateutil import parser from tests.utils import testutils @@ -19,7 +21,7 @@ class TestEventHubFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'eventhub_batch_functions' + return testutils.EMULATOR_TESTS_FOLDER / 'eventhub_batch_functions' @classmethod def get_libraries_to_install(cls): @@ -64,6 +66,9 @@ def test_eventhub_multiple(self): self.assertDictEqual(all_row_keys_seen, row_keys_seen) + @skipIf(sys.version_info.minor == 7, + "Using azure-eventhub SDK with the EventHub Emulator" + "requires Python 3.8+") @testutils.retryable_test(3, 5) def test_eventhub_multiple_with_metadata(self): # Generate a unique event body for EventHub event @@ -130,7 +135,7 @@ class TestEventHubBatchFunctionsStein(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'eventhub_batch_functions' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'eventhub_batch_functions' / \ 'eventhub_batch_functions_stein' @classmethod @@ -171,6 +176,9 @@ def test_eventhub_multiple(self): self.assertDictEqual(all_row_keys_seen, row_keys_seen) + @skipIf(sys.version_info.minor == 7, + "Using azure-eventhub SDK with the EventHub Emulator" + "requires Python 3.8+") @testutils.retryable_test(3, 5) def test_eventhub_multiple_with_metadata(self): # Generate a unique event body for EventHub event diff --git a/tests/endtoend/test_eventhub_functions.py b/tests/emulator_tests/test_eventhub_functions.py similarity index 90% rename from tests/endtoend/test_eventhub_functions.py rename to tests/emulator_tests/test_eventhub_functions.py index c4c3d74a..03088c73 100644 --- a/tests/endtoend/test_eventhub_functions.py +++ b/tests/emulator_tests/test_eventhub_functions.py @@ -1,8 +1,11 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import json +import sys import time +from unittest import skipIf + from tests.utils import testutils @@ -17,7 +20,7 @@ class TestEventHubFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'eventhub_functions' + return testutils.EMULATOR_TESTS_FOLDER / 'eventhub_functions' @classmethod def get_libraries_to_install(cls): @@ -52,6 +55,9 @@ def test_eventhub_trigger(self): # Check if the event body matches the initial data self.assertEqual(response, doc) + @skipIf(sys.version_info.minor == 7, + "Using azure-eventhub SDK with the EventHub Emulator" + "requires Python 3.8+") @testutils.retryable_test(3, 5) def test_eventhub_trigger_with_metadata(self): # Generate a unique event body for EventHub event @@ -106,13 +112,13 @@ class TestEventHubFunctionsStein(TestEventHubFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'eventhub_functions' / \ - 'eventhub_functions_stein' + return testutils.EMULATOR_TESTS_FOLDER / 'eventhub_functions' / \ + 'eventhub_functions_stein' class TestEventHubFunctionsSteinGeneric(TestEventHubFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'eventhub_functions' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'eventhub_functions' / \ 'eventhub_functions_stein' / 'generic' diff --git a/tests/endtoend/test_generic_functions.py b/tests/emulator_tests/test_generic_functions.py similarity index 83% rename from tests/endtoend/test_generic_functions.py rename to tests/emulator_tests/test_generic_functions.py index 361b67f7..8dc44c83 100644 --- a/tests/endtoend/test_generic_functions.py +++ b/tests/emulator_tests/test_generic_functions.py @@ -17,12 +17,14 @@ class TestGenericFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'generic_functions' + return testutils.EMULATOR_TESTS_FOLDER / 'generic_functions' def test_return_processed_last(self): # Tests the case where implicit and explicit return are true # in the same function and $return is processed before # the generic binding is + out_resp = self.webhost.request('POST', 'table_out_binding') + self.assertEqual(out_resp.status_code, 200) r = self.webhost.request('GET', 'return_processed_last') self.assertEqual(r.status_code, 200) @@ -31,11 +33,15 @@ def test_return_not_processed_last(self): # Tests the case where implicit and explicit return are true # in the same function and the generic binding is processed # before $return + out_resp = self.webhost.request('POST', 'table_out_binding') + self.assertEqual(out_resp.status_code, 200) r = self.webhost.request('GET', 'return_not_processed_last') self.assertEqual(r.status_code, 200) def test_return_types(self): + out_resp = self.webhost.request('POST', 'table_out_binding') + self.assertEqual(out_resp.status_code, 200) # Checking that the function app is okay time.sleep(10) # Checking webhost status. @@ -68,5 +74,5 @@ class TestGenericFunctionsStein(TestGenericFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'generic_functions' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'generic_functions' / \ 'generic_functions_stein' diff --git a/tests/endtoend/test_queue_functions.py b/tests/emulator_tests/test_queue_functions.py similarity index 92% rename from tests/endtoend/test_queue_functions.py rename to tests/emulator_tests/test_queue_functions.py index 015e53ae..79362816 100644 --- a/tests/endtoend/test_queue_functions.py +++ b/tests/emulator_tests/test_queue_functions.py @@ -9,7 +9,7 @@ class TestQueueFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'queue_functions' + return testutils.EMULATOR_TESTS_FOLDER / 'queue_functions' def test_queue_basic(self): r = self.webhost.request('POST', 'put_queue', @@ -91,13 +91,13 @@ class TestQueueFunctionsStein(TestQueueFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'queue_functions' / \ - 'queue_functions_stein' + return testutils.EMULATOR_TESTS_FOLDER / 'queue_functions' / \ + 'queue_functions_stein' class TestQueueFunctionsSteinGeneric(TestQueueFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'queue_functions' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'queue_functions' / \ 'queue_functions_stein' / 'generic' diff --git a/tests/endtoend/test_servicebus_functions.py b/tests/emulator_tests/test_servicebus_functions.py similarity index 84% rename from tests/endtoend/test_servicebus_functions.py rename to tests/emulator_tests/test_servicebus_functions.py index aaacd76d..2e6bd731 100644 --- a/tests/endtoend/test_servicebus_functions.py +++ b/tests/emulator_tests/test_servicebus_functions.py @@ -10,7 +10,7 @@ class TestServiceBusFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'servicebus_functions' + return testutils.EMULATOR_TESTS_FOLDER / 'servicebus_functions' @testutils.retryable_test(3, 5) def test_servicebus_basic(self): @@ -53,14 +53,13 @@ class TestServiceBusFunctionsStein(TestServiceBusFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'servicebus_functions' / \ - 'servicebus_functions_stein' + return testutils.EMULATOR_TESTS_FOLDER / 'servicebus_functions' / \ + 'servicebus_functions_stein' class TestServiceBusFunctionsSteinGeneric(TestServiceBusFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'servicebus_functions' / \ - 'servicebus_functions_stein' / \ - 'generic' + return testutils.EMULATOR_TESTS_FOLDER / 'servicebus_functions' / \ + 'servicebus_functions_stein' / 'generic' diff --git a/tests/endtoend/test_table_functions.py b/tests/emulator_tests/test_table_functions.py similarity index 84% rename from tests/endtoend/test_table_functions.py rename to tests/emulator_tests/test_table_functions.py index d6e367bd..b5282bd9 100644 --- a/tests/endtoend/test_table_functions.py +++ b/tests/emulator_tests/test_table_functions.py @@ -11,7 +11,7 @@ class TestTableFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'table_functions' + return testutils.EMULATOR_TESTS_FOLDER / 'table_functions' def test_table_bindings(self): out_resp = self.webhost.request('POST', 'table_out_binding') @@ -46,8 +46,8 @@ class TestTableFunctionsStein(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'table_functions' / \ - 'table_functions_stein' + return testutils.EMULATOR_TESTS_FOLDER / 'table_functions' / \ + 'table_functions_stein' def test_table_bindings(self): out_resp = self.webhost.request('POST', 'table_out_binding') @@ -68,6 +68,6 @@ class TestTableFunctionsGeneric(TestTableFunctionsStein): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'table_functions' / \ - 'table_functions_stein' /\ - 'generic' + return testutils.EMULATOR_TESTS_FOLDER / 'table_functions' / \ + 'table_functions_stein' / \ + 'generic' diff --git a/tests/emulator_tests/utils/eventhub/config.json b/tests/emulator_tests/utils/eventhub/config.json new file mode 100644 index 00000000..710935c1 --- /dev/null +++ b/tests/emulator_tests/utils/eventhub/config.json @@ -0,0 +1,51 @@ +{ + "UserConfig": { + "NamespaceConfig": [ + { + "Type": "EventHub", + "Name": "emulatorNs1", + "Entities": [ + { + "Name": "python-worker-ci-eventhub-batch", + "PartitionCount": 2, + "ConsumerGroups": [ + { + "Name": "cg1" + } + ] + }, + { + "Name": "python-worker-ci-eventhub-batch-metadata", + "PartitionCount": 2, + "ConsumerGroups": [ + { + "Name": "cg1" + } + ] + }, + { + "Name": "python-worker-ci-eventhub-one", + "PartitionCount": 2, + "ConsumerGroups": [ + { + "Name": "cg1" + } + ] + }, + { + "Name": "python-worker-ci-eventhub-one-metadata", + "PartitionCount": 2, + "ConsumerGroups": [ + { + "Name": "cg1" + } + ] + } + ] + } + ], + "LoggingConfig": { + "Type": "File" + } + } +} \ No newline at end of file diff --git a/tests/emulator_tests/utils/eventhub/docker-compose.yml b/tests/emulator_tests/utils/eventhub/docker-compose.yml new file mode 100644 index 00000000..2c40aa04 --- /dev/null +++ b/tests/emulator_tests/utils/eventhub/docker-compose.yml @@ -0,0 +1,34 @@ +name: microsoft-azure-eventhubs +services: + # Service for the Event Hubs Emulator + emulator: + container_name: "eventhubs-emulator" + image: "mcr.microsoft.com/azure-messaging/eventhubs-emulator:latest" + volumes: + - "./config.json:/Eventhubs_Emulator/ConfigFiles/Config.json" + ports: + - "5672:5672" + environment: + BLOB_SERVER: azurite + METADATA_SERVER: azurite + ACCEPT_EULA: Y + depends_on: + - azurite + networks: + eh-emulator: + aliases: + - "eventhubs-emulator" + # Service for the Azurite Storage Emulator + azurite: + container_name: "azurite" + image: "mcr.microsoft.com/azure-storage/azurite:latest" + ports: + - "10000:10000" + - "10001:10001" + - "10002:10002" + networks: + eh-emulator: + aliases: + - "azurite" +networks: + eh-emulator: \ No newline at end of file diff --git a/tests/emulator_tests/utils/servicebus/config.json b/tests/emulator_tests/utils/servicebus/config.json new file mode 100644 index 00000000..20cf8344 --- /dev/null +++ b/tests/emulator_tests/utils/servicebus/config.json @@ -0,0 +1,28 @@ +{ + "UserConfig": { + "Namespaces": [ + { + "Name": "sbemulatorns", + "Queues": [ + { + "Name": "testqueue", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + } + ] + } + ], + "Logging": { + "Type": "File" + } + } +} \ No newline at end of file diff --git a/tests/emulator_tests/utils/servicebus/docker-compose.yml b/tests/emulator_tests/utils/servicebus/docker-compose.yml new file mode 100644 index 00000000..c1781a85 --- /dev/null +++ b/tests/emulator_tests/utils/servicebus/docker-compose.yml @@ -0,0 +1,40 @@ +name: microsoft-azure-servicebus +services: + # Service for the Service Bus Emulator + sbemulator: + container_name: "servicebus-emulator" + image: mcr.microsoft.com/azure-messaging/servicebus-emulator:latest + volumes: + - "./config.json:/ServiceBus_Emulator/ConfigFiles/Config.json" + ports: + - "5672:5672" + environment: + SQL_SERVER: sqledge + MSSQL_SA_PASSWORD: ${AzureWebJobsSQLPassword} + ACCEPT_EULA: Y + depends_on: + - sqledge + networks: + sb-emulator: + aliases: + - "sb-emulator" + sqledge: + container_name: "sqledge" + image: "mcr.microsoft.com/azure-sql-edge:latest" + networks: + sb-emulator: + aliases: + - "sqledge" + environment: + ACCEPT_EULA: Y + MSSQL_SA_PASSWORD: ${AzureWebJobsSQLPassword} + # Service for the Azurite Storage Emulator + azurite: + container_name: "azurite-sb" + image: "mcr.microsoft.com/azure-storage/azurite:latest" + ports: + - "10003:10003" + - "10004:10004" + - "10005:10005" +networks: + sb-emulator: \ No newline at end of file diff --git a/tests/endtoend/test_worker_process_count_functions.py b/tests/endtoend/test_worker_process_count_functions.py index 8ee6577b..44abcd2e 100644 --- a/tests/endtoend/test_worker_process_count_functions.py +++ b/tests/endtoend/test_worker_process_count_functions.py @@ -1,12 +1,19 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import os + from datetime import datetime from threading import Thread +from unittest import skipIf from tests.utils import testutils +from azure_functions_worker.utils.common import is_envvar_true +from tests.utils.constants import CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST +@skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCount(testutils.WebHostTestCase): """Test the Http Trigger with setting up the python worker process count to 2. this test will check if both requests should be processed at the @@ -63,24 +70,30 @@ def http_req(res_num): self.assertTrue(time_diff_in_seconds < 1) +@skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCountStein(TestWorkerProcessCount): - @classmethod def get_script_dir(cls): return testutils.E2E_TESTS_FOLDER / 'http_functions' /\ 'http_functions_stein' +@skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCountWithBlueprintStein(TestWorkerProcessCount): - @classmethod def get_script_dir(cls): return testutils.E2E_TESTS_FOLDER / 'blueprint_functions' /\ 'functions_in_blueprint_only' +@skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCountWithBlueprintDiffDirStein(TestWorkerProcessCount): - @classmethod def get_script_dir(cls): return testutils.E2E_TESTS_FOLDER / 'blueprint_functions' /\ diff --git a/tests/extension_tests/http_v2_tests/test_http_v2.py b/tests/extension_tests/http_v2_tests/test_http_v2.py index 8409b7e5..8c1d5b48 100644 --- a/tests/extension_tests/http_v2_tests/test_http_v2.py +++ b/tests/extension_tests/http_v2_tests/test_http_v2.py @@ -8,12 +8,17 @@ import requests from tests.utils import testutils +from azure_functions_worker.utils.common import is_envvar_true +from tests.utils.constants import CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST from azure_functions_worker.constants import PYTHON_ENABLE_INIT_INDEXING REQUEST_TIMEOUT_SEC = 5 +@unittest.skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") @unittest.skipIf(sys.version_info.minor < 8, "HTTPv2" "is only supported for 3.8+.") class TestHttpFunctionsWithInitIndexing(testutils.WebHostTestCase): diff --git a/tests/unittests/test_http_functions_v2.py b/tests/unittests/test_http_functions_v2.py index 6aebad6a..b7e45667 100644 --- a/tests/unittests/test_http_functions_v2.py +++ b/tests/unittests/test_http_functions_v2.py @@ -204,6 +204,7 @@ def test_accept_json(self): self.assertEqual(r_json, {'a': 'abc', 'd': 42}) self.assertEqual(r.headers['content-type'], 'application/json') + @testutils.retryable_test(3, 5) def test_unhandled_error(self): r = self.webhost.request('GET', 'unhandled_error') self.assertEqual(r.status_code, 500) @@ -331,6 +332,7 @@ def check_log_import_module_troubleshooting_url(self, passed = True self.assertTrue(passed) + @testutils.retryable_test(3, 5) def test_print_logging_no_flush(self): r = self.webhost.request('GET', 'print_logging?message=Secret42') self.assertEqual(r.status_code, 200) diff --git a/tests/unittests/test_mock_blob_shared_memory_functions.py b/tests/unittests/test_mock_blob_shared_memory_functions.py index f1154542..63b06ca1 100644 --- a/tests/unittests/test_mock_blob_shared_memory_functions.py +++ b/tests/unittests/test_mock_blob_shared_memory_functions.py @@ -27,7 +27,7 @@ class TestMockBlobSharedMemoryFunctions(testutils.SharedMemoryTestCase, """ def setUp(self): super().setUp() - self.blob_funcs_dir = testutils.E2E_TESTS_FOLDER / 'blob_functions' + self.blob_funcs_dir = testutils.EMULATOR_TESTS_FOLDER / 'blob_functions' async def test_binary_blob_read_as_bytes_function(self): """ diff --git a/tests/utils/testutils.py b/tests/utils/testutils.py index aaeb40de..c04b134c 100644 --- a/tests/utils/testutils.py +++ b/tests/utils/testutils.py @@ -68,6 +68,7 @@ E2E_TESTS_ROOT = TESTS_ROOT / E2E_TESTS_FOLDER UNIT_TESTS_FOLDER = pathlib.Path('unittests') UNIT_TESTS_ROOT = TESTS_ROOT / UNIT_TESTS_FOLDER +EMULATOR_TESTS_FOLDER = pathlib.Path('emulator_tests') EXTENSION_TESTS_FOLDER = pathlib.Path('extension_tests') WEBHOST_DLL = "Microsoft.Azure.WebJobs.Script.WebHost.dll" DEFAULT_WEBHOST_DLL_PATH = (