HTTP/2 support

demo_server/demo_server/app.py

@@ -1,7 +1,11 @@
 from fastapi import FastAPI, HTTPException, Depends
 
-import uvicorn
+from hypercorn.config import Config
+from hypercorn.asyncio import serve
+import asyncio
 import os, binascii
+import uvicorn
+import sys
 
 from sqlmodel import create_engine, SQLModel, Session
 
@@ -22,6 +26,7 @@ def on_startup():
     SQLModel.metadata.create_all(engine)
 
 
+
 if __name__ == "__main__":
 
     app_port = os.getenv('DEMO_SERVER_PORT')
@@ -34,5 +39,17 @@ def on_startup():
     if app_host is None:
         app_host = "0.0.0.0"
 
-    uvicorn.run("app:app", reload=True, host=app_host, port=app_port)
+    use_http2 = False
+    for i in range(len(sys.argv)):
+        if sys.argv[i] == '--use_http2':
+            use_http2 = True
+
+    if not use_http2:
+        uvicorn.run("app:app", reload=True, host=app_host, port=app_port)
+    else:
+        config = Config()
+        config.bind = [app_host + ":" + str(app_port)]
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(serve(app, config))
+
 

demo_server/requirements.txt

@@ -1,3 +1,4 @@
 fastapi==0.78.0
+hypercorn==0.14.3
 uvicorn==0.18.1
 sqlmodel==0.0.6

docker/Dockerfile

@@ -11,6 +11,7 @@ RUN python3 -m ensurepip
 RUN pip3 install --upgrade pip
 RUN pip3 install requests
 RUN pip3 install applicationinsights
+RUN pip3 install h2
 COPY ./engine /RESTler/engine
 RUN python3 -m compileall -b /RESTler/engine
 COPY ./resultsAnalyzer /RESTler/resultsAnalyzer

restler-quick-start.py

@@ -42,9 +42,11 @@ def compile_spec(api_spec_path, restler_dll_path):
         print(f"command: {command}")
         subprocess.run(command, shell=True)
 
-def add_common_settings(ip, port, host, use_ssl, command):
+def add_common_settings(ip, port, host, use_ssl, use_http2, command):
     if not use_ssl:
         command = f"{command} --no_ssl"
+    if use_http2:
+        command = f"{command} --http2"
     if ip is not None:
         command = f"{command} --target_ip {ip}"
     if port is not None:
@@ -53,18 +55,18 @@ def add_common_settings(ip, port, host, use_ssl, command):
         command = f"{command} --host {host}"
     return command
 
-def replay_bug(ip, port, host, use_ssl, restler_dll_path, replay_log):
+def replay_bug(ip, port, host, use_ssl, use_http2, restler_dll_path, replay_log):
     """ Runs RESTler's replay mode on the specified replay file
     """
     with usedir(RESTLER_TEMP_DIR):
         command = (
             f"dotnet \"{restler_dll_path}\" replay --replay_log \"{replay_log}\""
         )
-        command = add_common_settings(ip, port, host, use_ssl, command)
+        command = add_common_settings(ip, port, host, use_ssl, use_http2, command)
         print(f"command: {command}\n")
         subprocess.run(command, shell=True)
 
-def replay_from_dir(ip, port, host, use_ssl, restler_dll_path, replay_dir):
+def replay_from_dir(ip, port, host, use_ssl, use_http2, restler_dll_path, replay_dir):
     import glob
     from pathlib import Path
     # get all the 500 replay files in the bug buckets directory
@@ -74,10 +76,10 @@ def replay_from_dir(ip, port, host, use_ssl, restler_dll_path, replay_dir):
         if "bug_buckets" in os.path.basename(file_path):
             continue
         print(f"Testing replay file: {file_path}")
-        replay_bug(ip, port, host, use_ssl, restler_dll_path, Path(file_path).absolute())
+        replay_bug(ip, port, host, use_ssl, use_http2, restler_dll_path, Path(file_path).absolute())
     pass
 
-def test_spec(ip, port, host, use_ssl, restler_dll_path, task):
+def test_spec(ip, port, host, use_ssl, use_http2, restler_dll_path, task):
     """ Runs RESTler's test mode on a specified Compile directory
 
     @param ip: The IP of the service to test
@@ -88,6 +90,8 @@ def test_spec(ip, port, host, use_ssl, restler_dll_path, task):
     @type  host: Str
     @param use_ssl: If False, set the --no_ssl parameter when executing RESTler
     @type  use_ssl: Boolean
+    @param use_http2: If True, set the --http2 parameter when executing RESTler
+    @type  use_http2: Boolean
     @param restler_dll_path: The absolute path to the RESTler driver's dll
     @type  restler_dll_path: Str
 
@@ -107,8 +111,8 @@ def test_spec(ip, port, host, use_ssl, restler_dll_path, task):
             f" --settings \"{settings_file_path}\""
         )
         print(f"command: {command}\n")
-        command = add_common_settings(ip, port, host, use_ssl, command)
-
+        command = add_common_settings(ip, port, host, use_ssl, use_http2, command)
+        print(f"command: {command}\n")
         subprocess.run(command, shell=True)
 
 if __name__ == '__main__':
@@ -129,6 +133,9 @@ def test_spec(ip, port, host, use_ssl, restler_dll_path, task):
     parser.add_argument('--use_ssl',
                         help='Set this flag if you want to use SSL validation for the socket',
                         action='store_true')
+    parser.add_argument('--use_http2',
+                        help='Set this flag if you want to use HTTP2',
+                        action='store_true')
     parser.add_argument('--host',
                         help='The hostname of the service to test',
                         type=str, required=False, default=None)
@@ -146,13 +153,13 @@ def test_spec(ip, port, host, use_ssl, restler_dll_path, task):
     print(f"\nrestler_dll_path: {restler_dll_path}\n")
 
     if args.task == "replay":
-        replay_from_dir(args.ip, args.port, args.host, args.use_ssl, restler_dll_path.absolute(), args.replay_bug_buckets_dir)
+        replay_from_dir(args.ip, args.port, args.host, args.use_ssl, args.use_http2, restler_dll_path.absolute(), args.replay_bug_buckets_dir)
     else:
         if args.api_spec_path is None:
             print("api_spec_path is required for all tasks except the replay task.")
             exit(-1)
         api_spec_path = os.path.abspath(args.api_spec_path)
         compile_spec(api_spec_path, restler_dll_path.absolute())
-        test_spec(args.ip, args.port, args.host, args.use_ssl, restler_dll_path.absolute(), args.task)
+        test_spec(args.ip, args.port, args.host, args.use_ssl, args.use_http2, restler_dll_path.absolute(), args.task)
 
     print(f"Test complete.\nSee {os.path.abspath(RESTLER_TEMP_DIR)} for results.")

restler/end_to_end_tests/test_quick_start.py

@@ -7,6 +7,7 @@
 created during quick start test.
 
 To call: python ./test_quick_start.py <path_to_restler_drop_directory>
+To call with HTTP/2: python ./test_quick_start.py <path_to_restler_drop_directory> --use_http2
 """
 import sys
 import os
@@ -51,10 +52,10 @@ def check_expected_output(restler_working_dir, expected_strings, output, task_di
                 net_log = nf.read()
             raise QuickStartFailedException(f"Failing because expected output '{expected_str}' was not found:\n{stdout}{out}{err}{net_log}")
 
-def test_test_task(restler_working_dir, swagger_path, restler_drop_dir):
+def test_test_task(restler_working_dir, swagger_path, restler_drop_dir, use_http2):
     # Run the quick start script
     output = subprocess.run(
-        f'python ./restler-quick-start.py --api_spec_path {swagger_path} --restler_drop_dir {restler_drop_dir} --task test',
+        f'python ./restler-quick-start.py --api_spec_path {swagger_path} --restler_drop_dir {restler_drop_dir} --task test {use_http2}',
         shell=True, capture_output=True
     )
     expected_strings = [
@@ -66,10 +67,10 @@ def test_test_task(restler_working_dir, swagger_path, restler_drop_dir):
     check_output_errors(output)
     check_expected_output(restler_working_dir, expected_strings, output, "Test")
 
-def test_fuzzlean_task(restler_working_dir, swagger_path, restler_drop_dir):
+def test_fuzzlean_task(restler_working_dir, swagger_path, restler_drop_dir, use_http2):
     # Run the quick start script
     output = subprocess.run(
-        f'python ./restler-quick-start.py --api_spec_path {swagger_path} --restler_drop_dir {restler_drop_dir} --task fuzz-lean',
+        f'python ./restler-quick-start.py --api_spec_path {swagger_path} --restler_drop_dir {restler_drop_dir} --task fuzz-lean {use_http2}',
         shell=True, capture_output=True
     )
     expected_strings = [
@@ -85,7 +86,7 @@ def test_fuzzlean_task(restler_working_dir, swagger_path, restler_drop_dir):
     check_output_errors(output)
     check_expected_output(restler_working_dir, expected_strings, output, "FuzzLean")
 
-def test_fuzz_task(restler_working_dir, swagger_path, restler_drop_dir):
+def test_fuzz_task(restler_working_dir, swagger_path, restler_drop_dir, use_http2):
     import json
     compile_dir = Path(restler_working_dir, f'Compile')
     settings_file_path = compile_dir.joinpath('engine_settings.json')
@@ -105,17 +106,17 @@ def test_fuzz_task(restler_working_dir, swagger_path, restler_drop_dir):
         'Task Fuzz succeeded.'
     ]
     output = subprocess.run(
-        f'python ./restler-quick-start.py --api_spec_path {swagger_path} --restler_drop_dir {restler_drop_dir} --task fuzz',
+        f'python ./restler-quick-start.py --api_spec_path {swagger_path} --restler_drop_dir {restler_drop_dir} --task fuzz {use_http2}',
         shell=True, capture_output=True
     )
     check_output_errors(output)
     # check_expected_output(restler_working_dir, expected_strings, output)
 
-def test_replay_task(restler_working_dir, task_output_dir, restler_drop_dir):
+def test_replay_task(restler_working_dir, task_output_dir, restler_drop_dir, use_http2):
     # Run the quick start script
     print(f"Testing replay for bugs found in task output dir: {task_output_dir}")
     output = subprocess.run(
-        f'python ./restler-quick-start.py --replay_bug_buckets_dir {task_output_dir} --restler_drop_dir {restler_drop_dir} --task replay',
+        f'python ./restler-quick-start.py --replay_bug_buckets_dir {task_output_dir} --restler_drop_dir {restler_drop_dir} --task replay {use_http2}',
         shell=True, capture_output=True
     )
     check_output_errors(output)
@@ -137,7 +138,7 @@ def test_replay_task(restler_working_dir, task_output_dir, restler_drop_dir):
     with open(network_log) as rf, open(original_bug_buckets_file_path) as of:
         orig_buckets = of.read()
         log_contents = rf.read()
-        if 'HTTP/1.1 500 Internal Server Error' not in log_contents:
+        if 'HTTP/1.1 500 Internal Server Error' not in log_contents and 'HTTP/2.0 500 Internal Server Error' not in log_contents:
             raise QuickStartFailedException(f"Failing because bug buckets {orig_buckets} were not reproduced.  Replay log: {log_contents}.")
         else:
             print("500 error was reproduced.")
@@ -165,7 +166,11 @@ def get_demo_server_output(demo_server_process):
     else:
         creationflags = 0
 
-    demo_server_process = subprocess.Popen([sys.executable, demo_server_path],
+    use_http2 = ""
+    if len(sys.argv) > 2:
+        use_http2 = sys.argv[2]
+
+    demo_server_process = subprocess.Popen([sys.executable, demo_server_path, use_http2],
                                             stdout=subprocess.PIPE,
                                             stderr=subprocess.STDOUT,
                                             creationflags=creationflags)
@@ -181,17 +186,17 @@ def get_demo_server_output(demo_server_process):
     test_failed = False
     try:
         print("+++++++++++++++++++++++++++++test...")
-        test_test_task(restler_working_dir, swagger_path, restler_drop_dir)
+        test_test_task(restler_working_dir, swagger_path, restler_drop_dir, use_http2)
 
         print("+++++++++++++++++++++++++++++fuzzlean...")
-        test_fuzzlean_task(restler_working_dir, swagger_path, restler_drop_dir)
+        test_fuzzlean_task(restler_working_dir, swagger_path, restler_drop_dir, use_http2)
 
         print("+++++++++++++++++++++++++++++replay...")
         fuzzlean_task_dir = os.path.join(curr, RESTLER_WORKING_DIR, 'FuzzLean')
-        test_replay_task(restler_working_dir, fuzzlean_task_dir, restler_drop_dir)
+        test_replay_task(restler_working_dir, fuzzlean_task_dir, restler_drop_dir, use_http2)
 
         #print("+++++++++++++++++++++++++++++fuzz...")
-        #test_fuzz_task(restler_working_dir, swagger_path, restler_drop_dir)
+        #test_fuzz_task(restler_working_dir, swagger_path, restler_drop_dir, use_http2)
 
     except Exception:
         test_failed = True