On shutdown (of any kind) log summary of test cases.

On all tests completing or failing, shutdown automatically.

trafficlight/__init__.py

@@ -12,25 +12,28 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import asyncio
 import json
 import logging
 import os
 import uuid
+from asyncio import Future
 from datetime import datetime, timedelta
 from typing import Any, Dict, Optional
 
 from quart import Quart
 
-import trafficlight
 import trafficlight.kiwi as kiwi
 from trafficlight.homerunner import HomerunnerClient
 from trafficlight.http.adapter import (
     adapter_shutdown,
     loop_check_for_new_tests,
     loop_cleanup_unresponsive_adapters,
+    stop_background_tasks,
+    interrupt_tasks,
 )
 from trafficlight.internals.testsuite import TestSuite
-from trafficlight.store import add_testsuite
+from trafficlight.store import add_testsuite, get_testsuites
 from trafficlight.tests import load_tests
 
 logger = logging.getLogger(__name__)
@@ -110,19 +113,34 @@ def create_app(test_config: Optional[Dict[str, Any]] = None) -> Quart:
     )
     app.jinja_env.filters["delaytime"] = format_delaytime
 
+    async def on_done(f: Future) -> None:
+
+        await app.shutdown()
+
     @app.before_serving
     async def startup() -> None:
         app.add_background_task(loop_cleanup_unresponsive_adapters)
         app.add_background_task(loop_check_for_new_tests)
         if kiwi.kiwi_client:
             await kiwi.kiwi_client.start_run()
+        finished = asyncio.gather(*list(map(lambda x: x.completed, get_testsuites())))
+        finished.add_done_callback(on_done)
 
     @app.after_serving
     async def shutdown() -> None:
-        trafficlight.http.adapter.stop_background_tasks = True
-        await trafficlight.http.adapter.interrupt_tasks()
+        adapter.stop_background_tasks = True
+        await adapter.interrupt_tasks()
         if kiwi.kiwi_client:
             await kiwi.kiwi_client.end_run()
         await adapter_shutdown()
+        print("Results:\n\n")
+        for testsuite in get_testsuites():
+            print(
+                f"{testsuite.name()}: {testsuite.successes()}/{len(testsuite.test_cases)} successful"
+            )
+            for testcase in testsuite.test_cases:
+                print(f"  {testcase.client_types}: {testcase.state}")
+                if testcase.state != "success" and testcase.state != "waiting":
+                    print(f"{testcase.exceptions}")
 
     return app

trafficlight/http/adapter.py

@@ -132,6 +132,7 @@ async def loop_cleanup_unresponsive_adapters() -> None:
         sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30))
         try:
             sleeping_tasks.add(sleep_task)
+            await sleep_task
         except asyncio.CancelledError:
             pass  # we don't mind this task being cancelled.
         finally:
@@ -146,6 +147,7 @@ async def loop_check_for_new_tests() -> None:
         sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30))
         try:
             sleeping_tasks.add(sleep_task)
+            await sleep_task
         except asyncio.CancelledError:
             pass  # we don't mind this task being cancelled.
         finally:

trafficlight/internals/testcase.py

@@ -1,3 +1,4 @@
+import asyncio
 import hashlib
 import logging
 import traceback
@@ -40,6 +41,7 @@ def __init__(
         self.servers: List[HomeServer] = []
         self.files: Dict[str, str] = {}
         self.adapters: Optional[Dict[str, Adapter]] = None
+        self.completed = asyncio.get_event_loop().create_future()
 
     def __repr__(self) -> str:
         return f"{self.test.name()} ({self.server_type} {self.client_types})"
@@ -111,7 +113,7 @@ async def run(
             # Treating an adapter that causes another type of exception as an error
             self.state = "error"
             self.exceptions.append(e.formatted_message)
-        except Exception:
+        except Exception as e:
             # Treating everything else as an error as well... eg compilation failures
             self.state = "error"
             self.exceptions.append("".join(traceback.format_exc()))
@@ -122,3 +124,4 @@ async def run(
                 server.finished()
             if kiwi.kiwi_client:
                 await kiwi.kiwi_client.report_status(self)
+            self.completed.set_result(True)

trafficlight/internals/testsuite.py

@@ -1,3 +1,4 @@
+import asyncio
 import hashlib
 from typing import List
 
@@ -10,6 +11,8 @@ def __init__(self, test: Test, test_cases: List[TestCase]):
         self.guid = hashlib.md5(f"TestSuite{test.name()}".encode("utf-8")).hexdigest()
         self.test = test
         self.test_cases = test_cases
+        futures = list(map(lambda c: c.completed, test_cases))
+        self.completed = asyncio.gather(*futures)
 
     def name(self) -> str:
         return self.test.name()