diff --git a/distributed/asv.conf.json b/distributed/asv.conf.json
index 0495c1f..aabc0e2 100644
--- a/distributed/asv.conf.json
+++ b/distributed/asv.conf.json
@@ -59,6 +59,7 @@
         "partd": [],
         "pandas": [],
         "distributed": [],
+        "pytest": [],
         "s3fs": [],
         "python-blosc": [],
         "lz4": []
diff --git a/distributed/benchmarks/scheduler.py b/distributed/benchmarks/scheduler.py
index eaa73bb..665dbf9 100644
--- a/distributed/benchmarks/scheduler.py
+++ b/distributed/benchmarks/scheduler.py
@@ -3,7 +3,8 @@
 import random
 
 from dask import compute, delayed
-from distributed import Client, wait
+from distributed import Client, wait, Scheduler
+from distributed.utils_test import gen_test
 
 
 class SchedulerComputeDepsInMemory(object):
@@ -35,3 +36,9 @@ def time_compute_deps_already_in_memory(self):
         xref https://github.com/dask/distributed/pull/3293
         """
         compute(*self.x, scheduler=self.client)
+
+
+@gen_test()
+async def test_start_scheduler():
+    async with Scheduler(port=0):
+        pass