execute batch downloads in parallel worker threads

- limit downloads to 10 at a time instead of starting all at once
- use more specific types for BatchDownloader#__call__
- add NEWS
- calculate byte lengths with a HEAD request
- add cli arg to limit download parallelism
- initial implementation of pooled progress bar
- pooled progress bar looks very nice
- factor out receiving thread exceptions into a contextmanager
- default batch parallelism to 10
- quiet all progress output from -q
- don't write colored output with --no-color
- make batch download parallelism 1 in html index test
- write a lot more documentation for the new progress bar logic
- use ProgressBarType enum everywhere

news/12923.feature.rst

@@ -0,0 +1 @@
+Download concrete dists for metadata-only resolves in parallel using worker threads. Add ``--batch-download-parallelism`` CLI flag to limit parallelism. Use ``ProgressBarType`` enum class for ``--progress-bar`` choices.

src/pip/_internal/cli/cmdoptions.py

@@ -22,6 +22,7 @@
 from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.cli.parser import ConfigOptionParser
+from pip._internal.cli.progress_bars import ProgressBarType
 from pip._internal.exceptions import CommandError
 from pip._internal.locations import USER_CACHE_DIR, get_src_prefix
 from pip._internal.models.format_control import FormatControl
@@ -226,11 +227,32 @@ class PipOption(Option):
     "--progress-bar",
     dest="progress_bar",
     type="choice",
-    choices=["on", "off", "raw"],
-    default="on",
-    help="Specify whether the progress bar should be used [on, off, raw] (default: on)",
+    choices=ProgressBarType.choices(),
+    default=ProgressBarType.ON.value,
+    help=(
+        "Specify whether the progress bar should be used"
+        f" {ProgressBarType.help_choices()} (default: %default)"
+    ),
+)
+
+
+batch_download_parallelism: Callable[..., Option] = partial(
+    Option,
+    "--batch-download-parallelism",
+    dest="batch_download_parallelism",
+    type="int",
+    default=10,
+    help=(
+        "Maximum parallelism employed for batch downloading of metadata-only dists"
+        " (default %default parallel requests)."
+        " Note that more than 10 downloads may overflow the requests connection pool,"
+        " which may affect performance."
+        " Note also that commands such as 'install --dry-run' should avoid downloads"
+        " entirely, and so will not be affected by this option."
+    ),
 )
 
+
 log: Callable[..., Option] = partial(
     PipOption,
     "--log",