test: add new https_serve_directory() and test certs

This commit adds a new `https_serve_directory()` test helper
and some custom self-signed and worthless certs that are used
during testing. They are not dynamically generated to avoid the
extra compuation time during tests (but they could be).

Generated via:
```
$ openssl req -new -newkey rsa:2048  -nodes -x509  \
   -subj "/C=DE/ST=Berlin/L=Berlin/O=Org/CN=localhost"   \
   -keyout "key1.pem" -out "cert1.pem"
```

This will allow us to test `https` download URLs as well in e.g.
the curl source.

osbuild/testutil/net.py

@@ -5,6 +5,7 @@
 import contextlib
 import http.server
 import socket
+import ssl
 import threading
 
 try:
@@ -61,15 +62,35 @@ def finish_request(self, request, client_address):
             request, client_address, self, directory=self.directory)
 
 
-@contextlib.contextmanager
-def http_serve_directory(rootdir, simulate_failures=0):
-    port = _get_free_port()
-    httpd = DirHTTPServer(
+def _httpd(rootdir, port, simulate_failures):
+    return DirHTTPServer(
         ("localhost", port),
         http.server.SimpleHTTPRequestHandler,
         directory=rootdir,
         simulate_failures=simulate_failures,
     )
+
+
+@contextlib.contextmanager
+def http_serve_directory(rootdir, simulate_failures=0):
+    port = _get_free_port()
+    httpd = _httpd(rootdir, port, simulate_failures)
+    threading.Thread(target=httpd.serve_forever).start()
+    try:
+        yield httpd
+    finally:
+        httpd.shutdown()
+
+
+@contextlib.contextmanager
+def https_serve_directory(rootdir, certfile, keyfile, simulate_failures=0):
+    port = _get_free_port()
+    httpd = _httpd(rootdir, port, simulate_failures)
+
+    ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+    ctx.load_cert_chain(certfile=certfile, keyfile=keyfile)
+    httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True)
+
     threading.Thread(target=httpd.serve_forever).start()
     try:
         yield httpd

test/data/certs/README.md

@@ -0,0 +1,10 @@
+This directory contains custom self-signed and worthless certs used
+during testing. They are not dynamically generated to avoid the extra
+compuation time during tests (but they could be).
+
+Generated via:
+```
+$ openssl req -new -newkey rsa:2048  -nodes -x509  \
+   -subj "/C=DE/ST=Berlin/L=Berlin/O=Org/CN=localhost"   \
+   -keyout "key1.pem" -out "cert1.pem"
+```

test/data/certs/cert1.pem

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDgzCCAmugAwIBAgIUf5m7xgPIxGpZjJKUiXEM3DxxzswwDQYJKoZIhvcNAQEL
+BQAwUTELMAkGA1UEBhMCREUxDzANBgNVBAgMBkJlcmxpbjEPMA0GA1UEBwwGQmVy
+bGluMQwwCgYDVQQKDANPcmcxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0yNDA3MDgx
+NjQwNTVaFw0yNTA3MDgxNjQwNTVaMFExCzAJBgNVBAYTAkRFMQ8wDQYDVQQIDAZC
+ZXJsaW4xDzANBgNVBAcMBkJlcmxpbjEMMAoGA1UECgwDT3JnMRIwEAYDVQQDDAls
+b2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtdTNHFv1W
+Id4xhS++hF3RfaUpCypHS7Uwjvp5TvjVjPFDX0ekkU99VfrD90ye+u6EbBFQTMu5
+kh7hsKGJF0TYQU66rWs7kHkg02HnsxtEtr6GIatiT7PE0oRwL4+oRJ2L62vkdRvH
+OXQCEGFrrB+DqRAbmeIjZllwv5DK/sXCmtaC8IBQzgH3TL6g9f00rqFt4KR64MAw
+wICIQ68TO9BXUSEhxVOOVNMEQQftFuZV+YuV+ie+dpW4zzsC1xCgGbdd3e9p5pf1
+TUMoIxKr1/m6QZB4qgXprKSPm+HNyKaYHnYS/lGZqXaIax+iLBPG7eUdtL2jy+x9
+yn2cYg12CUILAgMBAAGjUzBRMB0GA1UdDgQWBBSNKh3raaCE3knMc0IvDFSWx6Fp
+7TAfBgNVHSMEGDAWgBSNKh3raaCE3knMc0IvDFSWx6Fp7TAPBgNVHRMBAf8EBTAD
+AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBF9Egk6lXpLpW68dxjMCOjMSe8thrvVnRu
+295zPyUkiTlU0gagDpm6dxj1MzhixjiVQlHEt62OxL/aX0ihBJ1jdaqakY6+KOfP
+iN1m7k1bBcw63ebLaGdofioQAb1vKofXYBW8inYZDZcORQ70Ln/xhv4G9wRNqQDO
+++qvEbIkkvnnsdbVPMXgMXFd4O8ehEohB74E2Isd+2WeCfB5N0NQmyQ6GqDLIxJN
+VvXeq/hxw6dLD3mZ8BRbkR5HhlcbvaEO78s3UVn5dh5PKwZP3i0ayLuXZKIaLlRo
+C3rFzPVT3NFexp7YhR+K116F47d/ou11lUUrUBWICflgo0LWKGTQ
+-----END CERTIFICATE-----

test/data/certs/cert2.pem

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDgzCCAmugAwIBAgIUTJvdHvQE7E2PDKMafh2UjNnmxBswDQYJKoZIhvcNAQEL
+BQAwUTELMAkGA1UEBhMCREUxDzANBgNVBAgMBkJlcmxpbjEPMA0GA1UEBwwGQmVy
+bGluMQwwCgYDVQQKDANPcmcxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0yNDA3MDgx
+NjIxMTNaFw0yNDA4MDcxNjIxMTNaMFExCzAJBgNVBAYTAkRFMQ8wDQYDVQQIDAZC
+ZXJsaW4xDzANBgNVBAcMBkJlcmxpbjEMMAoGA1UECgwDT3JnMRIwEAYDVQQDDAls
+b2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCfG9olqb1+
+wDxxHBEi0RmMqj7XZC4V43+hWYfhdhY2r2MOMMnR6QoYUhKL2OZVX3wrOjoNUmsw
+M9O+lxu/UIinoSoGNGTW62SN8NcerxDjENxkLRyCosVgX7eAUBvWQ2A4wG0oEHfg
+xHphk35B2KR+HzhW6vjwzoL6RE8YTCgjsjkLTuK/slRmbuB6xY4HHlfUvTjhqTfJ
+ogvg4BAMepgfnUIcMz0LlmsAntMUt1ECShJQmqrKwflJ6FBddgcY2t65FRQjcMom
+JE6ufATN9wudcWZqjrEzuF94wBtdPyRUnItxdu6W6E8XiR05DgdQibdq2SA/y1Iy
+6kRSOU6IG0onAgMBAAGjUzBRMB0GA1UdDgQWBBSexO3NW0QfNDvhjCTmkuOb25tx
+3DAfBgNVHSMEGDAWgBSexO3NW0QfNDvhjCTmkuOb25tx3DAPBgNVHRMBAf8EBTAD
+AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAIvw/2dAQVelhF6C/z6wJR/gd3T9ABzuBn
+JlBRZC5WcEetLtzUnh9oDGaIrBHHOpKrrmXXs4R6LgmxvS6yfAmxwsniLzTkUHxs
+ZET/uHI6SoiKKwVtJdUpl4ykR11hW0jjrD+0Y/ZoSwLDOmkH3NbvTFW5+TmU7xtz
+LMr3hOx96OKTojtbczspOb4r4bAA8S1+DKFdl4lu/dxBDENYvDQpIo5giyVGumQi
+rsx52mxZ1w1mraJk0Rha1Z2hKsGi2BUem0NevpOakoQG7PwtJTvwIf4T+QktQuGl
+mSHRAe9rdcv4KDQXn5qlIQ4HI/b/2LPkgh5lFYKCKMRHujNSOJGC
+-----END CERTIFICATE-----

test/data/certs/key1.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCtdTNHFv1WId4x
+hS++hF3RfaUpCypHS7Uwjvp5TvjVjPFDX0ekkU99VfrD90ye+u6EbBFQTMu5kh7h
+sKGJF0TYQU66rWs7kHkg02HnsxtEtr6GIatiT7PE0oRwL4+oRJ2L62vkdRvHOXQC
+EGFrrB+DqRAbmeIjZllwv5DK/sXCmtaC8IBQzgH3TL6g9f00rqFt4KR64MAwwICI
+Q68TO9BXUSEhxVOOVNMEQQftFuZV+YuV+ie+dpW4zzsC1xCgGbdd3e9p5pf1TUMo
+IxKr1/m6QZB4qgXprKSPm+HNyKaYHnYS/lGZqXaIax+iLBPG7eUdtL2jy+x9yn2c
+Yg12CUILAgMBAAECggEAJhD+8F6AuVrUZty6Rh7JchSXXuMG47+fI2l1WZL1s/Pq
+7KSn1dtX7QMl9cS7aXZbSu9me09nNDH3mgMH6EyVfLoUW458ZoWFt6j8JvudA97x
+AAG6mWgxZQPoMbCWX9CzHIZNVsVGk9mtdlh+8MDE58F9+fiy8p7fCQnZblssLlEs
+89kic702OIiG/SihFMWDw+V+grsns/XOsUmqkYD0HMXbNFAN6lYvLy15Jh355Tao
+RoI1W0vILMY9SKQZ2ncEcbUs/7dqAt03Ds06BF+IHgzxxK2+sBLIIqGLCTmK/qqq
+KFI1otl3Sy5g8HkPm9cAhvXo73A7vLNWrwKYz1wd/QKBgQDkB+KCpi2vApx5DWo6
+wzh6By0XfjOtoQjtl69I6QV67S2GzZYuffQirwSwJoGtN+0n0RUlISu2qg/qZQw9
+ghuzHWhAXLQFPs+tiWOkoIQQh4x0J6dZto9NMCRPgSbvi8pJ9/ZHQAnkm3Ls/uV9
+Ju2kwC9uuUGNfbRWAFFrN0m/nwKBgQDCu7x0kZ18WlYpcOpC4GmWPVuNVshZ7RMm
+M5REpUSf39jdEJGJLb60dLqdbp2yBn+0EXWCGkS/s/Q9i+Nce6DbKrBbGkFFd6VO
+IyuZn9PPClZZi1Ixotwe3IE3Pd/ksy14eWW4L9008qT7wYachLQgL0UkrAanAAgj
+xsvec882FQKBgQCAtfKkzCf9IPwxV+EeShMWzfYlzJZe6wafFX9bKCxvtnB8Jo8e
+rvmemiuLQjDwFSFH2DyXOKHz+QoCxAaksUvxTigKLElwrIOM8H/N7Myeh+Q/rCJK
+AUHhz8qvFbdkfwMvvUbF8N+kqhm0rCvc+Jwq4k/jr63Qr6Oa55MNcaevAQKBgBfT
+xtxwkNeNaiuJ8HAca28BtPkXXE58jrJG72Zy3nG9fxiVlWk2mrYT7l2jElKdLvUG
+id1Qf4cBauo4+qnvSqBmmcub686nNCfab9RNVSppNmWggX8nUU3JZ1ouDP472l16
+ZTLjUGrRIdTYOjNenXeusQQqtGmsvFI8WeevPfR9AoGBALtytPWcdG1yPZV9TcA9
+AtGkewvwuaVKMUblyD5vu2IVEirPJ3oJczsGbfaJXbgwjJDnwpexC5zDig0bOdVk
+WK5lSLVNPOYLudziZwMDxrkREAGRWoRYFqK9kjH4VHtpE4E7nQb+rfeq5WCY2X+V
+iLC9nwzOQ0JmNvZZcEpOPGnk
+-----END PRIVATE KEY-----

test/data/certs/key2.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCfG9olqb1+wDxx
+HBEi0RmMqj7XZC4V43+hWYfhdhY2r2MOMMnR6QoYUhKL2OZVX3wrOjoNUmswM9O+
+lxu/UIinoSoGNGTW62SN8NcerxDjENxkLRyCosVgX7eAUBvWQ2A4wG0oEHfgxHph
+k35B2KR+HzhW6vjwzoL6RE8YTCgjsjkLTuK/slRmbuB6xY4HHlfUvTjhqTfJogvg
+4BAMepgfnUIcMz0LlmsAntMUt1ECShJQmqrKwflJ6FBddgcY2t65FRQjcMomJE6u
+fATN9wudcWZqjrEzuF94wBtdPyRUnItxdu6W6E8XiR05DgdQibdq2SA/y1Iy6kRS
+OU6IG0onAgMBAAECggEAB+bOMp33Ydq80FvgeKYsaqaZxatH+UukWahPQzcmFe59
+MNs4v/VJdw5nCrg+2Bvh2BO626aC2ZCkAgxnEwfg1tJTep4SKUo6DYKEbGsCn67a
+rcQ/wjfAgu9ooYquQeR0Bt1845x4bO/U/ruLGEy1/pLLrXqwhOT8mYktlGX+fadB
+dYgXxAg1SNn6XY5dr/XfRqOIzO8cCpY/vAbK1uGx7H3cRomIHIqh/EU+pAkABFBC
+Xye3ODy8v4AnGkPj75g9SlilWgZcrppqAu0bkTcTf3PM7bFj/GIi/+cCvbCJnmAx
+TGMeQ2bgOwyo9O9ztOtFluIScwnmz1e6wRHolqQQoQKBgQDSqc5wxiBc++W/JAT6
+WRDZRj1QbyoSswIRDZRvYC7K9M9XJotgVCOqOimKJIW8NHPTS75MGGSPJgi+UpV2
+DOecjT/aRRdM2XpZPeng55LuPuwP/dwprfDTa+A1trtVfEV70OjKwOKaQfO3KCC4
+9y3775YjnPqY9CeSbt67fpHyswKBgQDBWblKUoGbdlb06JNva33gJ7X2drLgXd1b
+3ZWMgp5hLwQuyE/6cWK3150DaQm5gwoNkGPkNNU1lDqAjERxusNdp6SWeRAS4VfL
+EdVGHcRVfP1juyvXt6NPD/e0nmQwrJHRE3cscoLZ6dv6p0kgR5wjWRlZ/gdtiEm2
+wRzHAxZ0vQKBgAwKjLRBfuQRueYdBvPQV3gz3ZlVYVBETx/uaMec9c/AAoH2wAy3
+gnpebLVqtQI29U/kinILCuRrxiAq1EQ1JJ/8KuQtAtHvtGCZe17RncJkclXPqt/v
+A07yYZRwZ3DRc59KJ6Q/NjSfBHE5p/Uqm8REw44xkNlhS0nBcJXC/FClAoGARW/j
+iz12Lk6hi/K6gBkqa4XXY1vdgZY0GPnK61s9ZclsxyBJGLf/q+ZTDRZ715On2Jcy
+SJJwv30sRV4adn2MC+ZYxXFUxHMJBHSzEys9hzYMCfT+Gcppa8tgOg1oDdLryxoi
+gmPdekTyK2JIcL9G46rr39XK9Rss3eUcYJxLREECgYEAjoRpaZWgGdlo32I0Udfd
+QTFGUTihB9h8MzTNCK/oX+MQ42T1OMN1asE2dw1VHVjlBTrrWrKum17ORI2Tor1R
+PRBo8N4ZIKrdfodiDFwfQVfhG3HtNfiOqqaNuJY3ugwAb0RoZw1TuHHke6AlXRNd
+MA5GZwa3jZUDYOY3o5HBeho=
+-----END PRIVATE KEY-----

test/mod/test_testutil_net.py

@@ -0,0 +1,39 @@
+import os.path
+import pathlib
+import subprocess
+
+from osbuild.testutil import make_fake_tree
+from osbuild.testutil.net import http_serve_directory, https_serve_directory
+
+
+def test_http_serve_directory_smoke(tmp_path):
+    make_fake_tree(tmp_path, {
+        "file1": "file1 content",
+        "dir1/file2": "file2 content",
+    })
+    with http_serve_directory(tmp_path) as httpd:
+        output = subprocess.check_output(
+            ["curl", f"http://localhost:{httpd.server_port}/file1"])
+        assert output == b"file1 content"
+        output = subprocess.check_output(
+            ["curl", f"http://localhost:{httpd.server_port}/dir1/file2"])
+        assert output == b"file2 content"
+
+
+def test_https_serve_directory_smoke(tmp_path):
+    make_fake_tree(tmp_path, {
+        "file1": "file1 content",
+    })
+    cert_dir = pathlib.Path(__file__).parent.parent / "data/certs"
+    cacertfile = cert_dir / "cert1.pem"
+    assert cacertfile.exists()
+    keyfile = cert_dir / "key1.pem"
+    assert keyfile.exists()
+
+    with https_serve_directory(tmp_path, cacertfile, keyfile) as httpd:
+        output = subprocess.check_output(
+            ["curl",
+             "--cacert", os.fspath(cacertfile),
+             f"https://localhost:{httpd.server_port}/file1"],
+        )
+        assert output == b"file1 content"