diff --git a/README.md b/README.md index 8fb02eb..7ed5bda 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ The environment variables can be used to configure various aspects of the inner | `CODER_CPUS` | Dictates the number of CPUs to allocate the inner container. It is recommended to set this using the Kubernetes [Downward API](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/#use-container-fields-as-values-for-environment-variables). | false | | `CODER_MEMORY` | Dictates the max memory (in bytes) to allocate the inner container. It is recommended to set this using the Kubernetes [Downward API](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/#use-container-fields-as-values-for-environment-variables). | false | | `CODER_DISABLE_IDMAPPED_MOUNT` | Disables idmapped mounts in sysbox. For more information, see the [Sysbox Documentation](https://github.com/nestybox/sysbox/blob/master/docs/user-guide/configuration.md#disabling-id-mapped-mounts-on-sysbox). | false | +| `CODER_EXTRA_CERTS_PATH` | A path to a file or directory containing CA certificates that should be made when communicating to external services (e.g. the Coder control plane or a Docker registry) | false | ## Coder Template diff --git a/buildlog/coder.go b/buildlog/coder.go index 00ab6e9..6a7447d 100644 --- a/buildlog/coder.go +++ b/buildlog/coder.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "net/http" "net/url" "time" @@ -143,9 +144,10 @@ func newAgentClientV2(ctx context.Context, logger slog.Logger, client *agentsdk. }, nil } -func OpenCoderClient(ctx context.Context, accessURL *url.URL, logger slog.Logger, token string) (CoderClient, error) { +func OpenCoderClient(ctx context.Context, logger slog.Logger, accessURL *url.URL, hc *http.Client, token string) (CoderClient, error) { client := agentsdk.New(accessURL) client.SetSessionToken(token) + client.SDK.HTTPClient = hc resp, err := client.SDK.BuildInfo(ctx) if err != nil { diff --git a/cli/docker.go b/cli/docker.go index 5abe59a..921bcfc 100644 --- a/cli/docker.go +++ b/cli/docker.go @@ -28,6 +28,7 @@ import ( "github.com/coder/envbox/dockerutil" "github.com/coder/envbox/slogkubeterminate" "github.com/coder/envbox/sysboxutil" + "github.com/coder/envbox/xhttp" "github.com/coder/envbox/xunix" ) @@ -101,6 +102,7 @@ var ( EnvDockerConfig = "CODER_DOCKER_CONFIG" EnvDebug = "CODER_DEBUG" EnvDisableIDMappedMount = "CODER_DISABLE_IDMAPPED_MOUNT" + EnvExtraCertsPath = "CODER_EXTRA_CERTS_PATH" ) var envboxPrivateMounts = map[string]struct{}{ @@ -138,6 +140,7 @@ type flags struct { cpus int memory int disableIDMappedMount bool + extraCertsPath string // Test flags. noStartupLogs bool @@ -158,6 +161,11 @@ func dockerCmd() *cobra.Command { blog buildlog.Logger = buildlog.NopLogger{} ) + httpClient, err := xhttp.Client(log, flags.extraCertsPath) + if err != nil { + return xerrors.Errorf("http client: %w", err) + } + if !flags.noStartupLogs { log = slog.Make(slogjson.Sink(cmd.ErrOrStderr()), slogkubeterminate.Make()).Leveled(slog.LevelDebug) blog = buildlog.JSONLogger{Encoder: json.NewEncoder(os.Stderr)} @@ -169,7 +177,7 @@ func dockerCmd() *cobra.Command { return xerrors.Errorf("parse coder URL %q: %w", flags.coderURL, err) } - agent, err := buildlog.OpenCoderClient(ctx, coderURL, log, flags.agentToken) + agent, err := buildlog.OpenCoderClient(ctx, log, coderURL, httpClient, flags.agentToken) if err != nil { // Don't fail workspace startup on // an inability to push build logs. @@ -349,6 +357,7 @@ func dockerCmd() *cobra.Command { cliflag.IntVarP(cmd.Flags(), &flags.cpus, "cpus", "", EnvCPUs, 0, "Number of CPUs to allocate inner container. e.g. 2") cliflag.IntVarP(cmd.Flags(), &flags.memory, "memory", "", EnvMemory, 0, "Max memory to allocate to the inner container in bytes.") cliflag.BoolVarP(cmd.Flags(), &flags.disableIDMappedMount, "disable-idmapped-mount", "", EnvDisableIDMappedMount, false, "Disable idmapped mounts in sysbox. Note that you may need an alternative (e.g. shiftfs).") + cliflag.StringVarP(cmd.Flags(), &flags.extraCertsPath, "extra-certs-path", "", EnvExtraCertsPath, "", "The path to a directory or file containing extra CA certificates.") // Test flags. cliflag.BoolVarP(cmd.Flags(), &flags.noStartupLogs, "no-startup-log", "", "", false, "Do not log startup logs. Useful for testing.") diff --git a/go.mod b/go.mod index 4497c1e..53ac218 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,8 @@ go 1.22.4 // https://github.com/tailscale/tailscale/compare/main...coder:tailscale:main replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240530071520-1ac63d3a4ee3 +replace github.com/gliderlabs/ssh => github.com/coder/ssh v0.0.0-20231128192721-70855dedb788 + require ( cdr.dev/slog v1.6.2-0.20240126064726-20367d4aede6 github.com/coder/coder/v2 v2.12.0 diff --git a/integration/docker_test.go b/integration/docker_test.go index fc1b7af..515d234 100644 --- a/integration/docker_test.go +++ b/integration/docker_test.go @@ -5,6 +5,7 @@ package integration_test import ( "fmt" + "net" "os" "path/filepath" "strconv" @@ -37,9 +38,9 @@ func TestDocker(t *testing.T) { runEnvbox := func() *dockertest.Resource { // Run the envbox container. resource := integrationtest.RunEnvbox(t, pool, &integrationtest.CreateDockerCVMConfig{ - Image: integrationtest.DockerdImage, - Username: "root", - Binds: binds, + Image: integrationtest.DockerdImage, + Username: "root", + OuterMounts: binds, }) // Wait for the inner container's docker daemon. @@ -98,8 +99,8 @@ func TestDocker(t *testing.T) { require.NoError(t, err) binds = append(binds, - bindMount(homeDir, "/home/coder", false), - bindMount(secretDir, "/var/secrets", true), + integrationtest.BindMount(homeDir, "/home/coder", false), + integrationtest.BindMount(secretDir, "/var/secrets", true), ) var ( @@ -144,7 +145,7 @@ func TestDocker(t *testing.T) { Username: "coder", InnerEnvFilter: envFilter, Envs: envs, - Binds: binds, + OuterMounts: binds, AddFUSE: true, AddTUN: true, BootstrapScript: bootstrapScript, @@ -272,6 +273,83 @@ func TestDocker(t *testing.T) { require.NoError(t, err) require.Equal(t, expectedHostname, strings.TrimSpace(string(hostname))) }) + + t.Run("SelfSignedCerts", func(t *testing.T) { + t.Parallel() + + var ( + dir = integrationtest.TmpDir(t) + binds = integrationtest.DefaultBinds(t, dir) + ) + + pool, err := dockertest.NewPool("") + require.NoError(t, err) + + // Create some listeners for the Docker and Coder + // services we'll be running with self signed certs. + bridgeIP := integrationtest.DockerBridgeIP(t) + coderListener, err := net.Listen("tcp", fmt.Sprintf("%s:0", bridgeIP)) + require.NoError(t, err) + defer coderListener.Close() + coderAddr := tcpAddr(t, coderListener) + + registryListener, err := net.Listen("tcp", fmt.Sprintf("%s:0", bridgeIP)) + require.NoError(t, err) + err = registryListener.Close() + require.NoError(t, err) + registryAddr := tcpAddr(t, registryListener) + + coderCert := integrationtest.GenerateTLSCertificate(t, "host.docker.internal", coderAddr.IP.String()) + dockerCert := integrationtest.GenerateTLSCertificate(t, "host.docker.internal", registryAddr.IP.String()) + + // Startup our fake Coder "control-plane". + recorder := integrationtest.FakeBuildLogRecorder(t, coderListener, coderCert) + + certDir := integrationtest.MkdirAll(t, dir, "certs") + + // Write the Coder cert disk. + coderCertPath := filepath.Join(certDir, "coder_cert.pem") + coderKeyPath := filepath.Join(certDir, "coder_key.pem") + integrationtest.WriteCertificate(t, coderCert, coderCertPath, coderKeyPath) + coderCertMount := integrationtest.BindMount(certDir, "/tmp/certs", false) + + // Write the Registry cert to disk. + regCertPath := filepath.Join(certDir, "registry_cert.crt") + regKeyPath := filepath.Join(certDir, "registry_key.pem") + integrationtest.WriteCertificate(t, dockerCert, regCertPath, regKeyPath) + + // Start up the docker registry and push an image + // to it that we can reference. + image := integrationtest.RunLocalDockerRegistry(t, pool, integrationtest.RegistryConfig{ + HostCertPath: regCertPath, + HostKeyPath: regKeyPath, + Image: integrationtest.UbuntuImage, + TLSPort: strconv.Itoa(registryAddr.Port), + }) + + // Mount the cert into the expected location + // for the Envbox Docker daemon. + regCAPath := filepath.Join("/etc/docker/certs.d", image.Registry(), "ca.crt") + registryCAMount := integrationtest.BindMount(regCertPath, regCAPath, false) + + envs := []string{ + integrationtest.EnvVar(cli.EnvAgentToken, "faketoken"), + integrationtest.EnvVar(cli.EnvAgentURL, fmt.Sprintf("https://%s:%d", "host.docker.internal", coderAddr.Port)), + integrationtest.EnvVar(cli.EnvExtraCertsPath, "/tmp/certs"), + } + + // Run the envbox container. + _ = integrationtest.RunEnvbox(t, pool, &integrationtest.CreateDockerCVMConfig{ + Image: image.String(), + Username: "coder", + Envs: envs, + OuterMounts: append(binds, coderCertMount, registryCAMount), + }) + + // This indicates we've made it all the way to end + // of the logs we attempt to push. + require.True(t, recorder.ContainsLog("Bootstrapping workspace...")) + }) } func requireSliceNoContains(t *testing.T, ss []string, els ...string) { @@ -297,9 +375,10 @@ func requireSliceContains(t *testing.T, ss []string, els ...string) { } } -func bindMount(src, dest string, ro bool) string { - if ro { - return fmt.Sprintf("%s:%s:%s", src, dest, "ro") - } - return fmt.Sprintf("%s:%s", src, dest) +func tcpAddr(t testing.TB, l net.Listener) *net.TCPAddr { + t.Helper() + + tcpAddr, ok := l.Addr().(*net.TCPAddr) + require.True(t, ok) + return tcpAddr } diff --git a/integration/integrationtest/certs.go b/integration/integrationtest/certs.go new file mode 100644 index 0000000..648dc43 --- /dev/null +++ b/integration/integrationtest/certs.go @@ -0,0 +1,82 @@ +package integrationtest + +import ( + "bytes" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func GenerateTLSCertificate(t testing.TB, commonName string, ipAddr string) tls.Certificate { + t.Helper() + + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + Organization: []string{"Acme Co"}, + CommonName: commonName, + }, + DNSNames: []string{commonName}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(time.Hour * 24 * 180), + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IPAddresses: []net.IP{net.ParseIP(ipAddr)}, + IsCA: true, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + require.NoError(t, err) + var certFile bytes.Buffer + require.NoError(t, err) + _, err = certFile.Write(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})) + require.NoError(t, err) + privateKeyBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) + require.NoError(t, err) + var keyFile bytes.Buffer + err = pem.Encode(&keyFile, &pem.Block{Type: "PRIVATE KEY", Bytes: privateKeyBytes}) + require.NoError(t, err) + cert, err := tls.X509KeyPair(certFile.Bytes(), keyFile.Bytes()) + require.NoError(t, err) + return cert +} + +func writePEM(t testing.TB, path string, typ string, contents []byte) { + t.Helper() + + f, err := os.Create(path) + require.NoError(t, err) + defer f.Close() + + err = pem.Encode(f, &pem.Block{ + Type: typ, + Bytes: contents, + }) + require.NoError(t, err) +} + +func WriteCertificate(t testing.TB, c tls.Certificate, certPath, keyPath string) { + require.Len(t, c.Certificate, 1, "expecting 1 certificate") + key, err := x509.MarshalPKCS8PrivateKey(c.PrivateKey) + require.NoError(t, err) + + cert := c.Certificate[0] + + writePEM(t, keyPath, "PRIVATE KEY", key) + writePEM(t, certPath, "CERTIFICATE", cert) +} diff --git a/integration/integrationtest/coder.go b/integration/integrationtest/coder.go new file mode 100644 index 0000000..c3eb5d9 --- /dev/null +++ b/integration/integrationtest/coder.go @@ -0,0 +1,95 @@ +package integrationtest + +import ( + "crypto/tls" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/codersdk/agentsdk" +) + +type BuildLogRecorder struct { + mu sync.Mutex + logs []string +} + +func (b *BuildLogRecorder) ContainsLog(l string) bool { + b.mu.Lock() + defer b.mu.Unlock() + for _, log := range b.logs { + if log == l { + return true + } + } + return false +} + +func (b *BuildLogRecorder) append(log string) { + b.mu.Lock() + defer b.mu.Unlock() + + b.logs = append(b.logs, log) +} + +// FakeBuildLogRecorder starts a server that fakes a Coder +// deployment for the purpose of pushing build logs. +// It returns a type for asserting that expected log +// make it through the expected endpoint. +func FakeBuildLogRecorder(t testing.TB, l net.Listener, cert tls.Certificate) *BuildLogRecorder { + t.Helper() + + recorder := &BuildLogRecorder{} + mux := http.NewServeMux() + mux.Handle("/api/v2/buildinfo", http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(http.StatusOK) + + enc := json.NewEncoder(w) + enc.SetEscapeHTML(true) + + // We can't really do much about these errors, it's probably due to a + // dropped connection. + _ = enc.Encode(&codersdk.BuildInfoResponse{ + Version: "v1.0.0", + }) + })) + + mux.Handle("/api/v2/workspaceagents/me/logs", http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + var logs agentsdk.PatchLogs + err := json.NewDecoder(r.Body).Decode(&logs) + require.NoError(t, err) + w.WriteHeader(http.StatusOK) + for _, log := range logs.Logs { + recorder.append(log.Output) + } + })) + + mux.Handle("/", http.HandlerFunc( + func(_ http.ResponseWriter, r *http.Request) { + t.Fatalf("unexpected route %v", r.URL.Path) + })) + + s := httptest.NewUnstartedServer(mux) + s.Listener = l + if cert.Certificate != nil { + //nolint:gosec + s.TLS = &tls.Config{ + Certificates: []tls.Certificate{cert}, + } + s.StartTLS() + } else { + s.Start() + } + + t.Cleanup(s.Close) + return recorder +} diff --git a/integration/integrationtest/docker.go b/integration/integrationtest/docker.go index 96ebd4e..86b8a00 100644 --- a/integration/integrationtest/docker.go +++ b/integration/integrationtest/docker.go @@ -4,10 +4,14 @@ import ( "bufio" "bytes" "context" + "crypto/tls" "encoding/json" "fmt" "io" + "net" + "net/http" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -34,6 +38,12 @@ const ( // UbuntuImage is just vanilla ubuntu (80MB) but the user is set to a non-root // user . UbuntuImage = "gcr.io/coder-dev-1/sreya/ubuntu-coder" + + // RegistryImage is used to assert that we add certs + // correctly to the docker daemon when pulling an image + // from a registry with a self signed cert. + registryImage = "gcr.io/coder-dev-1/sreya/registry" + registryTag = "2.8.3" ) // TODO use df to determine if an environment is running in a docker container or not. @@ -44,11 +54,11 @@ type CreateDockerCVMConfig struct { BootstrapScript string InnerEnvFilter []string Envs []string - Binds []string - Mounts []string - AddFUSE bool - AddTUN bool - CPUs int + + OuterMounts []docker.HostMount + AddFUSE bool + AddTUN bool + CPUs int } func (c CreateDockerCVMConfig) validate(t *testing.T) { @@ -63,6 +73,13 @@ func (c CreateDockerCVMConfig) validate(t *testing.T) { } } +type CoderdOptions struct { + TLSEnable bool + TLSCert string + TLSKey string + DefaultImage string +} + // RunEnvbox runs envbox, it returns once the inner container has finished // spinning up. func RunEnvbox(t *testing.T, pool *dockertest.Pool, conf *CreateDockerCVMConfig) *dockertest.Resource { @@ -72,9 +89,9 @@ func RunEnvbox(t *testing.T, pool *dockertest.Pool, conf *CreateDockerCVMConfig) // If binds aren't passed then we'll just create the minimum amount. // If someone is passing them we'll assume they know what they're doing. - if conf.Binds == nil { + if conf.OuterMounts == nil { tmpdir := TmpDir(t) - conf.Binds = DefaultBinds(t, tmpdir) + conf.OuterMounts = DefaultBinds(t, tmpdir) } conf.Envs = append(conf.Envs, cmdLineEnvs(conf)...) @@ -85,10 +102,11 @@ func RunEnvbox(t *testing.T, pool *dockertest.Pool, conf *CreateDockerCVMConfig) Entrypoint: []string{"/envbox", "docker"}, Env: conf.Envs, }, func(host *docker.HostConfig) { - host.Binds = conf.Binds + host.Mounts = conf.OuterMounts host.Privileged = true host.CPUPeriod = int64(dockerutil.DefaultCPUPeriod) host.CPUQuota = int64(conf.CPUs) * int64(dockerutil.DefaultCPUPeriod) + host.ExtraHosts = []string{"host.docker.internal:host-gateway"} }) require.NoError(t, err) // t.Cleanup(func() { _ = pool.Purge(resource) }) @@ -98,22 +116,11 @@ func RunEnvbox(t *testing.T, pool *dockertest.Pool, conf *CreateDockerCVMConfig) return resource } -// TmpDir returns a subdirectory in /tmp that can be used for test files. -func TmpDir(t *testing.T) string { - // We use os.MkdirTemp as oposed to t.TempDir since the envbox container will - // chown some of the created directories here to root:root causing the cleanup - // function to fail once the test exits. - tmpdir, err := os.MkdirTemp("", strings.ReplaceAll(t.Name(), "/", "_")) - require.NoError(t, err) - t.Logf("using tmpdir %s", tmpdir) - return tmpdir -} - // DefaultBinds returns the minimum amount of mounts necessary to spawn // envbox successfully. Since envbox will chown some of these directories // to root, they cannot be cleaned up post-test, meaning that it may be // necesssary to manually clear /tmp from time to time. -func DefaultBinds(t *testing.T, rootDir string) []string { +func DefaultBinds(t *testing.T, rootDir string) []docker.HostMount { t.Helper() // Create a bunch of mounts for the envbox container. Some proceses @@ -141,13 +148,39 @@ func DefaultBinds(t *testing.T, rootDir string) []string { err = os.MkdirAll(sysbox, 0o777) require.NoError(t, err) - return []string{ - fmt.Sprintf("%s:%s", cntDockerDir, "/var/lib/coder/docker"), - fmt.Sprintf("%s:%s", cntDir, "/var/lib/coder/containers"), - "/usr/src:/usr/src", - "/lib/modules:/lib/modules", - fmt.Sprintf("%s:/var/lib/sysbox", sysbox), - fmt.Sprintf("%s:/var/lib/docker", dockerDir), + return []docker.HostMount{ + { + Source: cntDockerDir, + Target: "/var/lib/coder/docker", + Type: "bind", + }, + { + Source: cntDir, + Target: "/var/lib/coder/containers", + Type: "bind", + }, + { + Source: "/usr/src", + Target: "/usr/src", + Type: "bind", + ReadOnly: true, + }, + { + Source: "/lib/modules", + Target: "/lib/modules", + Type: "bind", + ReadOnly: true, + }, + { + Source: sysbox, + Target: "/var/lib/sysbox", + Type: "bind", + }, + { + Source: dockerDir, + Target: "/var/lib/docker", + Type: "bind", + }, } } @@ -237,7 +270,7 @@ func ExecInnerContainer(t *testing.T, pool *dockertest.Pool, conf ExecConfig) ([ func ExecEnvbox(t *testing.T, pool *dockertest.Pool, conf ExecConfig) ([]byte, error) { t.Helper() - exec, err := pool.Client.CreateExec(docker.CreateExecOptions{ + cmd, err := pool.Client.CreateExec(docker.CreateExecOptions{ Cmd: conf.Cmd, AttachStdout: true, AttachStderr: true, @@ -247,13 +280,13 @@ func ExecEnvbox(t *testing.T, pool *dockertest.Pool, conf ExecConfig) ([]byte, e require.NoError(t, err) var buf bytes.Buffer - err = pool.Client.StartExec(exec.ID, docker.StartExecOptions{ + err = pool.Client.StartExec(cmd.ID, docker.StartExecOptions{ OutputStream: &buf, ErrorStream: &buf, }) require.NoError(t, err) - insp, err := pool.Client.InspectExec(exec.ID) + insp, err := pool.Client.InspectExec(cmd.ID) require.NoError(t, err) require.Equal(t, false, insp.Running) @@ -268,33 +301,210 @@ func ExecEnvbox(t *testing.T, pool *dockertest.Pool, conf ExecConfig) ([]byte, e // but using their env var alias. func cmdLineEnvs(c *CreateDockerCVMConfig) []string { envs := []string{ - envVar(cli.EnvInnerImage, c.Image), - envVar(cli.EnvInnerUsername, c.Username), + EnvVar(cli.EnvInnerImage, c.Image), + EnvVar(cli.EnvInnerUsername, c.Username), } if len(c.InnerEnvFilter) > 0 { - envs = append(envs, envVar(cli.EnvInnerEnvs, strings.Join(c.InnerEnvFilter, ","))) - } - - if len(c.Mounts) > 0 { - envs = append(envs, envVar(cli.EnvMounts, strings.Join(c.Mounts, ","))) + envs = append(envs, EnvVar(cli.EnvInnerEnvs, strings.Join(c.InnerEnvFilter, ","))) } if c.AddFUSE { - envs = append(envs, envVar(cli.EnvAddFuse, "true")) + envs = append(envs, EnvVar(cli.EnvAddFuse, "true")) } if c.AddTUN { - envs = append(envs, envVar(cli.EnvAddTun, "true")) + envs = append(envs, EnvVar(cli.EnvAddTun, "true")) } if c.BootstrapScript != "" { - envs = append(envs, envVar(cli.EnvBootstrap, c.BootstrapScript)) + envs = append(envs, EnvVar(cli.EnvBootstrap, c.BootstrapScript)) } return envs } -func envVar(k, v string) string { +func EnvVar(k, v string) string { return fmt.Sprintf("%s=%s", k, v) } + +func DockerBridgeIP(t testing.TB) string { + t.Helper() + + ifaces, err := net.Interfaces() + require.NoError(t, err) + + for _, iface := range ifaces { + if iface.Name != "docker0" { + continue + } + addrs, err := iface.Addrs() + require.NoError(t, err) + + for _, addr := range addrs { + if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if ipnet.IP.To4() != nil { + return ipnet.IP.String() + } + } + } + } + + t.Fatalf("failed to find docker bridge interface") + return "" +} + +type RegistryConfig struct { + HostCertPath string + HostKeyPath string + TLSPort string + Image string +} + +type RegistryImage string + +func (r RegistryImage) Registry() string { + return strings.Split(string(r), "/")[0] +} + +func (r RegistryImage) String() string { + return string(r) +} + +func RunLocalDockerRegistry(t testing.TB, pool *dockertest.Pool, conf RegistryConfig) RegistryImage { + t.Helper() + + const ( + certPath = "/certs/cert.pem" + keyPath = "/certs/key.pem" + ) + + resource, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: registryImage, + Tag: registryTag, + Env: []string{ + EnvVar("REGISTRY_HTTP_TLS_CERTIFICATE", certPath), + EnvVar("REGISTRY_HTTP_TLS_KEY", keyPath), + EnvVar("REGISTRY_HTTP_ADDR", "0.0.0.0:443"), + }, + ExposedPorts: []string{"443/tcp"}, + }, func(host *docker.HostConfig) { + host.Binds = []string{ + mountBinding(conf.HostCertPath, certPath), + mountBinding(conf.HostKeyPath, keyPath), + } + host.ExtraHosts = []string{"host.docker.internal:host-gateway"} + host.PortBindings = map[docker.Port][]docker.PortBinding{ + "443/tcp": {{ + HostIP: "0.0.0.0", + HostPort: conf.TLSPort, + }}, + } + }) + require.NoError(t, err) + + host := net.JoinHostPort("0.0.0.0", conf.TLSPort) + url := fmt.Sprintf("https://%s/v2/_catalog", host) + + waitForRegistry(t, pool, resource, url) + return pushLocalImage(t, pool, host, conf.Image) +} + +func waitForRegistry(t testing.TB, pool *dockertest.Pool, resource *dockertest.Resource, url string) { + t.Helper() + + //nolint:forcetypeassert + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{ + // We're not interested in asserting the validity + // of the certificate when pushing the image + // since this is setup. + //nolint:gosec + InsecureSkipVerify: true, + } + client := &http.Client{ + Transport: transport, + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + for r := retry.New(time.Second, time.Second); r.Wait(ctx); { + container, err := pool.Client.InspectContainer(resource.Container.ID) + require.NoError(t, err) + require.True(t, container.State.Running, "%v unexpectedly exited", container.ID) + + //nolint:noctx + res, err := client.Get(url) + if err != nil { + continue + } + _ = res.Body.Close() + if res.StatusCode == http.StatusOK { + return + } + } + require.NoError(t, ctx.Err()) +} + +func pushLocalImage(t testing.TB, pool *dockertest.Pool, host, remoteImage string) RegistryImage { + t.Helper() + + const registryHost = "127.0.0.1" + name := filepath.Base(remoteImage) + repoTag := strings.Split(name, ":") + tag := "latest" + if len(repoTag) == 2 { + tag = repoTag[1] + } + + tw := &testWriter{ + t: t, + } + err := pool.Client.PullImage(docker.PullImageOptions{ + Repository: strings.Split(remoteImage, ":")[0], + Tag: tag, + OutputStream: tw, + }, docker.AuthConfiguration{}) + require.NoError(t, err) + + _, port, err := net.SplitHostPort(host) + require.NoError(t, err) + + err = pool.Client.TagImage(remoteImage, docker.TagImageOptions{ + Repo: fmt.Sprintf("%s:%s/%s", registryHost, port, name), + Tag: tag, + }) + require.NoError(t, err) + + // Idk what to tell you but the pool.Client.PushImage + // function is bugged or I'm just dumb... + image := fmt.Sprintf("%s:%s/%s:%s", registryHost, port, name, tag) + cmd := exec.Command("docker", "push", image) + cmd.Stderr = tw + cmd.Stdout = tw + err = cmd.Run() + require.NoError(t, err) + return RegistryImage(fmt.Sprintf("host.docker.internal:%s/%s:%s", port, name, tag)) +} + +func mountBinding(src, dst string) string { + return fmt.Sprintf("%s:%s", src, dst) +} + +type testWriter struct { + t testing.TB +} + +func (t *testWriter) Write(b []byte) (int, error) { + t.t.Logf("%s", b) + return len(b), nil +} + +func BindMount(src, dst string, ro bool) docker.HostMount { + return docker.HostMount{ + Source: src, + Target: dst, + ReadOnly: ro, + Type: "bind", + } +} diff --git a/integration/integrationtest/os.go b/integration/integrationtest/os.go new file mode 100644 index 0000000..45aacd7 --- /dev/null +++ b/integration/integrationtest/os.go @@ -0,0 +1,38 @@ +package integrationtest + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TmpDir returns a subdirectory in /tmp that can be used for test files. +func TmpDir(t *testing.T) string { + // We use os.MkdirTemp as oposed to t.TempDir since the envbox container will + // chown some of the created directories here to root:root causing the cleanup + // function to fail once the test exits. + tmpdir, err := os.MkdirTemp("", strings.ReplaceAll(t.Name(), "/", "_")) + require.NoError(t, err) + t.Logf("using tmpdir %s", tmpdir) + return tmpdir +} + +func MkdirAll(t testing.TB, elem ...string) string { + t.Helper() + + path := filepath.Join(elem...) + err := os.MkdirAll(path, 0o777) + require.NoError(t, err) + return path +} + +func WriteFile(t *testing.T, path, contents string) { + t.Helper() + + //nolint:gosec + err := os.WriteFile(path, []byte(contents), 0o644) + require.NoError(t, err) +} diff --git a/xhttp/client.go b/xhttp/client.go new file mode 100644 index 0000000..e3e34ac --- /dev/null +++ b/xhttp/client.go @@ -0,0 +1,89 @@ +package xhttp + +import ( + "context" + "crypto/tls" + "crypto/x509" + "net/http" + "os" + "path/filepath" + + "golang.org/x/xerrors" + + "cdr.dev/slog" +) + +func Client(log slog.Logger, extraCertsPath string) (*http.Client, error) { + if extraCertsPath == "" { + return &http.Client{}, nil + } + + log = log.With(slog.F("root_path", extraCertsPath)) + log.Debug(context.Background(), "adding certs to default pool") + pool, err := certPool(log, extraCertsPath) + if err != nil { + return nil, xerrors.Errorf("cert pool: %w", err) + } + + //nolint:forcetypeassert + transport := (http.DefaultTransport.(*http.Transport)).Clone() + + //nolint:gosec + transport.TLSClientConfig = &tls.Config{ + RootCAs: pool, + } + + return &http.Client{ + Transport: transport, + }, nil +} + +func certPool(log slog.Logger, certsPath string) (*x509.CertPool, error) { + pool, err := x509.SystemCertPool() + if err != nil { + return nil, xerrors.Errorf("system cert pool: %w", err) + } + + fi, err := os.Stat(certsPath) + if err != nil { + return nil, xerrors.Errorf("stat %v: %w", certsPath, err) + } + + if !fi.IsDir() { + err = addCert(log, pool, certsPath) + if err != nil { + return nil, xerrors.Errorf("add cert: %w", err) + } + return pool, nil + } + + entries, err := os.ReadDir(certsPath) + if err != nil { + return nil, xerrors.Errorf("read dir %v: %w", certsPath, err) + } + + for _, entry := range entries { + path := filepath.Join(certsPath, entry.Name()) + err = addCert(log, pool, path) + if err != nil { + return nil, xerrors.Errorf("add cert: %w", err) + } + } + + return pool, nil +} + +func addCert(log slog.Logger, pool *x509.CertPool, path string) error { + b, err := os.ReadFile(path) + if err != nil { + return xerrors.Errorf("read file: %w", err) + } + + if !pool.AppendCertsFromPEM(b) { + log.Error(context.Background(), "failed to append cert", + slog.F("filepath", path)) + } else { + log.Debug(context.Background(), "added cert", slog.F("path", path)) + } + return nil +}