From 16766ae20559346b25691c6e7cfc346ee3a99fe3 Mon Sep 17 00:00:00 2001 From: Tom Mychost Date: Tue, 14 Nov 2023 17:22:14 +0000 Subject: [PATCH 1/5] Add DERP generate_204 endpoint for captive portal detection. --- hscontrol/app.go | 1 + hscontrol/derp/server/derp_server.go | 41 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/hscontrol/app.go b/hscontrol/app.go index 78b72bf51f..34e8bbe298 100644 --- a/hscontrol/app.go +++ b/hscontrol/app.go @@ -469,6 +469,7 @@ func (h *Headscale) createRouter(grpcMux *grpcRuntime.ServeMux) *mux.Router { router.HandleFunc("/derp", h.DERPServer.DERPHandler) router.HandleFunc("/derp/probe", derpServer.DERPProbeHandler) router.HandleFunc("/bootstrap-dns", derpServer.DERPBootstrapDNSHandler(h.DERPMap)) + router.HandleFunc("/generate_204", derpServer.DERPNoContextHandler) } apiRouter := router.PathPrefix("/api").Subrouter() diff --git a/hscontrol/derp/server/derp_server.go b/hscontrol/derp/server/derp_server.go index 52a63e9fd3..e1b127080c 100644 --- a/hscontrol/derp/server/derp_server.go +++ b/hscontrol/derp/server/derp_server.go @@ -26,6 +26,10 @@ import ( // headers and it will begin writing & reading the DERP protocol immediately // following its HTTP request. const fastStartHeader = "Derp-Fast-Start" +const ( + noContentChallengeHeader = "X-Tailscale-Challenge" + noContentResponseHeader = "X-Tailscale-Response" +) type DERPServer struct { serverURL string @@ -204,6 +208,43 @@ func DERPProbeHandler( } } +// DERPNoContextHandler is the endpoint clients use to determine if they are behind a captive portal +// Clients challenge this with the X-Tailscale-Challenge header and expect the challenge value within X-Tailscale-Response +// https://github.com/tailscale/tailscale/blob/955e2fcbfb4fe7ff9b8dbd665ba24ef2008c676e/cmd/derper/derper.go#L324 +func DERPNoContextHandler( + writer http.ResponseWriter, + req *http.Request, +) { + switch req.Method { + case http.MethodHead, http.MethodGet: + if challenge := req.Header.Get(noContentChallengeHeader); challenge != "" { + badChar := strings.IndexFunc(challenge, func(r rune) bool { + return !isChallengeChar(r) + }) != -1 + if len(challenge) <= 64 && !badChar { + writer.Header().Set(noContentResponseHeader, "response "+challenge) + } + } + writer.WriteHeader(http.StatusNoContent) + default: + writer.WriteHeader(http.StatusMethodNotAllowed) + _, err := writer.Write([]byte("bogus captive portal method")) + if err != nil { + log.Error(). + Caller(). + Err(err). + Msg("Failed to write response") + } + } +} + +func isChallengeChar(c rune) bool { + // Semi-randomly chosen as a limited set of valid characters + return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || + ('0' <= c && c <= '9') || + c == '.' || c == '-' || c == '_' +} + // DERPBootstrapDNSHandler implements the /bootsrap-dns endpoint // Described in https://github.com/tailscale/tailscale/issues/1405, // this endpoint provides a way to help a client when it fails to start up From ed1671e01c81a03244b960df79dfe8ae8a56ce14 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Thu, 15 Feb 2024 17:49:14 +0100 Subject: [PATCH 2/5] initial work on integration tests Signed-off-by: Kristoffer Dalby --- integration/embedded_derp_test.go | 74 +++++++++++++++++++++++++++++++ integration/hsic/hsic.go | 2 + 2 files changed, 76 insertions(+) diff --git a/integration/embedded_derp_test.go b/integration/embedded_derp_test.go index 15ab7addb7..82b77e2f5e 100644 --- a/integration/embedded_derp_test.go +++ b/integration/embedded_derp_test.go @@ -1,9 +1,11 @@ package integration import ( + "encoding/json" "fmt" "log" "net/url" + "strings" "testing" "github.com/juanfont/headscale/hscontrol/util" @@ -11,6 +13,7 @@ import ( "github.com/juanfont/headscale/integration/hsic" "github.com/juanfont/headscale/integration/tsic" "github.com/ory/dockertest/v3" + "tailscale.com/ipn/ipnstate" ) type EmbeddedDERPServerScenario struct { @@ -80,6 +83,77 @@ func TestDERPServerScenario(t *testing.T) { t.Logf("%d successful pings out of %d", success, len(allClients)*len(allIps)) } +func TestDERPValidateEmbedded(t *testing.T) { + IntegrationSkip(t) + + scenario, err := NewScenario() + assertNoErr(t, err) + defer scenario.Shutdown() + + spec := map[string]int{ + "user1": 1, + } + + headscaleConfig := map[string]string{ + "HEADSCALE_DERP_URLS": "", + "HEADSCALE_DERP_SERVER_ENABLED": "true", + "HEADSCALE_DERP_SERVER_REGION_ID": "999", + "HEADSCALE_DERP_SERVER_REGION_CODE": "headscale", + "HEADSCALE_DERP_SERVER_REGION_NAME": "Headscale Embedded DERP", + "HEADSCALE_DERP_SERVER_STUN_LISTEN_ADDR": "0.0.0.0:3478", + "HEADSCALE_DERP_SERVER_PRIVATE_KEY_PATH": "/tmp/derp.key", + + // Envknob for enabling DERP debug logs + "DERP_DEBUG_LOGS": "true", + "DERP_PROBER_DEBUG_LOGS": "true", + } + + err = scenario.CreateHeadscaleEnv( + spec, + []tsic.Option{}, + hsic.WithConfigEnv(headscaleConfig), + hsic.WithTestName("derpvalidate"), + hsic.WithExtraPorts([]string{"3478/udp"}), + hsic.WithTLS(), + hsic.WithHostnameAsServerURL(), + hsic.WithPort(80), + ) + assertNoErrHeadscaleEnv(t, err) + + allClients, err := scenario.ListTailscaleClients() + assertNoErrListClients(t, err) + + err = scenario.WaitForTailscaleSync() + assertNoErrSync(t, err) + + assertClientsState(t, allClients) + + if len(allClients) != 1 { + t.Fatalf("expected 1 client, got: %d", len(allClients)) + } + + client := allClients[0] + + var derpReport ipnstate.DebugDERPRegionReport + stdout, stderr, err := client.Execute([]string{"tailscale", "debug", "derp", "999"}) + if err != nil { + t.Fatalf("executing debug derp report, stderr: %s, err: %s", stderr, err) + } + + t.Logf("DERP report: \n%s", stdout) + + err = json.Unmarshal([]byte(stdout), &derpReport) + if err != nil { + t.Fatalf("unmarshalling debug derp report, content: %s, err: %s", stdout, err) + } + + for _, warn := range derpReport.Warnings { + if strings.Contains(warn, "captive portal check") { + t.Errorf("derp report contains warning about portal check, generate_204 endpoint not working") + } + } +} + func (s *EmbeddedDERPServerScenario) CreateHeadscaleEnv( users map[string]int, opts ...hsic.Option, diff --git a/integration/hsic/hsic.go b/integration/hsic/hsic.go index 5019895a3c..43c83d2661 100644 --- a/integration/hsic/hsic.go +++ b/integration/hsic/hsic.go @@ -112,6 +112,8 @@ func WithConfigEnv(configEnv map[string]string) Option { // WithPort sets the port on where to run Headscale. func WithPort(port int) Option { return func(hsic *HeadscaleInContainer) { + hsic.env["HEADSCALE_LISTEN_ADDR"] = fmt.Sprintf("0.0.0.0:%d", port) + hsic.env["HEADSCALE_SERVER_URL"] = fmt.Sprintf("http://headscale:%d", port) hsic.port = port } } From afa064ff613e6e0da8ad933c8e9e0729ba394663 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Thu, 15 Feb 2024 17:50:09 +0100 Subject: [PATCH 3/5] add gh for new testcase Signed-off-by: Kristoffer Dalby --- ...tegration-v2-TestDERPValidateEmbedded.yaml | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/test-integration-v2-TestDERPValidateEmbedded.yaml diff --git a/.github/workflows/test-integration-v2-TestDERPValidateEmbedded.yaml b/.github/workflows/test-integration-v2-TestDERPValidateEmbedded.yaml new file mode 100644 index 0000000000..8fa516bd87 --- /dev/null +++ b/.github/workflows/test-integration-v2-TestDERPValidateEmbedded.yaml @@ -0,0 +1,67 @@ +# DO NOT EDIT, generated with cmd/gh-action-integration-generator/main.go +# To regenerate, run "go generate" in cmd/gh-action-integration-generator/ + +name: Integration Test v2 - TestDERPValidateEmbedded + +on: [pull_request] + +concurrency: + group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + TestDERPValidateEmbedded: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main + - uses: satackey/action-docker-layer-caching@main + continue-on-error: true + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v34 + with: + files: | + *.nix + go.* + **/*.go + integration_test/ + config-example.yaml + + - name: Run TestDERPValidateEmbedded + uses: Wandalen/wretry.action@master + if: steps.changed-files.outputs.any_changed == 'true' + with: + attempt_limit: 5 + command: | + nix develop --command -- docker run \ + --tty --rm \ + --volume ~/.cache/hs-integration-go:/go \ + --name headscale-test-suite \ + --volume $PWD:$PWD -w $PWD/integration \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume $PWD/control_logs:/tmp/control \ + golang:1 \ + go run gotest.tools/gotestsum@latest -- ./... \ + -failfast \ + -timeout 120m \ + -parallel 1 \ + -run "^TestDERPValidateEmbedded$" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: logs + path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" From 61fe21d710acf51041ab1f0bc6d0c0c974d0cef3 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 16 Feb 2024 14:14:35 +0100 Subject: [PATCH 4/5] listen for generate_204 on 80 Signed-off-by: Kristoffer Dalby --- hscontrol/app.go | 31 ++++++++++++++++++++++++++++++- integration/embedded_derp_test.go | 15 ++++++++++----- integration/hsic/hsic.go | 2 +- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/hscontrol/app.go b/hscontrol/app.go index 34e8bbe298..371f5bf142 100644 --- a/hscontrol/app.go +++ b/hscontrol/app.go @@ -469,7 +469,11 @@ func (h *Headscale) createRouter(grpcMux *grpcRuntime.ServeMux) *mux.Router { router.HandleFunc("/derp", h.DERPServer.DERPHandler) router.HandleFunc("/derp/probe", derpServer.DERPProbeHandler) router.HandleFunc("/bootstrap-dns", derpServer.DERPBootstrapDNSHandler(h.DERPMap)) - router.HandleFunc("/generate_204", derpServer.DERPNoContextHandler) + + // Only add to main muxer if running on port 80 + if strings.HasSuffix(h.cfg.Addr, ":80") { + router.HandleFunc("/generate_204", derpServer.DERPNoContextHandler) + } } apiRouter := router.PathPrefix("/api").Subrouter() @@ -697,6 +701,31 @@ func (h *Headscale) Serve() error { log.Info(). Msgf("listening and serving HTTP on: %s", h.cfg.Addr) + // If headscale is not listening on port 80 and embedded DERP server + // is enabled, run a small http endpoint for generate204. + // This is not configurable as captive portal busting requires http/80. + if h.cfg.DERP.ServerEnabled || !strings.HasSuffix(h.cfg.Addr, ":80") { + httpDerpMux := http.NewServeMux() + httpDerpMux.HandleFunc("/generate_204", derpServer.DERPNoContextHandler) + + addr := "0.0.0.0:80" + httpDerpServer := &http.Server{ + Addr: addr, + Handler: httpDerpMux, + ReadTimeout: types.HTTPReadTimeout, + } + + httpDerpListener, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("binding port 80 for DERP HTTP endpoint: %w", err) + } + + errorGroup.Go(func() error { return httpDerpServer.Serve(httpDerpListener) }) + + log.Info(). + Msgf("listening and serving HTTP DERP generate_204 on: %s", addr) + } + promMux := http.NewServeMux() promMux.Handle("/metrics", promhttp.Handler()) diff --git a/integration/embedded_derp_test.go b/integration/embedded_derp_test.go index 82b77e2f5e..bb4308dec8 100644 --- a/integration/embedded_derp_test.go +++ b/integration/embedded_derp_test.go @@ -58,7 +58,7 @@ func TestDERPServerScenario(t *testing.T) { spec, hsic.WithConfigEnv(headscaleConfig), hsic.WithTestName("derpserver"), - hsic.WithExtraPorts([]string{"3478/udp"}), + hsic.WithExtraPorts("3478/udp"), hsic.WithTLS(), hsic.WithHostnameAsServerURL(), ) @@ -88,7 +88,7 @@ func TestDERPValidateEmbedded(t *testing.T) { scenario, err := NewScenario() assertNoErr(t, err) - defer scenario.Shutdown() + // defer scenario.Shutdown() spec := map[string]int{ "user1": 1, @@ -103,6 +103,10 @@ func TestDERPValidateEmbedded(t *testing.T) { "HEADSCALE_DERP_SERVER_STUN_LISTEN_ADDR": "0.0.0.0:3478", "HEADSCALE_DERP_SERVER_PRIVATE_KEY_PATH": "/tmp/derp.key", + // Magic DNS breaks the docker DNS system which means + // DERP cannot look up the DERP server for some things. + "HEADSCALE_DNS_CONFIG_MAGIC_DNS": "0", + // Envknob for enabling DERP debug logs "DERP_DEBUG_LOGS": "true", "DERP_PROBER_DEBUG_LOGS": "true", @@ -113,10 +117,9 @@ func TestDERPValidateEmbedded(t *testing.T) { []tsic.Option{}, hsic.WithConfigEnv(headscaleConfig), hsic.WithTestName("derpvalidate"), - hsic.WithExtraPorts([]string{"3478/udp"}), + hsic.WithExtraPorts("3478/udp", "80/tcp"), hsic.WithTLS(), hsic.WithHostnameAsServerURL(), - hsic.WithPort(80), ) assertNoErrHeadscaleEnv(t, err) @@ -149,7 +152,9 @@ func TestDERPValidateEmbedded(t *testing.T) { for _, warn := range derpReport.Warnings { if strings.Contains(warn, "captive portal check") { - t.Errorf("derp report contains warning about portal check, generate_204 endpoint not working") + t.Errorf( + "derp report contains warning about portal check, generate_204 endpoint not working", + ) } } } diff --git a/integration/hsic/hsic.go b/integration/hsic/hsic.go index 43c83d2661..0bdf031e3f 100644 --- a/integration/hsic/hsic.go +++ b/integration/hsic/hsic.go @@ -119,7 +119,7 @@ func WithPort(port int) Option { } // WithExtraPorts exposes additional ports on the container (e.g. 3478/udp for STUN). -func WithExtraPorts(ports []string) Option { +func WithExtraPorts(ports ...string) Option { return func(hsic *HeadscaleInContainer) { hsic.extraPorts = ports } From a0fd4e3277ca6fb24b15ec5514494f10d6dd1857 Mon Sep 17 00:00:00 2001 From: TotoTheDragon Date: Fri, 16 Feb 2024 16:43:58 +0100 Subject: [PATCH 5/5] Remove DNS overwrites --- integration/tsic/tsic.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index 854d5a71f1..490e621945 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -202,13 +202,6 @@ func New( ExtraHosts: tsic.withExtraHosts, } - if tsic.headscaleHostname != "" { - tailscaleOptions.ExtraHosts = []string{ - "host.docker.internal:host-gateway", - fmt.Sprintf("%s:host-gateway", tsic.headscaleHostname), - } - } - if tsic.workdir != "" { tailscaleOptions.WorkingDir = tsic.workdir }