Lowering perms to CAP_NET_ADMIN (#65)

* feat: lowering perms to CAP_NET_ADMIN

* refactor: improve cleanup

* test: improve testing

* refactor: remove dead code

* refactor: cleanup code

* refactor: cleanup code

* fix: linter

* fix: MacOS

* fix: MacOS

Author: evgeniy-scherbina

.github/workflows/ci.yml

@@ -73,6 +73,36 @@ jobs:
     - name: Download and verify dependencies
       run: make deps
 
+      # Before (default):
+      #  - /etc/resolv.conf -> /run/systemd/resolve/stub-resolv.conf
+      #  - stub-resolv.conf points to 127.0.0.53 (systemd-resolved stub listener)
+      #  - systemd-resolved forwards to the real upstream file:
+      #      /run/systemd/resolve/resolv.conf
+      #  Flow: /etc/resolv.conf -> stub-resolv.conf (127.0.0.53) -> systemd-resolved -> /run/systemd/resolve/resolv.conf
+      #
+      # After (bind-mount):
+      #  - /etc/resolv.conf is bind-mounted to /run/systemd/resolve/resolv.conf
+      #  - processes read upstream nameservers directly from /run/systemd/resolve/resolv.conf
+      #  Flow: /etc/resolv.conf -> /run/systemd/resolve/resolv.conf
+      #
+      # This makes processes talk directly to the upstream DNS servers and
+      # bypasses the systemd-resolved *stub listener* (127.0.0.53).
+      #
+      # Important nuance: systemd-resolved itself is NOT stopped; it still runs and updates
+      # /run/systemd/resolve/resolv.conf. Because this is a bind (not a copy), updates to the
+      # upstream list are visible. Trade-off: we lose the stub’s features (caching,
+      # split-DNS/VPN per-interface behavior, DNSSEC/DoT/DoH mediation, mDNS/LLMNR).
+      #
+      # Reason: network namespaces have their own network stack (interfaces, routes and loopback).
+      # A process inside a network namespace resolves 127.0.0.53 against that namespace’s loopback, not the host’s,
+      # and systemd-resolved usually listens on the host loopback. As a result the stub at 127.0.0.53 is often
+      # unreachable from an isolated namespace and DNS lookups fail.
+      # Bind-mounting /run/systemd/resolve/resolv.conf over /etc/resolv.conf forces processes to use the upstream
+      # nameservers directly, avoiding that failure.
+    - name: Change DNS configuration
+      if: runner.os == 'Linux'
+      run: sudo mount --bind /run/systemd/resolve/resolv.conf /etc/resolv.conf
+
     - name: Run unit tests
       run: make unit-test
 

Makefile

@@ -62,7 +62,7 @@ e2e-test:
 		echo "E2E tests require Linux platform. Current platform: $$(uname)"; \
 		exit 1; \
 	fi
-	sudo $(shell which go) test -v -race ./e2e_tests
+	sudo $(shell which go) test -v -race ./e2e_tests -count=1
 	@echo "✓ E2E tests passed!"
 
 # Run tests with coverage (needs sudo for E2E tests)

boundary.go

@@ -55,8 +55,8 @@ func New(ctx context.Context, config Config) (*Boundary, error) {
 }
 
 func (b *Boundary) Start() error {
-	// Start the jailer (network isolation)
-	err := b.jailer.Start()
+	// Configure the jailer (network isolation)
+	err := b.jailer.ConfigureBeforeCommandExecution()
 	if err != nil {
 		return fmt.Errorf("failed to start jailer: %v", err)
 	}
@@ -78,6 +78,10 @@ func (b *Boundary) Command(command []string) *exec.Cmd {
 	return b.jailer.Command(command)
 }
 
+func (b *Boundary) ConfigureAfterCommandExecution(processPID int) error {
+	return b.jailer.ConfigureAfterCommandExecution(processPID)
+}
+
 func (b *Boundary) Close() error {
 	// Stop proxy server
 	if b.proxyServer != nil {

cli/cli.go

@@ -3,8 +3,10 @@ package cli
 import (
 	"context"
 	"fmt"
+	"log"
 	"log/slog"
 	"os"
+	"os/exec"
 	"os/signal"
 	"path/filepath"
 	"strings"
@@ -92,8 +94,39 @@ func BaseCommand() *serpent.Command {
 	}
 }
 
+func isChild() bool {
+	return os.Getenv("CHILD") == "true"
+}
+
 // Run executes the boundary command with the given configuration and arguments
 func Run(ctx context.Context, config Config, args []string) error {
+	if isChild() {
+		log.Printf("boundary CHILD process is started")
+
+		vethNetJail := os.Getenv("VETH_JAIL_NAME")
+		err := jail.SetupChildNetworking(vethNetJail)
+		if err != nil {
+			return fmt.Errorf("failed to setup child networking: %v", err)
+		}
+		log.Printf("child networking is successfully configured")
+
+		// Program to run
+		bin := args[0]
+		args = args[1:]
+
+		cmd := exec.Command(bin, args...)
+		cmd.Stdin = os.Stdin
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		err = cmd.Run()
+		if err != nil {
+			log.Printf("failed to run %s: %v", bin, err)
+			return err
+		}
+
+		return nil
+	}
+
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
@@ -191,15 +224,26 @@ func Run(ctx context.Context, config Config, args []string) error {
 	// Execute command in boundary
 	go func() {
 		defer cancel()
-		cmd := boundaryInstance.Command(args)
-		cmd.Stderr = os.Stderr
-		cmd.Stdout = os.Stdout
-		cmd.Stdin = os.Stdin
+		cmd := boundaryInstance.Command(os.Args)
+
+		logger.Debug("Executing command in boundary", "command", strings.Join(os.Args, " "))
+		err := cmd.Start()
+		if err != nil {
+			logger.Error("Command failed to start", "error", err)
+			return
+		}
+
+		err = boundaryInstance.ConfigureAfterCommandExecution(cmd.Process.Pid)
+		if err != nil {
+			logger.Error("configuration after command execution failed", "error", err)
+			return
+		}
 
-		logger.Debug("Executing command in boundary", "command", strings.Join(args, " "))
-		err := cmd.Run()
+		logger.Debug("waiting on a child process to finish")
+		err = cmd.Wait()
 		if err != nil {
 			logger.Error("Command execution failed", "error", err)
+			return
 		}
 	}()
 

e2e_tests/boundary_integration_test.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -37,29 +38,15 @@ func findProjectRoot(t *testing.T) string {
 	}
 }
 
-// getNamespaceName gets the single network namespace name
-// Fails if there are 0 or multiple namespaces
-func getNamespaceName(t *testing.T) string {
-	cmd := exec.Command("ip", "netns", "list")
+func getChildProcessPID(t *testing.T) int {
+	cmd := exec.Command("pgrep", "-f", "boundary-test", "-n")
 	output, err := cmd.Output()
-	require.NoError(t, err, "Failed to list network namespaces")
-
-	lines := strings.Split(string(output), "\n")
-	var namespaces []string
-
-	for _, line := range lines {
-		line = strings.TrimSpace(line)
-		if line != "" {
-			// Extract namespace name (first field)
-			parts := strings.Fields(line)
-			if len(parts) > 0 {
-				namespaces = append(namespaces, parts[0])
-			}
-		}
-	}
+	require.NoError(t, err)
 
-	require.Len(t, namespaces, 1, "Expected exactly one network namespace, found %d: %v", len(namespaces), namespaces)
-	return namespaces[0]
+	pidStr := strings.TrimSpace(string(output))
+	pid, err := strconv.Atoi(pidStr)
+	require.NoError(t, err)
+	return pid
 }
 
 func TestBoundaryIntegration(t *testing.T) {
@@ -73,18 +60,18 @@ func TestBoundaryIntegration(t *testing.T) {
 	require.NoError(t, err, "Failed to build boundary binary")
 
 	// Create context for boundary process
-	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
 	// Start boundary process with sudo
 	boundaryCmd := exec.CommandContext(ctx, "/tmp/boundary-test",
 		"--allow", "dev.coder.com",
 		"--allow", "jsonplaceholder.typicode.com",
 		"--log-level", "debug",
-		"--", "bash", "-c", "sleep 10 && echo 'Test completed'")
+		"--", "/bin/bash", "-c", "/usr/bin/sleep 10 && /usr/bin/echo 'Test completed'")
 
-	// Suppress output to prevent terminal corruption
-	boundaryCmd.Stdout = os.Stdout // Let it go to /dev/null
+	boundaryCmd.Stdin = os.Stdin
+	boundaryCmd.Stdout = os.Stdout
 	boundaryCmd.Stderr = os.Stderr
 
 	// Start the process
@@ -94,13 +81,13 @@ func TestBoundaryIntegration(t *testing.T) {
 	// Give boundary time to start
 	time.Sleep(2 * time.Second)
 
-	// Get the namespace name that boundary created
-	namespaceName := getNamespaceName(t)
+	pidInt := getChildProcessPID(t)
+	pid := fmt.Sprintf("%v", pidInt)
 
 	// Test HTTP request through boundary (from inside the jail)
 	t.Run("HTTPRequestThroughBoundary", func(t *testing.T) {
 		// Run curl directly in the namespace using ip netns exec
-		curlCmd := exec.Command("sudo", "ip", "netns", "exec", namespaceName,
+		curlCmd := exec.Command("sudo", "nsenter", "-t", pid, "-n", "--",
 			"curl", "http://jsonplaceholder.typicode.com/todos/1")
 
 		// Capture stderr separately
@@ -128,7 +115,7 @@ func TestBoundaryIntegration(t *testing.T) {
 		certPath := fmt.Sprintf("%v/ca-cert.pem", configDir)
 
 		// Run curl directly in the namespace using ip netns exec
-		curlCmd := exec.Command("sudo", "ip", "netns", "exec", namespaceName,
+		curlCmd := exec.Command("sudo", "sudo", "nsenter", "-t", pid, "-n", "--",
 			"env", fmt.Sprintf("SSL_CERT_FILE=%v", certPath), "curl", "-s", "https://dev.coder.com/api/v2")
 
 		// Capture stderr separately
@@ -149,7 +136,7 @@ func TestBoundaryIntegration(t *testing.T) {
 	// Test blocked domain (from inside the jail)
 	t.Run("BlockedDomainTest", func(t *testing.T) {
 		// Run curl directly in the namespace using ip netns exec
-		curlCmd := exec.Command("sudo", "ip", "netns", "exec", namespaceName,
+		curlCmd := exec.Command("sudo", "sudo", "nsenter", "-t", pid, "-n", "--",
 			"curl", "-s", "http://example.com")
 
 		// Capture stderr separately
@@ -163,6 +150,11 @@ func TestBoundaryIntegration(t *testing.T) {
 		require.Contains(t, string(output), "Request Blocked by Boundary")
 	})
 
+	// Gracefully close process, call cleanup methods
+	err = boundaryCmd.Process.Signal(os.Interrupt)
+	require.NoError(t, err, "Failed to interrupt boundary process")
+	time.Sleep(time.Second * 1)
+
 	// Clean up
 	cancel()                 // This will terminate the boundary process
 	err = boundaryCmd.Wait() // Wait for process to finish

e2e_tests/iptables_cleanup_test.go

@@ -0,0 +1,93 @@
+package e2e_tests
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	filterTable = "filter"
+	natTable    = "nat"
+)
+
+func getIptablesRules(tableName string) (string, error) {
+	cmd := exec.Command("sudo", "iptables", "-L", "-n", "-t", tableName)
+	output, err := cmd.Output()
+	if err != nil {
+		return "", fmt.Errorf("failed to get iptables rules: %v", err)
+	}
+	rules := string(output)
+
+	return rules, nil
+}
+
+func TestIPTablesCleanup(t *testing.T) {
+	// Step 1: Capture initial iptables rules
+	initialFilterRules, err := getIptablesRules(filterTable)
+	require.NoError(t, err)
+	initialNatRules, err := getIptablesRules(natTable)
+	require.NoError(t, err)
+
+	// Step 2: Run Boundary
+	// Find project root by looking for go.mod file
+	projectRoot := findProjectRoot(t)
+
+	// Build the boundary binary
+	buildCmd := exec.Command("go", "build", "-o", "/tmp/boundary-test", "./cmd/...")
+	buildCmd.Dir = projectRoot
+	err = buildCmd.Run()
+	require.NoError(t, err, "Failed to build boundary binary")
+
+	// Create context for boundary process
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// Start boundary process with sudo
+	boundaryCmd := exec.CommandContext(ctx, "/tmp/boundary-test",
+		"--allow", "dev.coder.com",
+		"--allow", "jsonplaceholder.typicode.com",
+		"--log-level", "debug",
+		"--", "/bin/bash", "-c", "/usr/bin/sleep 10 && /usr/bin/echo 'Test completed'")
+
+	boundaryCmd.Stdin = os.Stdin
+	boundaryCmd.Stdout = os.Stdout
+	boundaryCmd.Stderr = os.Stderr
+
+	// Start the process
+	err = boundaryCmd.Start()
+	require.NoError(t, err, "Failed to start boundary process")
+
+	// Give boundary time to start
+	time.Sleep(2 * time.Second)
+
+	// Gracefully close process, call cleanup methods
+	err = boundaryCmd.Process.Signal(os.Interrupt)
+	require.NoError(t, err, "Failed to interrupt boundary process")
+	time.Sleep(time.Second * 1)
+
+	// Step 3: Clean up
+	cancel()                 // This will terminate the boundary process
+	err = boundaryCmd.Wait() // Wait for process to finish
+	if err != nil {
+		t.Logf("Boundary process finished with error: %v", err)
+	}
+
+	// Clean up binary
+	err = os.Remove("/tmp/boundary-test")
+	require.NoError(t, err, "Failed to remove /tmp/boundary-test")
+
+	// Step 4: Capture iptables rules after boundary has executed
+	filterRules, err := getIptablesRules(filterTable)
+	require.NoError(t, err)
+	natRules, err := getIptablesRules(natTable)
+	require.NoError(t, err)
+
+	require.Equal(t, initialFilterRules, filterRules)
+	require.Equal(t, initialNatRules, natRules)
+}

jail/jail.go

@@ -8,8 +8,9 @@ import (
 )
 
 type Jailer interface {
-	Start() error
+	ConfigureBeforeCommandExecution() error
 	Command(command []string) *exec.Cmd
+	ConfigureAfterCommandExecution(processPID int) error
 	Close() error
 }