Merge pull request #125 from fosrl/dev

1.6.0

.github/ISSUE_TEMPLATE/1.bug_report.yml

@@ -14,12 +14,13 @@ body:
           label: Environment
           description: Please fill out the relevant details below for your environment.
           value: |
-              - OS Type & Version: (e.g., Ubuntu 22.04)
+              - OS Type & Version:
               - Pangolin Version:
+              - Edition (Community or Enterprise):
               - Gerbil Version:
               - Traefik Version:
               - Newt Version:
-              - Olm Version: (if applicable)
+              - Client Version:
       validations:
           required: true
 

dns/override/dns_override_android.go

@@ -20,3 +20,9 @@ func CleanupStaleState(interfaceName string) error {
 	_ = interfaceName
 	return nil
 }
+
+// ForceResetDNS is a no-op on Android.
+func ForceResetDNS(interfaceName string) error {
+	_ = interfaceName
+	return nil
+}

dns/override/dns_override_darwin.go

@@ -15,6 +15,13 @@ var configurator platform.DNSConfigurator
 // SetupDNSOverride configures the system DNS to use the DNS proxy on macOS
 // Uses scutil for DNS configuration
 func SetupDNSOverride(interfaceName string, proxyIp netip.Addr) error {
+	// Defensively clear any stale DNS state from a previous unclean shutdown
+	// before installing the new override. This makes a second tunnel start
+	// safe even if the previous client crashed without restoring DNS.
+	if err := CleanupStaleState(interfaceName); err != nil {
+		logger.Warn("Pre-setup stale DNS cleanup failed (continuing): %v", err)
+	}
+
 	var err error
 	configurator, err = platform.NewDarwinDNSConfigurator()
 	if err != nil {
@@ -78,3 +85,31 @@ func CleanupStaleState(interfaceName string) error {
 	logger.Info("Stale DNS state cleanup completed successfully")
 	return nil
 }
+
+// ForceResetDNS forcibly clears any DNS override state, whether or not the
+// current process installed it. This is intended for the "reset-dns" CLI
+// command and for the watchdog process to recover from a stuck override
+// left behind by a crashed client.
+func ForceResetDNS(interfaceName string) error {
+	logger.Info("Forcing DNS reset on Darwin (interface=%s)", interfaceName)
+
+	// First clean up any persisted state from a previous session.
+	cleanupErr := CleanupStaleState(interfaceName)
+
+	// Then, if the current process happens to hold a live configurator,
+	// instruct it to restore DNS as well so in-memory state is consistent.
+	if configurator != nil {
+		if err := configurator.RestoreDNS(); err != nil {
+			logger.Warn("ForceResetDNS: in-memory restore failed: %v", err)
+		}
+		configurator = nil
+	}
+
+	// As a last-resort defense, sweep any scutil keys matching our naming
+	// convention even if no state file exists.
+	if err := platform.SweepOlmScutilKeys(); err != nil {
+		logger.Warn("ForceResetDNS: scutil sweep failed: %v", err)
+	}
+
+	return cleanupErr
+}

dns/override/dns_override_ios.go

@@ -19,3 +19,9 @@ func CleanupStaleState(interfaceName string) error {
 	_ = interfaceName
 	return nil
 }
+
+// ForceResetDNS is a no-op on iOS.
+func ForceResetDNS(interfaceName string) error {
+	_ = interfaceName
+	return nil
+}

dns/override/dns_override_unix.go

@@ -15,6 +15,13 @@ var configurator platform.DNSConfigurator
 // SetupDNSOverride configures the system DNS to use the DNS proxy on Linux/FreeBSD
 // Detects the DNS manager by reading /etc/resolv.conf and verifying runtime availability
 func SetupDNSOverride(interfaceName string, proxyIp netip.Addr) error {
+	// Defensively clear any stale DNS state from a previous unclean shutdown
+	// before installing the new override. This makes a second tunnel start
+	// safe even if the previous client crashed without restoring DNS.
+	if err := CleanupStaleState(interfaceName); err != nil {
+		logger.Warn("Pre-setup stale DNS cleanup failed (continuing): %v", err)
+	}
+
 	var err error
 
 	// Detect which DNS manager is in use by checking /etc/resolv.conf and runtime availability
@@ -144,3 +151,25 @@ func CleanupStaleState(interfaceName string) error {
 	logger.Info("Stale DNS state cleanup completed successfully")
 	return nil
 }
+
+// ForceResetDNS forcibly clears any DNS override state, whether or not the
+// current process installed it. This is intended for the "reset-dns" CLI
+// command and for the watchdog process to recover from a stuck override
+// left behind by a crashed client.
+func ForceResetDNS(interfaceName string) error {
+	logger.Info("Forcing DNS reset on Linux/FreeBSD (interface=%s)", interfaceName)
+
+	// First clean up any persisted state from a previous session.
+	cleanupErr := CleanupStaleState(interfaceName)
+
+	// Then, if the current process happens to hold a live configurator,
+	// instruct it to restore DNS as well so in-memory state is consistent.
+	if configurator != nil {
+		if err := configurator.RestoreDNS(); err != nil {
+			logger.Warn("ForceResetDNS: in-memory restore failed: %v", err)
+		}
+		configurator = nil
+	}
+
+	return cleanupErr
+}

dns/override/dns_override_windows.go

@@ -15,6 +15,12 @@ var configurator platform.DNSConfigurator
 // SetupDNSOverride configures the system DNS to use the DNS proxy on Windows
 // Uses registry-based configuration (automatically extracts interface GUID)
 func SetupDNSOverride(interfaceName string, proxyIp netip.Addr) error {
+	// Defensively clear any stale DNS state from a previous unclean shutdown
+	// before installing the new override.
+	if err := CleanupStaleState(interfaceName); err != nil {
+		logger.Warn("Pre-setup stale DNS cleanup failed (continuing): %v", err)
+	}
+
 	var err error
 	configurator, err = platform.NewWindowsDNSConfigurator(interfaceName)
 	if err != nil {
@@ -77,3 +83,17 @@ func CleanupStaleState(interfaceName string) error {
 	logger.Debug("Windows DNS cleanup: no stale state to clean (interface-specific)")
 	return nil
 }
+
+// ForceResetDNS forcibly clears any DNS override state. On Windows this is
+// largely a no-op because the registry override is tied to the interface
+// GUID and is reclaimed when the interface is torn down.
+func ForceResetDNS(interfaceName string) error {
+	logger.Info("Forcing DNS reset on Windows (interface=%s)", interfaceName)
+	if configurator != nil {
+		if err := configurator.RestoreDNS(); err != nil {
+			logger.Warn("ForceResetDNS: in-memory restore failed: %v", err)
+		}
+		configurator = nil
+	}
+	return CleanupStaleState(interfaceName)
+}

dns/override/watchdog.go

@@ -0,0 +1,136 @@
+package olm
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/fosrl/newt/logger"
+)
+
+// WatchdogConfig configures the DNS override watchdog. The watchdog runs as
+// an external process (spawned via SpawnWatchdog) and monitors a parent olm
+// process. When the parent appears to have died without restoring DNS, the
+// watchdog forcibly resets the system DNS configuration.
+type WatchdogConfig struct {
+	// ParentPID is the PID of the olm process that installed the DNS
+	// override. The watchdog exits when this PID is no longer alive.
+	ParentPID int
+
+	// SocketPath is the path to the olm Unix domain socket (or named pipe
+	// on Windows). The watchdog uses it as a secondary liveness signal.
+	// May be empty if no socket-based API is enabled.
+	SocketPath string
+
+	// InterfaceName is the name of the WireGuard interface whose DNS
+	// override should be reset on parent death.
+	InterfaceName string
+
+	// CheckInterval is how often to poll the parent's liveness.
+	// Defaults to 5 seconds when zero.
+	CheckInterval time.Duration
+
+	// FailureThreshold is the number of consecutive failed liveness checks
+	// before the watchdog declares the parent dead and resets DNS.
+	// Defaults to 3 when zero.
+	FailureThreshold int
+}
+
+// RunWatchdog runs the watchdog loop in the current process until either
+// (a) the parent dies and DNS is reset, or (b) ctx is cancelled.
+func RunWatchdog(ctx context.Context, cfg WatchdogConfig) error {
+	if cfg.ParentPID <= 0 {
+		return fmt.Errorf("watchdog: invalid parent PID %d", cfg.ParentPID)
+	}
+	if cfg.CheckInterval <= 0 {
+		cfg.CheckInterval = 5 * time.Second
+	}
+	if cfg.FailureThreshold <= 0 {
+		cfg.FailureThreshold = 3
+	}
+
+	logger.Info("DNS watchdog started: parent=%d interval=%s threshold=%d socket=%q interface=%q",
+		cfg.ParentPID, cfg.CheckInterval, cfg.FailureThreshold, cfg.SocketPath, cfg.InterfaceName)
+
+	ticker := time.NewTicker(cfg.CheckInterval)
+	defer ticker.Stop()
+
+	consecutiveFailures := 0
+
+	for {
+		select {
+		case <-ctx.Done():
+			logger.Info("DNS watchdog context cancelled, exiting cleanly")
+			return ctx.Err()
+		case <-ticker.C:
+		}
+
+		alive := isParentAlive(cfg.ParentPID, cfg.SocketPath)
+		if alive {
+			if consecutiveFailures > 0 {
+				logger.Debug("DNS watchdog: parent recovered after %d failures", consecutiveFailures)
+			}
+			consecutiveFailures = 0
+			continue
+		}
+
+		consecutiveFailures++
+		logger.Warn("DNS watchdog: parent liveness check failed (%d/%d)",
+			consecutiveFailures, cfg.FailureThreshold)
+
+		if consecutiveFailures >= cfg.FailureThreshold {
+			logger.Warn("DNS watchdog: parent declared dead, forcing DNS reset")
+			if err := ForceResetDNS(cfg.InterfaceName); err != nil {
+				logger.Error("DNS watchdog: ForceResetDNS failed: %v", err)
+				return err
+			}
+			logger.Info("DNS watchdog: DNS reset complete, exiting")
+			return nil
+		}
+	}
+}
+
+// isParentAlive returns true if the parent process appears to be alive. It
+// considers the parent alive if EITHER the PID is still running OR the
+// socket-based health endpoint responds. This dual check avoids false
+// positives where one signal is flaky (e.g., socket blocked but process
+// still recovering).
+func isParentAlive(pid int, socketPath string) bool {
+	if pidAlive(pid) {
+		return true
+	}
+	// Process is gone; double-check via socket to avoid races where PID
+	// recycling or signal-0 quirks lie to us. Socket should already be
+	// gone too.
+	if socketPath != "" && socketHealthy(socketPath) {
+		return true
+	}
+	return false
+}
+
+// socketHealthy attempts a fast /health request over the unix socket.
+func socketHealthy(socketPath string) bool {
+	if _, err := os.Stat(socketPath); err != nil {
+		return false
+	}
+
+	client := &http.Client{
+		Timeout: 2 * time.Second,
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+				d := net.Dialer{Timeout: 2 * time.Second}
+				return d.DialContext(ctx, "unix", socketPath)
+			},
+		},
+	}
+
+	resp, err := client.Get("http://localhost/health")
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+	return resp.StatusCode == http.StatusOK
+}

dns/override/watchdog_spawn_unix.go

@@ -0,0 +1,119 @@
+//go:build !windows
+
+package olm
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strconv"
+	"syscall"
+
+	"github.com/fosrl/newt/logger"
+)
+
+// SpawnWatchdogConfig captures the inputs needed to launch the external
+// watchdog subprocess that monitors the calling olm process and forces a
+// DNS reset if the parent dies before restoring DNS.
+type SpawnWatchdogConfig struct {
+	// Executable is the path to the binary that will host the watchdog
+	// (typically os.Executable()). The binary must understand the
+	// watchdog subcommand layout described below.
+	Executable string
+
+	// Subcommand is the argv prefix the binary uses to enter watchdog
+	// mode (e.g., []string{"watchdog"} or []string{"dns", "watchdog"}).
+	Subcommand []string
+
+	// InterfaceName is the WireGuard interface whose DNS override should
+	// be reset if the parent dies.
+	InterfaceName string
+
+	// SocketPath is the parent's olm API socket path (may be empty).
+	SocketPath string
+
+	// LogFile, if non-empty, is the path the watchdog writes its stdout
+	// and stderr to. If empty, /dev/null is used.
+	LogFile string
+}
+
+// SpawnWatchdog launches the watchdog subprocess in a detached process group
+// so that it survives the death of the parent. The returned *exec.Cmd is the
+// handle the parent should call StopWatchdog on during clean shutdown.
+//
+// The spawned process is invoked as:
+//
+//	<Executable> <Subcommand...> --parent-pid=<ppid> \
+//	    --interface=<InterfaceName> [--socket=<SocketPath>]
+//
+// Both pangolin (cli) and olm should map their watchdog subcommand to
+// RunWatchdog.
+func SpawnWatchdog(cfg SpawnWatchdogConfig) (*exec.Cmd, error) {
+	if cfg.Executable == "" {
+		return nil, fmt.Errorf("watchdog: executable is required")
+	}
+	if len(cfg.Subcommand) == 0 {
+		return nil, fmt.Errorf("watchdog: subcommand is required")
+	}
+
+	args := append([]string{}, cfg.Subcommand...)
+	args = append(args,
+		"--parent-pid="+strconv.Itoa(os.Getpid()),
+		"--interface="+cfg.InterfaceName,
+	)
+	if cfg.SocketPath != "" {
+		args = append(args, "--socket="+cfg.SocketPath)
+	}
+
+	cmd := exec.Command(cfg.Executable, args...)
+
+	// Detach: new session so the watchdog is not killed by a signal
+	// delivered to the parent's process group.
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setsid: true,
+	}
+
+	// Direct watchdog output to a log file or /dev/null so it doesn't
+	// share file descriptors with the parent's TTY.
+	logTarget := cfg.LogFile
+	if logTarget == "" {
+		logTarget = os.DevNull
+	}
+	logFile, err := os.OpenFile(logTarget, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
+	if err != nil {
+		return nil, fmt.Errorf("watchdog: open log file: %w", err)
+	}
+	cmd.Stdin = nil
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+
+	if err := cmd.Start(); err != nil {
+		_ = logFile.Close()
+		return nil, fmt.Errorf("watchdog: start: %w", err)
+	}
+
+	// We don't need our handle on the log file after the subprocess
+	// inherits it.
+	_ = logFile.Close()
+
+	logger.Info("DNS watchdog spawned (pid=%d, exe=%s)", cmd.Process.Pid, cfg.Executable)
+	return cmd, nil
+}
+
+// StopWatchdog asks the watchdog to exit cleanly via SIGTERM and reaps it.
+// Safe to call with a nil cmd.
+func StopWatchdog(cmd *exec.Cmd) {
+	if cmd == nil || cmd.Process == nil {
+		return
+	}
+	pid := cmd.Process.Pid
+	if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
+		logger.Debug("DNS watchdog stop signal failed (pid=%d): %v", pid, err)
+	}
+	// Reap in the background; we don't want to block shutdown if the
+	// watchdog is wedged.
+	go func() {
+		_ = cmd.Wait()
+		logger.Debug("DNS watchdog (pid=%d) reaped", pid)
+	}()
+}

dns/override/watchdog_spawn_windows.go

@@ -0,0 +1,29 @@
+//go:build windows
+
+package olm
+
+import (
+	"os/exec"
+)
+
+// SpawnWatchdogConfig is provided on Windows for API symmetry but the
+// watchdog itself is effectively a no-op there (see watchdog_windows.go).
+type SpawnWatchdogConfig struct {
+	Executable    string
+	Subcommand    []string
+	InterfaceName string
+	SocketPath    string
+	LogFile       string
+}
+
+// SpawnWatchdog is a no-op on Windows; DNS overrides are interface-GUID
+// scoped and reclaimed when the interface is removed.
+func SpawnWatchdog(cfg SpawnWatchdogConfig) (*exec.Cmd, error) {
+	_ = cfg
+	return nil, nil
+}
+
+// StopWatchdog is a no-op on Windows.
+func StopWatchdog(cmd *exec.Cmd) {
+	_ = cmd
+}

dns/override/watchdog_unix.go

@@ -0,0 +1,25 @@
+//go:build !windows
+
+package olm
+
+import (
+	"os"
+	"syscall"
+)
+
+// pidAlive returns true if the process with the given PID is still alive.
+// On Unix-like systems we use signal 0, which performs error checking but
+// does not deliver an actual signal.
+func pidAlive(pid int) bool {
+	if pid <= 0 {
+		return false
+	}
+	proc, err := os.FindProcess(pid)
+	if err != nil {
+		return false
+	}
+	if err := proc.Signal(syscall.Signal(0)); err != nil {
+		return false
+	}
+	return true
+}

dns/override/watchdog_windows.go

@@ -0,0 +1,15 @@
+//go:build windows
+
+package olm
+
+// pidAlive on Windows. Reliable PID probing on Windows requires syscall
+// OpenProcess with PROCESS_QUERY_LIMITED_INFORMATION followed by
+// GetExitCodeProcess, which is non-trivial. Since DNS override on Windows
+// is interface-GUID-scoped and is naturally cleaned up when the WireGuard
+// interface goes away, the watchdog is effectively a no-op on Windows.
+// We always report the parent as alive so the watchdog never tears down
+// DNS based on PID checks.
+func pidAlive(pid int) bool {
+	_ = pid
+	return true
+}

dns/platform/darwin.go

@@ -422,6 +422,17 @@ func (d *DarwinDNSConfigurator) clearState() error {
 // configurator from a previous unclean shutdown. This is a static function that can be
 // called without creating a configurator instance, useful for cleanup before network operations.
 func CleanupStaleDarwinDNS() error {
+	// Always sweep orphaned Olm scutil keys regardless of whether a state
+	// file exists. This protects against cases where the state file was
+	// lost (e.g., user home wiped, write failed) but DNS keys are still
+	// installed in the running scutil session.
+	defer func() {
+		_ = SweepOlmScutilKeys()
+		// Flush DNS cache after any sweep so changes take effect.
+		_ = exec.Command(dscacheutilPath, "-flushcache").Run()
+		_ = exec.Command("killall", "-HUP", "mDNSResponder").Run()
+	}()
+
 	stateFilePath := getDNSStateFilePath()
 
 	// Check if state file exists
@@ -473,3 +484,59 @@ func CleanupStaleDarwinDNS() error {
 
 	return nil
 }
+
+// SweepOlmScutilKeys enumerates scutil State:/Network/Service/Olm-* keys and
+// removes any that are present. This is a best-effort safety net used when
+// state files have been lost or never written.
+func SweepOlmScutilKeys() error {
+	// list scutil keys matching our naming convention
+	listOutput, err := runScutilOnce("list State:/Network/Service/Olm-.*/DNS\n")
+	if err != nil {
+		return fmt.Errorf("scutil list: %w", err)
+	}
+
+	var keys []string
+	scanner := bufio.NewScanner(bytes.NewReader(listOutput))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		// scutil output format: subKey [0] = State:/Network/Service/Olm-Override/DNS
+		idx := strings.Index(line, "State:/Network/Service/Olm-")
+		if idx < 0 {
+			continue
+		}
+		key := strings.TrimSpace(line[idx:])
+		if key != "" {
+			keys = append(keys, key)
+		}
+	}
+
+	if len(keys) == 0 {
+		return nil
+	}
+
+	logger.Info("Sweeping %d orphaned Olm scutil DNS keys", len(keys))
+
+	var commands strings.Builder
+	for _, key := range keys {
+		commands.WriteString(fmt.Sprintf("remove %s\n", key))
+	}
+
+	if _, err := runScutilOnce(commands.String()); err != nil {
+		return fmt.Errorf("scutil sweep remove: %w", err)
+	}
+
+	return nil
+}
+
+// runScutilOnce runs a one-shot scutil command sequence wrapped with open/quit
+// without requiring a configurator instance.
+func runScutilOnce(commands string) ([]byte, error) {
+	wrapped := fmt.Sprintf("open\n%squit\n", commands)
+	cmd := exec.Command(scutilPath)
+	cmd.Stdin = strings.NewReader(wrapped)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("scutil command failed: %w, output: %s", err, output)
+	}
+	return output, nil
+}

dns_commands.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"time"
+
+	"github.com/fosrl/newt/logger"
+	dnsOverride "github.com/fosrl/olm/dns/override"
+)
+
+const (
+	defaultWatchdogInterval  = 5 * time.Second
+	defaultWatchdogThreshold = 3
+)
+
+// runDNSWatchdogCommand handles the `olm watchdog` subcommand. The watchdog
+// is meant to be spawned by an olm process after it installs a DNS
+// override, and forcibly resets the system DNS if that parent dies before
+// restoring it.
+func runDNSWatchdogCommand(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("watchdog", flag.ContinueOnError)
+	parentPID := fs.Int("parent-pid", 0, "PID of the olm process to monitor")
+	socketPath := fs.String("socket", "", "Path to the olm API unix socket (optional)")
+	interfaceName := fs.String("interface", "", "WireGuard interface name (used for cleanup)")
+	interval := fs.Duration("interval", defaultWatchdogInterval, "Liveness check interval")
+	threshold := fs.Int("threshold", defaultWatchdogThreshold, "Consecutive failures before DNS reset")
+
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	if *parentPID <= 0 {
+		return fmt.Errorf("--parent-pid is required and must be positive")
+	}
+
+	// Ensure logger is initialised for the watchdog process.
+	logger.Init(nil)
+
+	return dnsOverride.RunWatchdog(ctx, dnsOverride.WatchdogConfig{
+		ParentPID:        *parentPID,
+		SocketPath:       *socketPath,
+		InterfaceName:    *interfaceName,
+		CheckInterval:    *interval,
+		FailureThreshold: *threshold,
+	})
+}
+
+// runResetDNSCommand handles the `olm reset-dns` subcommand. It forcibly
+// removes any DNS override state left behind on the system.
+func runResetDNSCommand(args []string) error {
+	fs := flag.NewFlagSet("reset-dns", flag.ContinueOnError)
+	interfaceName := fs.String("interface", "olm", "WireGuard interface name")
+
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	logger.Init(nil)
+
+	if err := dnsOverride.ForceResetDNS(*interfaceName); err != nil {
+		return err
+	}
+
+	fmt.Println("DNS reset complete")
+	return nil
+}

main.go

@@ -164,6 +164,26 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	// Internal DNS subcommands. These are handled before normal flag
+	// parsing because they have their own argument layouts and need to
+	// run without setting up the full olm runtime.
+	if len(os.Args) > 1 {
+		switch os.Args[1] {
+		case "watchdog", "dns-watchdog":
+			if err := runDNSWatchdogCommand(signalCtx, os.Args[2:]); err != nil {
+				fmt.Fprintf(os.Stderr, "watchdog failed: %v\n", err)
+				os.Exit(1)
+			}
+			return
+		case "reset-dns":
+			if err := runResetDNSCommand(os.Args[2:]); err != nil {
+				fmt.Fprintf(os.Stderr, "reset-dns failed: %v\n", err)
+				os.Exit(1)
+			}
+			return
+		}
+	}
+
 	// Run in console mode
 	runOlmMainWithArgs(ctx, cancel, signalCtx, os.Args[1:])
 }
@@ -221,6 +241,8 @@ func runOlmMainWithArgs(ctx context.Context, cancel context.CancelFunc, signalCt
 		OnExit:       cancel, // Pass cancel function directly to trigger shutdown
 		OnTerminated: cancel,
 		PprofAddr:    ":4444", // TODO: REMOVE OR MAKE CONFIGURABLE
+		// Re-invoke this binary in watchdog mode to clean up DNS if we die.
+		WatchdogSubcommand: []string{"watchdog"},
 	}
 
 	olm, err := olmpkg.Init(ctx, olmConfig)

olm/connect.go

@@ -240,6 +240,12 @@ func (o *Olm) handleConnect(msg websocket.WSMessage) {
 			return
 		}
 
+		// Start the external watchdog (if configured). The watchdog will
+		// reset DNS if this process dies before it can call
+		// RestoreDNSOverride. This is a no-op when no watchdog
+		// subcommand has been configured on the OlmConfig.
+		o.startDNSWatchdog(o.tunnelConfig.InterfaceName)
+
 		network.SetDNSServers([]string{o.dnsProxy.GetProxyIP().String()})
 	}
 

olm/data.go

@@ -236,19 +236,22 @@ func (o *Olm) handleSync(msg websocket.WSMessage) {
 	// Find peers to add (in expected but not in current) and peers to update
 	for siteId, expectedSite := range expectedPeers {
 		if _, exists := currentPeerMap[siteId]; !exists {
+			// Only trigger add if this is NOT a JIT-only config (i.e., has more than just siteId and aliases)
+			jitOnly := expectedSite.PublicKey == ""
+			if jitOnly {
+				logger.Debug("Sync: Registering aliases for JIT-only site %d", siteId)
+				if err := pm.AddPeer(expectedSite); err != nil {
+					logger.Error("Sync: Failed to register aliases for JIT site %d: %v", siteId, err)
+				}
+				continue
+			}
+
 			// New peer - add it using the add flow (with holepunch)
 			logger.Info("Sync: Adding new peer for site %d", siteId)
 
 			o.holePunchManager.TriggerHolePunch()
 			o.holePunchManager.ResetServerHolepunchInterval() // start sending immediately again so we fill in the endpoint on the cloud
 
-			// // TODO: do we need to send the message to the cloud to add the peer that way?
-			// if err := o.peerManager.AddPeer(expectedSite); err != nil {
-			// 	logger.Error("Sync: Failed to add peer %d: %v", siteId, err)
-			// } else {
-			// 	logger.Info("Sync: Successfully added peer for site %d", siteId)
-			// }
-
 			// add the peer via the server
 			// this is important because newt needs to get triggered as well to add the peer once the hp is complete
 			chainId := fmt.Sprintf("sync-%d", expectedSite.SiteId)

olm/olm.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 	"os"
+	"os/exec"
 	"sync"
 	"time"
 
@@ -68,13 +69,17 @@ type Olm struct {
 	stopRegister   func()
 	updateRegister func(newData any)
 
-	stopPeerSends  map[string]func()
-	stopPeerInits  map[string]func()
+	stopPeerSends   map[string]func()
+	stopPeerInits   map[string]func()
 	jitPendingSites map[int]string // siteId -> chainId for in-flight JIT requests
-	peerSendMu    sync.Mutex
+	peerSendMu      sync.Mutex
 
 	// WaitGroup to track tunnel lifecycle
 	tunnelWg sync.WaitGroup
+
+	// External DNS watchdog process (spawned after DNS override is installed).
+	// nil when no watchdog is running.
+	dnsWatchdogCmd *exec.Cmd
 }
 
 // getPeerManager safely returns the current peerManager under a read-lock.
@@ -188,10 +193,10 @@ func Init(ctx context.Context, config OlmConfig) (*Olm, error) {
 	apiServer.SetAgent(config.Agent)
 
 	newOlm := &Olm{
-		logFile:       logFile,
-		olmCtx:        ctx,
-		apiServer:     apiServer,
-		olmConfig:     config,
+		logFile:         logFile,
+		olmCtx:          ctx,
+		apiServer:       apiServer,
+		olmConfig:       config,
 		stopPeerSends:   make(map[string]func()),
 		stopPeerInits:   make(map[string]func()),
 		jitPendingSites: make(map[int]string),
@@ -327,6 +332,54 @@ func (o *Olm) registerAPICallbacks() {
 	)
 }
 
+// startDNSWatchdog launches an external watchdog process that will reset
+// system DNS if this olm process dies before it can call
+// RestoreDNSOverride. It is a no-op when the OlmConfig has no
+// WatchdogSubcommand configured, or when the watchdog has already been
+// started for this Olm instance.
+func (o *Olm) startDNSWatchdog(interfaceName string) {
+	if o.dnsWatchdogCmd != nil {
+		return
+	}
+	if len(o.olmConfig.WatchdogSubcommand) == 0 {
+		logger.Debug("DNS watchdog disabled (no WatchdogSubcommand configured)")
+		return
+	}
+
+	executable := o.olmConfig.WatchdogExecutable
+	if executable == "" {
+		exe, err := os.Executable()
+		if err != nil {
+			logger.Warn("DNS watchdog: failed to resolve executable: %v", err)
+			return
+		}
+		executable = exe
+	}
+
+	cmd, err := dnsOverride.SpawnWatchdog(dnsOverride.SpawnWatchdogConfig{
+		Executable:    executable,
+		Subcommand:    o.olmConfig.WatchdogSubcommand,
+		InterfaceName: interfaceName,
+		SocketPath:    o.olmConfig.SocketPath,
+		LogFile:       o.olmConfig.WatchdogLogFile,
+	})
+	if err != nil {
+		logger.Warn("DNS watchdog: spawn failed: %v", err)
+		return
+	}
+	o.dnsWatchdogCmd = cmd
+}
+
+// stopDNSWatchdog stops any previously spawned DNS watchdog process.
+// Safe to call when no watchdog was started.
+func (o *Olm) stopDNSWatchdog() {
+	if o.dnsWatchdogCmd == nil {
+		return
+	}
+	dnsOverride.StopWatchdog(o.dnsWatchdogCmd)
+	o.dnsWatchdogCmd = nil
+}
+
 func (o *Olm) StartTunnel(config TunnelConfig) {
 	if o.tunnelRunning {
 		logger.Info("Tunnel already running")
@@ -467,7 +520,8 @@ func (o *Olm) StartTunnel(config TunnelConfig) {
 				"userToken":   userToken,
 				"fingerprint": o.fingerprint,
 				"postures":    o.postures,
-			}, 2*time.Second, 20)
+				"chainId":     generateChainId(), // use a random chainId for registration updates - it won't be used for cancellation since registration is a one-time message but for tracking the session
+			}, 2*time.Second, 20) // after 18 tries on the server side we send the error so dont change this without changing that
 
 			// Invoke onRegistered callback if configured
 			if o.olmConfig.OnRegistered != nil {
@@ -595,6 +649,11 @@ func (o *Olm) Close() {
 		logger.Error("Failed to restore DNS: %v", err)
 	}
 
+	// Stop the watchdog *after* a successful DNS restore so that if we
+	// somehow crash mid-restore the watchdog still has a chance to clean
+	// up. The watchdog itself is a no-op if it was never spawned.
+	o.stopDNSWatchdog()
+
 	if o.holePunchManager != nil {
 		o.holePunchManager.Stop()
 		o.holePunchManager = nil

olm/peer.go

@@ -20,6 +20,12 @@ func (o *Olm) handleWgPeerAdd(msg websocket.WSMessage) {
 		return
 	}
 
+	// Check if connection setup is complete
+	if !o.registered {
+		logger.Warn("Not connected, ignoring add-peer message")
+		return
+	}
+
 	pm := o.getPeerManager()
 	if pm == nil {
 		logger.Debug("Ignoring add-peer message: peerManager is nil (shutdown in progress)")
@@ -82,6 +88,12 @@ func (o *Olm) handleWgPeerRemove(msg websocket.WSMessage) {
 		return
 	}
 
+	// Check if connection setup is complete
+	if !o.registered {
+		logger.Warn("Not connected, ignoring remove-peer message")
+		return
+	}
+
 	pm := o.getPeerManager()
 	if pm == nil {
 		logger.Debug("Ignoring remove-peer message: peerManager is nil (shutdown in progress)")
@@ -125,6 +137,12 @@ func (o *Olm) handleWgPeerUpdate(msg websocket.WSMessage) {
 		return
 	}
 
+	// Check if connection setup is complete
+	if !o.registered {
+		logger.Warn("Not connected, ignoring update-peer message")
+		return
+	}
+
 	pm := o.getPeerManager()
 	if pm == nil {
 		logger.Debug("Ignoring update-peer message: peerManager is nil (shutdown in progress)")
@@ -171,6 +189,9 @@ func (o *Olm) handleWgPeerUpdate(msg websocket.WSMessage) {
 	if updateData.RemoteSubnets != nil {
 		siteConfig.RemoteSubnets = updateData.RemoteSubnets
 	}
+	if updateData.Aliases != nil {
+		siteConfig.Aliases = updateData.Aliases
+	}
 
 	if err := pm.UpdatePeer(siteConfig); err != nil {
 		logger.Error("Failed to update peer: %v", err)
@@ -180,8 +201,10 @@ func (o *Olm) handleWgPeerUpdate(msg websocket.WSMessage) {
 	// If the endpoint changed, trigger holepunch to refresh NAT mappings
 	if updateData.Endpoint != "" && updateData.Endpoint != existingPeer.Endpoint {
 		logger.Info("Endpoint changed for site %d, triggering holepunch to refresh NAT mappings", updateData.SiteId)
-		_ = o.holePunchManager.TriggerHolePunch()
-		o.holePunchManager.ResetServerHolepunchInterval()
+		if o.holePunchManager != nil {
+			_ = o.holePunchManager.TriggerHolePunch()
+			o.holePunchManager.ResetServerHolepunchInterval()
+		}
 	}
 
 	logger.Info("Successfully updated peer for site %d", updateData.SiteId)

olm/types.go

@@ -48,6 +48,21 @@ type OlmConfig struct {
 	OnAuthError  func(statusCode int, message string) // Called when auth fails (401/403)
 	OnOlmError   func(code string, message string)    // Called when registration fails
 	OnExit       func()                               // Called when exit is requested via API
+
+	// DNS watchdog (optional). When WatchdogSubcommand is non-empty, the
+	// olm package will spawn an external watchdog subprocess after a DNS
+	// override is installed. The watchdog will reset the system DNS if
+	// this process dies before it can call RestoreDNSOverride.
+	//
+	// The watchdog is launched as:
+	//   <WatchdogExecutable> <WatchdogSubcommand...> \
+	//       --parent-pid=<pid> --interface=<name> [--socket=<path>]
+	//
+	// When WatchdogExecutable is empty, os.Executable() of the calling
+	// process is used. WatchdogLogFile defaults to /dev/null.
+	WatchdogExecutable string
+	WatchdogSubcommand []string
+	WatchdogLogFile    string
 }
 
 type TunnelConfig struct {
@@ -61,7 +76,7 @@ type TunnelConfig struct {
 	MTU           int
 	DNS           string
 	UpstreamDNS   []string
-	PublicDNS   []string
+	PublicDNS     []string
 	InterfaceName string
 
 	// Advanced

peers/manager.go

@@ -33,7 +33,7 @@ type PeerManagerConfig struct {
 	SharedBind *bind.SharedBind
 	// WSClient is optional - if nil, relay messages won't be sent
 	WSClient  *websocket.Client
-	APIServer   *api.API
+	APIServer *api.API
 	PublicDNS []string
 }
 
@@ -52,7 +52,7 @@ type PeerManager struct {
 	// key is the CIDR string, value is a set of siteIds that want this IP
 	allowedIPClaims map[string]map[int]bool
 	APIServer       *api.API
-	publicDNS     []string
+	publicDNS       []string
 
 	PersistentKeepalive int
 
@@ -71,7 +71,7 @@ func NewPeerManager(config PeerManagerConfig) *PeerManager {
 		allowedIPOwners: make(map[string]int),
 		allowedIPClaims: make(map[string]map[int]bool),
 		APIServer:       config.APIServer,
-		publicDNS:     config.PublicDNS,
+		publicDNS:       config.PublicDNS,
 	}
 
 	// Create the peer monitor
@@ -116,7 +116,7 @@ func (pm *PeerManager) GetAllPeers() []SiteConfig {
 func (pm *PeerManager) AddPeer(siteConfig SiteConfig) error {
 	pm.mu.Lock()
 	defer pm.mu.Unlock()
-	
+
 	for _, alias := range siteConfig.Aliases {
 		address := net.ParseIP(alias.AliasAddress)
 		if address == nil {
@@ -124,7 +124,7 @@ func (pm *PeerManager) AddPeer(siteConfig SiteConfig) error {
 		}
 		pm.dnsProxy.AddDNSRecord(alias.Alias, address, siteConfig.SiteId)
 	}
-	
+
 	if siteConfig.PublicKey == "" {
 		logger.Debug("Skip adding site %d because no pub key", siteConfig.SiteId)
 		return nil
@@ -162,7 +162,7 @@ func (pm *PeerManager) AddPeer(siteConfig SiteConfig) error {
 	if err := network.AddRoutes(siteConfig.RemoteSubnets, pm.interfaceName); err != nil {
 		logger.Error("Failed to add routes for remote subnets: %v", err)
 	}
-	
+
 	monitorAddress := strings.Split(siteConfig.ServerIP, "/")[0]
 	monitorPeer := net.JoinHostPort(monitorAddress, strconv.Itoa(int(siteConfig.ServerPort+1))) // +1 for the monitor port
 
@@ -189,7 +189,7 @@ func (pm *PeerManager) AddPeer(siteConfig SiteConfig) error {
 func (pm *PeerManager) UpdateAllPeersPersistentKeepalive(interval int) map[int]error {
 	pm.mu.RLock()
 	defer pm.mu.RUnlock()
-	
+
 	pm.PersistentKeepalive = interval
 
 	errors := make(map[int]error)
@@ -311,6 +311,29 @@ func (pm *PeerManager) UpdatePeer(siteConfig SiteConfig) error {
 		return fmt.Errorf("peer with site ID %d not found", siteConfig.SiteId)
 	}
 
+	// Update aliases
+	// Remove old aliases
+	for _, alias := range oldPeer.Aliases {
+		address := net.ParseIP(alias.AliasAddress)
+		if address == nil {
+			continue
+		}
+		pm.dnsProxy.RemoveDNSRecord(alias.Alias, address)
+	}
+	// Add new aliases
+	for _, alias := range siteConfig.Aliases {
+		address := net.ParseIP(alias.AliasAddress)
+		if address == nil {
+			continue
+		}
+		pm.dnsProxy.AddDNSRecord(alias.Alias, address, siteConfig.SiteId)
+	}
+
+	if siteConfig.PublicKey == "" {
+		logger.Debug("Skip updating site %d because no pub key", siteConfig.SiteId)
+		return nil
+	}
+
 	// If public key changed, remove old peer first
 	if siteConfig.PublicKey != oldPeer.PublicKey {
 		if err := RemovePeer(pm.device, siteConfig.SiteId, oldPeer.PublicKey); err != nil {
@@ -434,24 +457,6 @@ func (pm *PeerManager) UpdatePeer(siteConfig SiteConfig) error {
 		}
 	}
 
-	// Update aliases
-	// Remove old aliases
-	for _, alias := range oldPeer.Aliases {
-		address := net.ParseIP(alias.AliasAddress)
-		if address == nil {
-			continue
-		}
-		pm.dnsProxy.RemoveDNSRecord(alias.Alias, address)
-	}
-	// Add new aliases
-	for _, alias := range siteConfig.Aliases {
-		address := net.ParseIP(alias.AliasAddress)
-		if address == nil {
-			continue
-		}
-		pm.dnsProxy.AddDNSRecord(alias.Alias, address, siteConfig.SiteId)
-	}
-
 	pm.peerMonitor.UpdateHolepunchEndpoint(siteConfig.SiteId, siteConfig.Endpoint)
 
 	monitorAddress := strings.Split(siteConfig.ServerIP, "/")[0]