Merge pull request #126 from fosrl/dev

Handle remove on a per site level correctly

.github/ISSUE_TEMPLATE/1.bug_report.yml

@@ -14,12 +14,13 @@ body:
           label: Environment
           description: Please fill out the relevant details below for your environment.
           value: |
-              - OS Type & Version: (e.g., Ubuntu 22.04)
+              - OS Type & Version:
               - Pangolin Version:
+              - Edition (Community or Enterprise):
               - Gerbil Version:
               - Traefik Version:
               - Newt Version:
-              - Olm Version: (if applicable)
+              - Client Version:
       validations:
           required: true
 

dns/dns_proxy.go

@@ -755,6 +755,12 @@ func (p *DNSProxy) RemoveDNSRecord(domain string, ip net.IP) {
 	p.recordStore.RemoveRecord(domain, ip)
 }
 
+// RemoveDNSRecordForSite removes DNS records for a domain that are owned by a specific site.
+// If ip is nil, removes all records for the domain that are owned by that site.
+func (p *DNSProxy) RemoveDNSRecordForSite(domain string, ip net.IP, siteId int) {
+	p.recordStore.RemoveRecordForSite(domain, ip, siteId)
+}
+
 // GetDNSRecords returns all IP addresses for a domain and record type.
 // The second return value indicates whether the domain exists.
 func (p *DNSProxy) GetDNSRecords(domain string, recordType RecordType) ([]net.IP, bool) {

dns/dns_records.go

@@ -23,6 +23,7 @@ type recordSet struct {
 	A      []net.IP
 	AAAA   []net.IP
 	SiteId int
+	owners map[string]map[int]bool // IP string -> owning site IDs
 }
 
 // DNSRecordStore manages local DNS records for A, AAAA, and PTR queries.
@@ -71,9 +72,17 @@ func (s *DNSRecordStore) AddRecord(domain string, ip net.IP, siteId int) error {
 	}
 
 	if m[domain] == nil {
-		m[domain] = &recordSet{SiteId: siteId}
+		m[domain] = &recordSet{SiteId: siteId, owners: make(map[string]map[int]bool)}
 	}
 	rs := m[domain]
+	if rs.owners == nil {
+		rs.owners = make(map[string]map[int]bool)
+	}
+	ipKey := ip.String()
+	if rs.owners[ipKey] == nil {
+		rs.owners[ipKey] = make(map[int]bool)
+	}
+	rs.owners[ipKey][siteId] = true
 	if isV4 {
 		for _, existing := range rs.A {
 			if existing.Equal(ip) {
@@ -97,7 +106,6 @@ func (s *DNSRecordStore) AddRecord(domain string, ip net.IP, siteId int) error {
 	return nil
 }
 
-
 // AddPTRRecord adds a PTR record mapping an IP address to a domain name
 // ip should be a valid IPv4 or IPv6 address
 // domain should be in FQDN format (e.g., "example.com.")
@@ -123,6 +131,16 @@ func (s *DNSRecordStore) AddPTRRecord(ip net.IP, domain string) error {
 // If ip is nil, removes all records for the domain (including wildcards)
 // Automatically removes corresponding PTR records for non-wildcard domains
 func (s *DNSRecordStore) RemoveRecord(domain string, ip net.IP) {
+	s.removeRecord(domain, ip, 0, false)
+}
+
+// RemoveRecordForSite removes DNS records owned by a specific site.
+// If ip is nil, it removes all records for the domain owned by that site.
+func (s *DNSRecordStore) RemoveRecordForSite(domain string, ip net.IP, siteId int) {
+	s.removeRecord(domain, ip, siteId, true)
+}
+
+func (s *DNSRecordStore) removeRecord(domain string, ip net.IP, siteId int, bySite bool) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
@@ -142,8 +160,20 @@ func (s *DNSRecordStore) RemoveRecord(domain string, ip net.IP) {
 	if rs == nil {
 		return
 	}
+	if rs.owners == nil {
+		rs.owners = make(map[string]map[int]bool)
+	}
 
 	if ip == nil {
+		if bySite {
+			rs.A = s.removeOwnedIPs(rs, rs.A, siteId, !isWildcard, domain)
+			rs.AAAA = s.removeOwnedIPs(rs, rs.AAAA, siteId, !isWildcard, domain)
+			if len(rs.A) == 0 && len(rs.AAAA) == 0 {
+				delete(m, domain)
+			}
+			return
+		}
+
 		// Remove all records for this domain
 		if !isWildcard {
 			for _, ipAddr := range rs.A {
@@ -162,6 +192,19 @@ func (s *DNSRecordStore) RemoveRecord(domain string, ip net.IP) {
 	}
 
 	// Remove specific IP
+	ipKey := ip.String()
+	if bySite {
+		owners := rs.owners[ipKey]
+		if len(owners) == 0 {
+			return
+		}
+		delete(owners, siteId)
+		if len(owners) > 0 {
+			return
+		}
+		delete(rs.owners, ipKey)
+	}
+
 	if ip.To4() != nil {
 		rs.A = removeIP(rs.A, ip)
 		if !isWildcard {
@@ -177,6 +220,7 @@ func (s *DNSRecordStore) RemoveRecord(domain string, ip net.IP) {
 			}
 		}
 	}
+	delete(rs.owners, ipKey)
 
 	// Clean up empty record sets
 	if len(rs.A) == 0 && len(rs.AAAA) == 0 {
@@ -184,6 +228,33 @@ func (s *DNSRecordStore) RemoveRecord(domain string, ip net.IP) {
 	}
 }
 
+func (s *DNSRecordStore) removeOwnedIPs(rs *recordSet, ips []net.IP, siteId int, removePTR bool, domain string) []net.IP {
+	kept := make([]net.IP, 0, len(ips))
+	for _, ipAddr := range ips {
+		ipKey := ipAddr.String()
+		owners := rs.owners[ipKey]
+		if len(owners) == 0 {
+			kept = append(kept, ipAddr)
+			continue
+		}
+
+		delete(owners, siteId)
+		if len(owners) > 0 {
+			kept = append(kept, ipAddr)
+			continue
+		}
+
+		delete(rs.owners, ipKey)
+		if removePTR {
+			if ptrDomain, exists := s.ptrRecords[ipKey]; exists && ptrDomain == domain {
+				delete(s.ptrRecords, ipKey)
+			}
+		}
+	}
+
+	return kept
+}
+
 // RemovePTRRecord removes a PTR record for an IP address
 func (s *DNSRecordStore) RemovePTRRecord(ip net.IP) {
 	s.mu.Lock()

dns/dns_records_test.go

@@ -782,6 +782,37 @@ func TestAutomaticPTRRecordOnRemove(t *testing.T) {
 	}
 }
 
+func TestRemoveRecordForSiteKeepsSharedAliasIP(t *testing.T) {
+	store := NewDNSRecordStore()
+
+	domain := "shared.example.com."
+	ip := net.ParseIP("192.168.1.100")
+
+	if err := store.AddRecord(domain, ip, 10); err != nil {
+		t.Fatalf("Failed to add record for site 10: %v", err)
+	}
+	if err := store.AddRecord(domain, ip, 20); err != nil {
+		t.Fatalf("Failed to add record for site 20: %v", err)
+	}
+
+	store.RemoveRecordForSite(domain, ip, 10)
+
+	ips, exists := store.GetRecords(domain, RecordTypeA)
+	if !exists {
+		t.Fatal("Expected shared record to still exist after removing one site owner")
+	}
+	if len(ips) != 1 || !ips[0].Equal(ip) {
+		t.Fatalf("Expected shared IP to remain after first owner removal, got %v", ips)
+	}
+
+	store.RemoveRecordForSite(domain, ip, 20)
+
+	ips, exists = store.GetRecords(domain, RecordTypeA)
+	if exists {
+		t.Fatalf("Expected domain to be removed after last owner removal, got %v", ips)
+	}
+}
+
 func TestAutomaticPTRRecordOnRemoveAll(t *testing.T) {
 	store := NewDNSRecordStore()
 

dns/override/dns_override_android.go

@@ -20,3 +20,9 @@ func CleanupStaleState(interfaceName string) error {
 	_ = interfaceName
 	return nil
 }
+
+// ForceResetDNS is a no-op on Android.
+func ForceResetDNS(interfaceName string) error {
+	_ = interfaceName
+	return nil
+}

dns/override/dns_override_darwin.go

@@ -15,6 +15,13 @@ var configurator platform.DNSConfigurator
 // SetupDNSOverride configures the system DNS to use the DNS proxy on macOS
 // Uses scutil for DNS configuration
 func SetupDNSOverride(interfaceName string, proxyIp netip.Addr) error {
+	// Defensively clear any stale DNS state from a previous unclean shutdown
+	// before installing the new override. This makes a second tunnel start
+	// safe even if the previous client crashed without restoring DNS.
+	if err := CleanupStaleState(interfaceName); err != nil {
+		logger.Warn("Pre-setup stale DNS cleanup failed (continuing): %v", err)
+	}
+
 	var err error
 	configurator, err = platform.NewDarwinDNSConfigurator()
 	if err != nil {
@@ -78,3 +85,31 @@ func CleanupStaleState(interfaceName string) error {
 	logger.Info("Stale DNS state cleanup completed successfully")
 	return nil
 }
+
+// ForceResetDNS forcibly clears any DNS override state, whether or not the
+// current process installed it. This is intended for the "reset-dns" CLI
+// command and for the watchdog process to recover from a stuck override
+// left behind by a crashed client.
+func ForceResetDNS(interfaceName string) error {
+	logger.Info("Forcing DNS reset on Darwin (interface=%s)", interfaceName)
+
+	// First clean up any persisted state from a previous session.
+	cleanupErr := CleanupStaleState(interfaceName)
+
+	// Then, if the current process happens to hold a live configurator,
+	// instruct it to restore DNS as well so in-memory state is consistent.
+	if configurator != nil {
+		if err := configurator.RestoreDNS(); err != nil {
+			logger.Warn("ForceResetDNS: in-memory restore failed: %v", err)
+		}
+		configurator = nil
+	}
+
+	// As a last-resort defense, sweep any scutil keys matching our naming
+	// convention even if no state file exists.
+	if err := platform.SweepOlmScutilKeys(); err != nil {
+		logger.Warn("ForceResetDNS: scutil sweep failed: %v", err)
+	}
+
+	return cleanupErr
+}

dns/override/dns_override_ios.go

@@ -19,3 +19,9 @@ func CleanupStaleState(interfaceName string) error {
 	_ = interfaceName
 	return nil
 }
+
+// ForceResetDNS is a no-op on iOS.
+func ForceResetDNS(interfaceName string) error {
+	_ = interfaceName
+	return nil
+}

dns/override/dns_override_unix.go

@@ -15,6 +15,13 @@ var configurator platform.DNSConfigurator
 // SetupDNSOverride configures the system DNS to use the DNS proxy on Linux/FreeBSD
 // Detects the DNS manager by reading /etc/resolv.conf and verifying runtime availability
 func SetupDNSOverride(interfaceName string, proxyIp netip.Addr) error {
+	// Defensively clear any stale DNS state from a previous unclean shutdown
+	// before installing the new override. This makes a second tunnel start
+	// safe even if the previous client crashed without restoring DNS.
+	if err := CleanupStaleState(interfaceName); err != nil {
+		logger.Warn("Pre-setup stale DNS cleanup failed (continuing): %v", err)
+	}
+
 	var err error
 
 	// Detect which DNS manager is in use by checking /etc/resolv.conf and runtime availability
@@ -144,3 +151,25 @@ func CleanupStaleState(interfaceName string) error {
 	logger.Info("Stale DNS state cleanup completed successfully")
 	return nil
 }
+
+// ForceResetDNS forcibly clears any DNS override state, whether or not the
+// current process installed it. This is intended for the "reset-dns" CLI
+// command and for the watchdog process to recover from a stuck override
+// left behind by a crashed client.
+func ForceResetDNS(interfaceName string) error {
+	logger.Info("Forcing DNS reset on Linux/FreeBSD (interface=%s)", interfaceName)
+
+	// First clean up any persisted state from a previous session.
+	cleanupErr := CleanupStaleState(interfaceName)
+
+	// Then, if the current process happens to hold a live configurator,
+	// instruct it to restore DNS as well so in-memory state is consistent.
+	if configurator != nil {
+		if err := configurator.RestoreDNS(); err != nil {
+			logger.Warn("ForceResetDNS: in-memory restore failed: %v", err)
+		}
+		configurator = nil
+	}
+
+	return cleanupErr
+}

dns/override/dns_override_windows.go

@@ -15,6 +15,12 @@ var configurator platform.DNSConfigurator
 // SetupDNSOverride configures the system DNS to use the DNS proxy on Windows
 // Uses registry-based configuration (automatically extracts interface GUID)
 func SetupDNSOverride(interfaceName string, proxyIp netip.Addr) error {
+	// Defensively clear any stale DNS state from a previous unclean shutdown
+	// before installing the new override.
+	if err := CleanupStaleState(interfaceName); err != nil {
+		logger.Warn("Pre-setup stale DNS cleanup failed (continuing): %v", err)
+	}
+
 	var err error
 	configurator, err = platform.NewWindowsDNSConfigurator(interfaceName)
 	if err != nil {
@@ -77,3 +83,17 @@ func CleanupStaleState(interfaceName string) error {
 	logger.Debug("Windows DNS cleanup: no stale state to clean (interface-specific)")
 	return nil
 }
+
+// ForceResetDNS forcibly clears any DNS override state. On Windows this is
+// largely a no-op because the registry override is tied to the interface
+// GUID and is reclaimed when the interface is torn down.
+func ForceResetDNS(interfaceName string) error {
+	logger.Info("Forcing DNS reset on Windows (interface=%s)", interfaceName)
+	if configurator != nil {
+		if err := configurator.RestoreDNS(); err != nil {
+			logger.Warn("ForceResetDNS: in-memory restore failed: %v", err)
+		}
+		configurator = nil
+	}
+	return CleanupStaleState(interfaceName)
+}

dns/override/watchdog.go

@@ -0,0 +1,136 @@
+package olm
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/fosrl/newt/logger"
+)
+
+// WatchdogConfig configures the DNS override watchdog. The watchdog runs as
+// an external process (spawned via SpawnWatchdog) and monitors a parent olm
+// process. When the parent appears to have died without restoring DNS, the
+// watchdog forcibly resets the system DNS configuration.
+type WatchdogConfig struct {
+	// ParentPID is the PID of the olm process that installed the DNS
+	// override. The watchdog exits when this PID is no longer alive.
+	ParentPID int
+
+	// SocketPath is the path to the olm Unix domain socket (or named pipe
+	// on Windows). The watchdog uses it as a secondary liveness signal.
+	// May be empty if no socket-based API is enabled.
+	SocketPath string
+
+	// InterfaceName is the name of the WireGuard interface whose DNS
+	// override should be reset on parent death.
+	InterfaceName string
+
+	// CheckInterval is how often to poll the parent's liveness.
+	// Defaults to 5 seconds when zero.
+	CheckInterval time.Duration
+
+	// FailureThreshold is the number of consecutive failed liveness checks
+	// before the watchdog declares the parent dead and resets DNS.
+	// Defaults to 3 when zero.
+	FailureThreshold int
+}
+
+// RunWatchdog runs the watchdog loop in the current process until either
+// (a) the parent dies and DNS is reset, or (b) ctx is cancelled.
+func RunWatchdog(ctx context.Context, cfg WatchdogConfig) error {
+	if cfg.ParentPID <= 0 {
+		return fmt.Errorf("watchdog: invalid parent PID %d", cfg.ParentPID)
+	}
+	if cfg.CheckInterval <= 0 {
+		cfg.CheckInterval = 5 * time.Second
+	}
+	if cfg.FailureThreshold <= 0 {
+		cfg.FailureThreshold = 3
+	}
+
+	logger.Info("DNS watchdog started: parent=%d interval=%s threshold=%d socket=%q interface=%q",
+		cfg.ParentPID, cfg.CheckInterval, cfg.FailureThreshold, cfg.SocketPath, cfg.InterfaceName)
+
+	ticker := time.NewTicker(cfg.CheckInterval)
+	defer ticker.Stop()
+
+	consecutiveFailures := 0
+
+	for {
+		select {
+		case <-ctx.Done():
+			logger.Info("DNS watchdog context cancelled, exiting cleanly")
+			return ctx.Err()
+		case <-ticker.C:
+		}
+
+		alive := isParentAlive(cfg.ParentPID, cfg.SocketPath)
+		if alive {
+			if consecutiveFailures > 0 {
+				logger.Debug("DNS watchdog: parent recovered after %d failures", consecutiveFailures)
+			}
+			consecutiveFailures = 0
+			continue
+		}
+
+		consecutiveFailures++
+		logger.Warn("DNS watchdog: parent liveness check failed (%d/%d)",
+			consecutiveFailures, cfg.FailureThreshold)
+
+		if consecutiveFailures >= cfg.FailureThreshold {
+			logger.Warn("DNS watchdog: parent declared dead, forcing DNS reset")
+			if err := ForceResetDNS(cfg.InterfaceName); err != nil {
+				logger.Error("DNS watchdog: ForceResetDNS failed: %v", err)
+				return err
+			}
+			logger.Info("DNS watchdog: DNS reset complete, exiting")
+			return nil
+		}
+	}
+}
+
+// isParentAlive returns true if the parent process appears to be alive. It
+// considers the parent alive if EITHER the PID is still running OR the
+// socket-based health endpoint responds. This dual check avoids false
+// positives where one signal is flaky (e.g., socket blocked but process
+// still recovering).
+func isParentAlive(pid int, socketPath string) bool {
+	if pidAlive(pid) {
+		return true
+	}
+	// Process is gone; double-check via socket to avoid races where PID
+	// recycling or signal-0 quirks lie to us. Socket should already be
+	// gone too.
+	if socketPath != "" && socketHealthy(socketPath) {
+		return true
+	}
+	return false
+}
+
+// socketHealthy attempts a fast /health request over the unix socket.
+func socketHealthy(socketPath string) bool {
+	if _, err := os.Stat(socketPath); err != nil {
+		return false
+	}
+
+	client := &http.Client{
+		Timeout: 2 * time.Second,
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+				d := net.Dialer{Timeout: 2 * time.Second}
+				return d.DialContext(ctx, "unix", socketPath)
+			},
+		},
+	}
+
+	resp, err := client.Get("http://localhost/health")
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+	return resp.StatusCode == http.StatusOK
+}

dns/override/watchdog_spawn_unix.go

@@ -0,0 +1,119 @@
+//go:build !windows
+
+package olm
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strconv"
+	"syscall"
+
+	"github.com/fosrl/newt/logger"
+)
+
+// SpawnWatchdogConfig captures the inputs needed to launch the external
+// watchdog subprocess that monitors the calling olm process and forces a
+// DNS reset if the parent dies before restoring DNS.
+type SpawnWatchdogConfig struct {
+	// Executable is the path to the binary that will host the watchdog
+	// (typically os.Executable()). The binary must understand the
+	// watchdog subcommand layout described below.
+	Executable string
+
+	// Subcommand is the argv prefix the binary uses to enter watchdog
+	// mode (e.g., []string{"watchdog"} or []string{"dns", "watchdog"}).
+	Subcommand []string
+
+	// InterfaceName is the WireGuard interface whose DNS override should
+	// be reset if the parent dies.
+	InterfaceName string
+
+	// SocketPath is the parent's olm API socket path (may be empty).
+	SocketPath string
+
+	// LogFile, if non-empty, is the path the watchdog writes its stdout
+	// and stderr to. If empty, /dev/null is used.
+	LogFile string
+}
+
+// SpawnWatchdog launches the watchdog subprocess in a detached process group
+// so that it survives the death of the parent. The returned *exec.Cmd is the
+// handle the parent should call StopWatchdog on during clean shutdown.
+//
+// The spawned process is invoked as:
+//
+//	<Executable> <Subcommand...> --parent-pid=<ppid> \
+//	    --interface=<InterfaceName> [--socket=<SocketPath>]
+//
+// Both pangolin (cli) and olm should map their watchdog subcommand to
+// RunWatchdog.
+func SpawnWatchdog(cfg SpawnWatchdogConfig) (*exec.Cmd, error) {
+	if cfg.Executable == "" {
+		return nil, fmt.Errorf("watchdog: executable is required")
+	}
+	if len(cfg.Subcommand) == 0 {
+		return nil, fmt.Errorf("watchdog: subcommand is required")
+	}
+
+	args := append([]string{}, cfg.Subcommand...)
+	args = append(args,
+		"--parent-pid="+strconv.Itoa(os.Getpid()),
+		"--interface="+cfg.InterfaceName,
+	)
+	if cfg.SocketPath != "" {
+		args = append(args, "--socket="+cfg.SocketPath)
+	}
+
+	cmd := exec.Command(cfg.Executable, args...)
+
+	// Detach: new session so the watchdog is not killed by a signal
+	// delivered to the parent's process group.
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setsid: true,
+	}
+
+	// Direct watchdog output to a log file or /dev/null so it doesn't
+	// share file descriptors with the parent's TTY.
+	logTarget := cfg.LogFile
+	if logTarget == "" {
+		logTarget = os.DevNull
+	}
+	logFile, err := os.OpenFile(logTarget, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
+	if err != nil {
+		return nil, fmt.Errorf("watchdog: open log file: %w", err)
+	}
+	cmd.Stdin = nil
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+
+	if err := cmd.Start(); err != nil {
+		_ = logFile.Close()
+		return nil, fmt.Errorf("watchdog: start: %w", err)
+	}
+
+	// We don't need our handle on the log file after the subprocess
+	// inherits it.
+	_ = logFile.Close()
+
+	logger.Info("DNS watchdog spawned (pid=%d, exe=%s)", cmd.Process.Pid, cfg.Executable)
+	return cmd, nil
+}
+
+// StopWatchdog asks the watchdog to exit cleanly via SIGTERM and reaps it.
+// Safe to call with a nil cmd.
+func StopWatchdog(cmd *exec.Cmd) {
+	if cmd == nil || cmd.Process == nil {
+		return
+	}
+	pid := cmd.Process.Pid
+	if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
+		logger.Debug("DNS watchdog stop signal failed (pid=%d): %v", pid, err)
+	}
+	// Reap in the background; we don't want to block shutdown if the
+	// watchdog is wedged.
+	go func() {
+		_ = cmd.Wait()
+		logger.Debug("DNS watchdog (pid=%d) reaped", pid)
+	}()
+}

dns/override/watchdog_spawn_windows.go

@@ -0,0 +1,29 @@
+//go:build windows
+
+package olm
+
+import (
+	"os/exec"
+)
+
+// SpawnWatchdogConfig is provided on Windows for API symmetry but the
+// watchdog itself is effectively a no-op there (see watchdog_windows.go).
+type SpawnWatchdogConfig struct {
+	Executable    string
+	Subcommand    []string
+	InterfaceName string
+	SocketPath    string
+	LogFile       string
+}
+
+// SpawnWatchdog is a no-op on Windows; DNS overrides are interface-GUID
+// scoped and reclaimed when the interface is removed.
+func SpawnWatchdog(cfg SpawnWatchdogConfig) (*exec.Cmd, error) {
+	_ = cfg
+	return nil, nil
+}
+
+// StopWatchdog is a no-op on Windows.
+func StopWatchdog(cmd *exec.Cmd) {
+	_ = cmd
+}

dns/override/watchdog_unix.go

@@ -0,0 +1,25 @@
+//go:build !windows
+
+package olm
+
+import (
+	"os"
+	"syscall"
+)
+
+// pidAlive returns true if the process with the given PID is still alive.
+// On Unix-like systems we use signal 0, which performs error checking but
+// does not deliver an actual signal.
+func pidAlive(pid int) bool {
+	if pid <= 0 {
+		return false
+	}
+	proc, err := os.FindProcess(pid)
+	if err != nil {
+		return false
+	}
+	if err := proc.Signal(syscall.Signal(0)); err != nil {
+		return false
+	}
+	return true
+}

dns/override/watchdog_windows.go

@@ -0,0 +1,15 @@
+//go:build windows
+
+package olm
+
+// pidAlive on Windows. Reliable PID probing on Windows requires syscall
+// OpenProcess with PROCESS_QUERY_LIMITED_INFORMATION followed by
+// GetExitCodeProcess, which is non-trivial. Since DNS override on Windows
+// is interface-GUID-scoped and is naturally cleaned up when the WireGuard
+// interface goes away, the watchdog is effectively a no-op on Windows.
+// We always report the parent as alive so the watchdog never tears down
+// DNS based on PID checks.
+func pidAlive(pid int) bool {
+	_ = pid
+	return true
+}

dns/platform/darwin.go

@@ -422,6 +422,17 @@ func (d *DarwinDNSConfigurator) clearState() error {
 // configurator from a previous unclean shutdown. This is a static function that can be
 // called without creating a configurator instance, useful for cleanup before network operations.
 func CleanupStaleDarwinDNS() error {
+	// Always sweep orphaned Olm scutil keys regardless of whether a state
+	// file exists. This protects against cases where the state file was
+	// lost (e.g., user home wiped, write failed) but DNS keys are still
+	// installed in the running scutil session.
+	defer func() {
+		_ = SweepOlmScutilKeys()
+		// Flush DNS cache after any sweep so changes take effect.
+		_ = exec.Command(dscacheutilPath, "-flushcache").Run()
+		_ = exec.Command("killall", "-HUP", "mDNSResponder").Run()
+	}()
+
 	stateFilePath := getDNSStateFilePath()
 
 	// Check if state file exists
@@ -473,3 +484,59 @@ func CleanupStaleDarwinDNS() error {
 
 	return nil
 }
+
+// SweepOlmScutilKeys enumerates scutil State:/Network/Service/Olm-* keys and
+// removes any that are present. This is a best-effort safety net used when
+// state files have been lost or never written.
+func SweepOlmScutilKeys() error {
+	// list scutil keys matching our naming convention
+	listOutput, err := runScutilOnce("list State:/Network/Service/Olm-.*/DNS\n")
+	if err != nil {
+		return fmt.Errorf("scutil list: %w", err)
+	}
+
+	var keys []string
+	scanner := bufio.NewScanner(bytes.NewReader(listOutput))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		// scutil output format: subKey [0] = State:/Network/Service/Olm-Override/DNS
+		idx := strings.Index(line, "State:/Network/Service/Olm-")
+		if idx < 0 {
+			continue
+		}
+		key := strings.TrimSpace(line[idx:])
+		if key != "" {
+			keys = append(keys, key)
+		}
+	}
+
+	if len(keys) == 0 {
+		return nil
+	}
+
+	logger.Info("Sweeping %d orphaned Olm scutil DNS keys", len(keys))
+
+	var commands strings.Builder
+	for _, key := range keys {
+		commands.WriteString(fmt.Sprintf("remove %s\n", key))
+	}
+
+	if _, err := runScutilOnce(commands.String()); err != nil {
+		return fmt.Errorf("scutil sweep remove: %w", err)
+	}
+
+	return nil
+}
+
+// runScutilOnce runs a one-shot scutil command sequence wrapped with open/quit
+// without requiring a configurator instance.
+func runScutilOnce(commands string) ([]byte, error) {
+	wrapped := fmt.Sprintf("open\n%squit\n", commands)
+	cmd := exec.Command(scutilPath)
+	cmd.Stdin = strings.NewReader(wrapped)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("scutil command failed: %w, output: %s", err, output)
+	}
+	return output, nil
+}

dns_commands.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"time"
+
+	"github.com/fosrl/newt/logger"
+	dnsOverride "github.com/fosrl/olm/dns/override"
+)
+
+const (
+	defaultWatchdogInterval  = 5 * time.Second
+	defaultWatchdogThreshold = 3
+)
+
+// runDNSWatchdogCommand handles the `olm watchdog` subcommand. The watchdog
+// is meant to be spawned by an olm process after it installs a DNS
+// override, and forcibly resets the system DNS if that parent dies before
+// restoring it.
+func runDNSWatchdogCommand(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("watchdog", flag.ContinueOnError)
+	parentPID := fs.Int("parent-pid", 0, "PID of the olm process to monitor")
+	socketPath := fs.String("socket", "", "Path to the olm API unix socket (optional)")
+	interfaceName := fs.String("interface", "", "WireGuard interface name (used for cleanup)")
+	interval := fs.Duration("interval", defaultWatchdogInterval, "Liveness check interval")
+	threshold := fs.Int("threshold", defaultWatchdogThreshold, "Consecutive failures before DNS reset")
+
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	if *parentPID <= 0 {
+		return fmt.Errorf("--parent-pid is required and must be positive")
+	}
+
+	// Ensure logger is initialised for the watchdog process.
+	logger.Init(nil)
+
+	return dnsOverride.RunWatchdog(ctx, dnsOverride.WatchdogConfig{
+		ParentPID:        *parentPID,
+		SocketPath:       *socketPath,
+		InterfaceName:    *interfaceName,
+		CheckInterval:    *interval,
+		FailureThreshold: *threshold,
+	})
+}
+
+// runResetDNSCommand handles the `olm reset-dns` subcommand. It forcibly
+// removes any DNS override state left behind on the system.
+func runResetDNSCommand(args []string) error {
+	fs := flag.NewFlagSet("reset-dns", flag.ContinueOnError)
+	interfaceName := fs.String("interface", "olm", "WireGuard interface name")
+
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	logger.Init(nil)
+
+	if err := dnsOverride.ForceResetDNS(*interfaceName); err != nil {
+		return err
+	}
+
+	fmt.Println("DNS reset complete")
+	return nil
+}

main.go

@@ -164,6 +164,26 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	// Internal DNS subcommands. These are handled before normal flag
+	// parsing because they have their own argument layouts and need to
+	// run without setting up the full olm runtime.
+	if len(os.Args) > 1 {
+		switch os.Args[1] {
+		case "watchdog", "dns-watchdog":
+			if err := runDNSWatchdogCommand(signalCtx, os.Args[2:]); err != nil {
+				fmt.Fprintf(os.Stderr, "watchdog failed: %v\n", err)
+				os.Exit(1)
+			}
+			return
+		case "reset-dns":
+			if err := runResetDNSCommand(os.Args[2:]); err != nil {
+				fmt.Fprintf(os.Stderr, "reset-dns failed: %v\n", err)
+				os.Exit(1)
+			}
+			return
+		}
+	}
+
 	// Run in console mode
 	runOlmMainWithArgs(ctx, cancel, signalCtx, os.Args[1:])
 }
@@ -221,6 +241,8 @@ func runOlmMainWithArgs(ctx context.Context, cancel context.CancelFunc, signalCt
 		OnExit:       cancel, // Pass cancel function directly to trigger shutdown
 		OnTerminated: cancel,
 		PprofAddr:    ":4444", // TODO: REMOVE OR MAKE CONFIGURABLE
+		// Re-invoke this binary in watchdog mode to clean up DNS if we die.
+		WatchdogSubcommand: []string{"watchdog"},
 	}
 
 	olm, err := olmpkg.Init(ctx, olmConfig)

olm/connect.go

@@ -240,6 +240,12 @@ func (o *Olm) handleConnect(msg websocket.WSMessage) {
 			return
 		}
 
+		// Start the external watchdog (if configured). The watchdog will
+		// reset DNS if this process dies before it can call
+		// RestoreDNSOverride. This is a no-op when no watchdog
+		// subcommand has been configured on the OlmConfig.
+		o.startDNSWatchdog(o.tunnelConfig.InterfaceName)
+
 		network.SetDNSServers([]string{o.dnsProxy.GetProxyIP().String()})
 	}
 

olm/olm.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 	"os"
+	"os/exec"
 	"sync"
 	"time"
 
@@ -68,13 +69,17 @@ type Olm struct {
 	stopRegister   func()
 	updateRegister func(newData any)
 
-	stopPeerSends  map[string]func()
-	stopPeerInits  map[string]func()
+	stopPeerSends   map[string]func()
+	stopPeerInits   map[string]func()
 	jitPendingSites map[int]string // siteId -> chainId for in-flight JIT requests
-	peerSendMu    sync.Mutex
+	peerSendMu      sync.Mutex
 
 	// WaitGroup to track tunnel lifecycle
 	tunnelWg sync.WaitGroup
+
+	// External DNS watchdog process (spawned after DNS override is installed).
+	// nil when no watchdog is running.
+	dnsWatchdogCmd *exec.Cmd
 }
 
 // getPeerManager safely returns the current peerManager under a read-lock.
@@ -188,10 +193,10 @@ func Init(ctx context.Context, config OlmConfig) (*Olm, error) {
 	apiServer.SetAgent(config.Agent)
 
 	newOlm := &Olm{
-		logFile:       logFile,
-		olmCtx:        ctx,
-		apiServer:     apiServer,
-		olmConfig:     config,
+		logFile:         logFile,
+		olmCtx:          ctx,
+		apiServer:       apiServer,
+		olmConfig:       config,
 		stopPeerSends:   make(map[string]func()),
 		stopPeerInits:   make(map[string]func()),
 		jitPendingSites: make(map[int]string),
@@ -327,6 +332,54 @@ func (o *Olm) registerAPICallbacks() {
 	)
 }
 
+// startDNSWatchdog launches an external watchdog process that will reset
+// system DNS if this olm process dies before it can call
+// RestoreDNSOverride. It is a no-op when the OlmConfig has no
+// WatchdogSubcommand configured, or when the watchdog has already been
+// started for this Olm instance.
+func (o *Olm) startDNSWatchdog(interfaceName string) {
+	if o.dnsWatchdogCmd != nil {
+		return
+	}
+	if len(o.olmConfig.WatchdogSubcommand) == 0 {
+		logger.Debug("DNS watchdog disabled (no WatchdogSubcommand configured)")
+		return
+	}
+
+	executable := o.olmConfig.WatchdogExecutable
+	if executable == "" {
+		exe, err := os.Executable()
+		if err != nil {
+			logger.Warn("DNS watchdog: failed to resolve executable: %v", err)
+			return
+		}
+		executable = exe
+	}
+
+	cmd, err := dnsOverride.SpawnWatchdog(dnsOverride.SpawnWatchdogConfig{
+		Executable:    executable,
+		Subcommand:    o.olmConfig.WatchdogSubcommand,
+		InterfaceName: interfaceName,
+		SocketPath:    o.olmConfig.SocketPath,
+		LogFile:       o.olmConfig.WatchdogLogFile,
+	})
+	if err != nil {
+		logger.Warn("DNS watchdog: spawn failed: %v", err)
+		return
+	}
+	o.dnsWatchdogCmd = cmd
+}
+
+// stopDNSWatchdog stops any previously spawned DNS watchdog process.
+// Safe to call when no watchdog was started.
+func (o *Olm) stopDNSWatchdog() {
+	if o.dnsWatchdogCmd == nil {
+		return
+	}
+	dnsOverride.StopWatchdog(o.dnsWatchdogCmd)
+	o.dnsWatchdogCmd = nil
+}
+
 func (o *Olm) StartTunnel(config TunnelConfig) {
 	if o.tunnelRunning {
 		logger.Info("Tunnel already running")
@@ -467,7 +520,8 @@ func (o *Olm) StartTunnel(config TunnelConfig) {
 				"userToken":   userToken,
 				"fingerprint": o.fingerprint,
 				"postures":    o.postures,
-			}, 2*time.Second, 20)
+				"chainId":     generateChainId(), // use a random chainId for registration updates - it won't be used for cancellation since registration is a one-time message but for tracking the session
+			}, 2*time.Second, 20) // after 18 tries on the server side we send the error so dont change this without changing that
 
 			// Invoke onRegistered callback if configured
 			if o.olmConfig.OnRegistered != nil {
@@ -595,6 +649,11 @@ func (o *Olm) Close() {
 		logger.Error("Failed to restore DNS: %v", err)
 	}
 
+	// Stop the watchdog *after* a successful DNS restore so that if we
+	// somehow crash mid-restore the watchdog still has a chance to clean
+	// up. The watchdog itself is a no-op if it was never spawned.
+	o.stopDNSWatchdog()
+
 	if o.holePunchManager != nil {
 		o.holePunchManager.Stop()
 		o.holePunchManager = nil

olm/types.go

@@ -48,6 +48,21 @@ type OlmConfig struct {
 	OnAuthError  func(statusCode int, message string) // Called when auth fails (401/403)
 	OnOlmError   func(code string, message string)    // Called when registration fails
 	OnExit       func()                               // Called when exit is requested via API
+
+	// DNS watchdog (optional). When WatchdogSubcommand is non-empty, the
+	// olm package will spawn an external watchdog subprocess after a DNS
+	// override is installed. The watchdog will reset the system DNS if
+	// this process dies before it can call RestoreDNSOverride.
+	//
+	// The watchdog is launched as:
+	//   <WatchdogExecutable> <WatchdogSubcommand...> \
+	//       --parent-pid=<pid> --interface=<name> [--socket=<path>]
+	//
+	// When WatchdogExecutable is empty, os.Executable() of the calling
+	// process is used. WatchdogLogFile defaults to /dev/null.
+	WatchdogExecutable string
+	WatchdogSubcommand []string
+	WatchdogLogFile    string
 }
 
 type TunnelConfig struct {
@@ -61,7 +76,7 @@ type TunnelConfig struct {
 	MTU           int
 	DNS           string
 	UpstreamDNS   []string
-	PublicDNS   []string
+	PublicDNS     []string
 	InterfaceName string
 
 	// Advanced

peers/manager.go

@@ -768,7 +768,7 @@ func (pm *PeerManager) RemoveAlias(siteId int, aliasName string) error {
 	if aliasToRemove != nil {
 		address := net.ParseIP(aliasToRemove.AliasAddress)
 		if address != nil {
-			pm.dnsProxy.RemoveDNSRecord(aliasName, address)
+			pm.dnsProxy.RemoveDNSRecordForSite(aliasName, address, siteId)
 		}
 	}