feat: forward guest DNS to Docker's embedded resolver

.github/contributors.yaml

@@ -107,3 +107,6 @@ users:
   Chennamma-Hotkar:
     name: Chennamma Hotkar
     email: channuhotkar@gmail.com
+  alimx07:
+    name: Ali Mohamed
+    email: amx746@gmail.com

pkg/network/localhost.go

@@ -0,0 +1,361 @@
+// Copyright (c) 2023-2026, Nubificus LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package network
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+)
+
+type LocalhostForwarder interface {
+	Name() string
+	// Forward installs the host-side plumbing (tc filters + customRules) that exposes
+	// the loopback service to the guest.
+	Forward(tap, redirect netlink.Link, ifInfo Interface) (ForwardResult, error)
+}
+
+type ForwardResult struct {
+	DNSServer      net.IP
+	ResolvConfPath string
+}
+
+const (
+	dockerCustomNetDNS = "127.0.0.11"
+)
+
+// localhostForwarders holds the ordered set of runtime probes; the first with non-nil return wins
+var localhostForwarders = []func(containerID string) (LocalhostForwarder, error){
+	// we detect docker only for now.
+	detectDocker,
+}
+
+func DetectLocalhostForwarder(containerID string) (LocalhostForwarder, error) {
+	for _, detect := range localhostForwarders {
+		fwd, err := detect(containerID)
+		if err != nil {
+			return nil, err
+		}
+		if fwd != nil {
+			return fwd, nil
+		}
+	}
+	return nil, nil
+}
+
+var (
+	// dockerDNSTCPRe and dockerDNSUDPRe extract the tcp/udp ports that Docker's
+	// embedded DNS resolver (127.0.0.11) listens on.
+	dockerDNSTCPRe = regexp.MustCompile(`-p\s+tcp.*to-destination\s+127\.0\.0\.11:(\d+)`)
+	dockerDNSUDPRe = regexp.MustCompile(`-p\s+udp.*to-destination\s+127\.0\.0\.11:(\d+)`)
+)
+
+type dockerLocalhost struct {
+	resolvConfPath string
+}
+
+func (dockerLocalhost) Name() string {
+	return "docker"
+}
+
+func detectDocker(containerID string) (LocalhostForwarder, error) {
+	containerBase := filepath.Join("/var/lib/docker/containers", containerID)
+	if _, err := os.Stat(containerBase); err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil // not a Docker runtime
+		}
+		return nil, err
+	}
+
+	resolvConfPath := filepath.Join(containerBase, "resolv.conf")
+	data, err := os.ReadFile(resolvConfPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, fmt.Errorf("resolv.conf is missing: %w", err)
+		}
+		return nil, err
+	}
+
+	// When we run container in custom user network
+	// Docker add 127.0.0.11 as the embedded DNS resolver
+	for _, line := range strings.Split(string(data), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) >= 2 && fields[0] == "nameserver" && fields[1] == dockerCustomNetDNS {
+			return dockerLocalhost{resolvConfPath: resolvConfPath}, nil
+		}
+	}
+	return nil, nil
+}
+
+func (d dockerLocalhost) Forward(tap, redirect netlink.Link, ifInfo Interface) (ForwardResult, error) {
+	resolvIP, err := getLastSubnetIP(ifInfo.IP, ifInfo.Mask)
+	if err != nil {
+		return ForwardResult{}, fmt.Errorf("could not derive virtual resolver IP: %w", err)
+	}
+	// The guest reuses the container's MAC, so replies must be addressed to it.
+	guestMAC := redirect.Attrs().HardwareAddr
+
+	path, err := exec.LookPath("iptables")
+	if err != nil {
+		return ForwardResult{}, err
+	}
+
+	tcpPort, udpPort, err := dockerDNSPorts(path)
+	if err != nil {
+		return ForwardResult{}, err
+	}
+	netlog.Debugf("Docker DNS resolver: tcp/%s udp/%s", tcpPort, udpPort)
+
+	if err := addDNATRule(path, resolvIP, "udp", udpPort); err != nil {
+		return ForwardResult{}, err
+	}
+	if err := addDNATRule(path, resolvIP, "tcp", tcpPort); err != nil {
+		return ForwardResult{}, err
+	}
+	netlog.Debug("Applied custom Docker iptables rules for NAT")
+
+	loopback, err := netlink.LinkByName("lo")
+	if err != nil {
+		return ForwardResult{}, err
+	}
+	if err := addClsactQdisc(loopback); err != nil {
+		return ForwardResult{}, fmt.Errorf("addClsactQdisc(interface=%s) failed: %w", loopback.Attrs().Name, err)
+	}
+	if err := addTapToLoRedirect(tap, resolvIP); err != nil {
+		return ForwardResult{}, fmt.Errorf("addTapToLoRedirect(%s) failed: %w", tap.Attrs().Name, err)
+	}
+	if err := addARPFilter(tap, resolvIP); err != nil {
+		return ForwardResult{}, fmt.Errorf("addARPFilter(%s) failed: %w", tap.Attrs().Name, err)
+	}
+	if err := addLoToTapRedirect(tap, resolvIP, guestMAC); err != nil {
+		return ForwardResult{}, fmt.Errorf("addLoToTapRedirect(%s) failed: %w", tap.Attrs().Name, err)
+	}
+
+	// FIXME: This adds resolvIP as a static IP on the tap in the host namespace
+	// just for ARP replies, which breaks with more than one tap in the netns.
+	// Alternatives: eBPF or ... ?
+	addr := &netlink.Addr{IPNet: &net.IPNet{IP: resolvIP, Mask: net.CIDRMask(32, 32)}}
+	if err := netlink.AddrReplace(tap, addr); err != nil {
+		return ForwardResult{}, err
+	}
+
+	return ForwardResult{DNSServer: resolvIP, ResolvConfPath: d.resolvConfPath}, nil
+}
+
+func dockerDNSPorts(path string) (tcpPort, udpPort string, err error) {
+	var stdout, stderr bytes.Buffer
+	cmd := exec.Cmd{
+		Path:   path,
+		Args:   []string{path, "-t", "nat", "-S", "DOCKER_OUTPUT", "--wait", "1"},
+		Stdout: &stdout,
+		Stderr: &stderr,
+	}
+	if err := cmd.Run(); err != nil {
+		if _, ok := err.(*exec.ExitError); ok {
+			return "", "", fmt.Errorf("iptables command %s failed: %s", cmd.String(), stderr.String())
+		}
+		return "", "", err
+	}
+
+	out := stdout.String()
+	if m := dockerDNSTCPRe.FindStringSubmatch(out); m != nil {
+		tcpPort = m[1]
+	}
+	if m := dockerDNSUDPRe.FindStringSubmatch(out); m != nil {
+		udpPort = m[1]
+	}
+	if tcpPort == "" || udpPort == "" {
+		return "", "", fmt.Errorf("could not find Docker DNS ports in DOCKER_OUTPUT chain")
+	}
+	return tcpPort, udpPort, nil
+}
+
+func addDNATRule(path string, resolvIP net.IP, proto, port string) error {
+	var stdout, stderr bytes.Buffer
+	cmd := exec.Cmd{
+		Path: path,
+		Args: []string{
+			path, "-t", "nat", "-A", "PREROUTING",
+			"-i", "lo",
+			"-d", resolvIP.String(),
+			"-p", proto,
+			"-j", "DNAT",
+			"--to-destination", "127.0.0.11:" + port,
+			"--wait", "1",
+		},
+		Stdout: &stdout,
+		Stderr: &stderr,
+	}
+	if err := cmd.Run(); err != nil {
+		if _, ok := err.(*exec.ExitError); ok {
+			netlog.Debugf("Iptables error %s", err.Error())
+			return fmt.Errorf("iptables command %s failed: %s", cmd.String(), stderr.String())
+		}
+		return err
+	}
+	return nil
+}
+
+// getLastSubnetIP returns the last usable host of the subnet ip/mask belongs to
+// (broadcast - 1).
+// e.g. 172.23.0.5 + 255.255.0.0 -> 172.23.255.254
+func getLastSubnetIP(ip, mask string) (net.IP, error) {
+	ipv4 := net.ParseIP(ip).To4()
+	if ipv4 == nil {
+		return net.IP{}, fmt.Errorf("invalid IPv4 address %q", ip)
+	}
+	m := net.IPMask(net.ParseIP(mask).To4())
+	if m == nil {
+		return net.IP{}, fmt.Errorf("invalid mask %q", mask)
+	}
+
+	last := make(net.IP, net.IPv4len)
+	for i := range net.IPv4len {
+		last[i] = ipv4[i] | ^m[i] // broadcast
+	}
+	// broadcast - 1, with borrow
+	for i := net.IPv4len - 1; i >= 0; i-- {
+		if last[i]--; last[i] != 0xff {
+			break
+		}
+	}
+	return last, nil
+}
+
+func uint16Ptr(v uint16) *uint16 {
+	return &v
+}
+
+func addClsactQdisc(link netlink.Link) error {
+	clsact := &netlink.Clsact{
+		QdiscAttrs: netlink.QdiscAttrs{
+			LinkIndex: link.Attrs().Index,
+			Parent:    netlink.HANDLE_CLSACT,
+		},
+	}
+	return netlink.QdiscAdd((clsact))
+}
+
+func addARPFilter(source netlink.Link, resolvIP net.IP) error {
+	v4 := resolvIP.To4()
+	if v4 == nil {
+		return fmt.Errorf("addARPFilter: %q is not an IPv4 address", resolvIP)
+	}
+
+	//	tc filter add dev <tapX> ingress protocol arp pref 50 \
+	//	   u32 match u32 <resolvIP> 0xffffffff at 24 action pass
+	return netlink.FilterAdd(&netlink.U32{
+		FilterAttrs: netlink.FilterAttrs{
+			LinkIndex: source.Attrs().Index,
+			Parent:    netlink.MakeHandle(0xffff, 0),
+			Priority:  50,
+			Protocol:  unix.ETH_P_ARP,
+		},
+		Sel: &netlink.TcU32Sel{
+			Nkeys: 1,
+			Flags: netlink.TC_U32_TERMINAL,
+			Keys: []netlink.TcU32Key{{
+				Val:  binary.BigEndian.Uint32(v4),
+				Mask: 0xffffffff,
+				Off:  24, // ARP target protocol address (TIP)
+			}},
+		},
+		Actions: []netlink.Action{
+			&netlink.GenericAction{
+				ActionAttrs: netlink.ActionAttrs{
+					Action: netlink.TC_ACT_OK, // "action pass"
+				},
+			},
+		},
+	})
+}
+
+func addTapToLoRedirect(source netlink.Link, resolvIP net.IP) error {
+
+	//	tc filter add dev <tapX> ingress protocol ip pref x \
+	//	    u32 match ip dst <resolvIP> \
+	//	    action skbedit ptype host pipe \
+	//	    action mirred ingress redirect dev lo
+
+	return netlink.FilterAdd(&netlink.Flower{
+		FilterAttrs: netlink.FilterAttrs{
+			LinkIndex: source.Attrs().Index,
+			Parent:    netlink.MakeHandle(0xffff, 0),
+			Priority:  100,
+			Protocol:  unix.ETH_P_IP,
+		},
+		DestIP: resolvIP,
+		Actions: []netlink.Action{
+			&netlink.SkbEditAction{
+				ActionAttrs: netlink.ActionAttrs{
+					Action: netlink.TC_ACT_PIPE,
+				},
+				PType: uint16Ptr(unix.PACKET_HOST),
+			}, &netlink.MirredAction{
+				ActionAttrs: netlink.ActionAttrs{
+					Action: netlink.TC_ACT_STOLEN,
+				},
+				MirredAction: netlink.TCA_INGRESS_REDIR,
+				Ifindex:      1, // LOOPBACK
+			},
+		},
+	})
+}
+
+func addLoToTapRedirect(target netlink.Link, resolvIP net.IP, guestMAC net.HardwareAddr) error {
+	v4 := resolvIP.To4()
+	if v4 == nil {
+		return fmt.Errorf("addLoToTapRedirect: %q is not an IPv4 address", resolvIP)
+	}
+
+	//	tc filter add dev lo egress protocol ip pref x \
+	//	    u32 match ip src <resolvIP> \
+	//	    action pedit ex munge eth dst set <guestMAC> pipe \
+	//	    action mirred egress redirect dev <tapX>
+
+	return netlink.FilterAdd(&netlink.Flower{
+		FilterAttrs: netlink.FilterAttrs{
+			LinkIndex: 1, // LOOPBACK
+			Parent:    netlink.HANDLE_MIN_EGRESS,
+			Priority:  100,
+			Protocol:  unix.ETH_P_IP,
+		},
+		SrcIP: resolvIP,
+		Actions: []netlink.Action{
+			&netlink.PeditAction{
+				ActionAttrs: netlink.ActionAttrs{
+					Action: netlink.TC_ACT_PIPE,
+				},
+				DstMacAddr: guestMAC,
+			},
+			&netlink.MirredAction{
+				ActionAttrs: netlink.ActionAttrs{
+					Action: netlink.TC_ACT_STOLEN,
+				},
+				MirredAction: netlink.TCA_EGRESS_REDIR,
+				Ifindex:      target.Attrs().Index,
+			},
+		},
+	})
+}

pkg/network/network.go

@@ -34,11 +34,13 @@ const (
 var netlog = logrus.WithField("subsystem", "network")
 
 type UnikernelNetworkInfo struct {
-	TapDevice string
-	EthDevice Interface
+	TapDevice      string
+	EthDevice      Interface
+	DNSServer      string
+	ResolvConfPath string
 }
 type Manager interface {
-	NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error)
+	NetworkSetup(uid uint32, gid uint32, fwd LocalhostForwarder) (*UnikernelNetworkInfo, error)
 }
 
 type Interface struct {