From 8fd09b6412043fedbdada1c4615962f572e2aa92 Mon Sep 17 00:00:00 2001 From: Ali Mohamed Date: Sun, 28 Jun 2026 10:43:53 +0300 Subject: [PATCH 1/2] feat(network): forward guest DNS to Docker's embedded resolver On Docker user custom networks the DNS resolver (127.0.0.11) is loopback only and unreachable by the unikernel guest. Detect this case per container, expose the resolver via a virtual resolver IP using tc redirects between the tap and lo + a PREROUTING DNAT, and rewrite the guest's resolv.conf to point at that IP. Signed-off-by: Ali Mohamed --- pkg/network/localhost.go | 361 ++++++++++++++++++++++++++ pkg/network/network.go | 8 +- pkg/network/network_dynamic.go | 18 +- pkg/network/network_static.go | 2 +- pkg/unikontainers/types/types.go | 14 +- pkg/unikontainers/unikernels/linux.go | 33 +++ pkg/unikontainers/unikontainers.go | 16 +- 7 files changed, 435 insertions(+), 17 deletions(-) create mode 100644 pkg/network/localhost.go diff --git a/pkg/network/localhost.go b/pkg/network/localhost.go new file mode 100644 index 00000000..5baeefd3 --- /dev/null +++ b/pkg/network/localhost.go @@ -0,0 +1,361 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package network + +import ( + "bytes" + "encoding/binary" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +type LocalhostForwarder interface { + Name() string + // Forward installs the host-side plumbing (tc filters + customRules) that exposes + // the loopback service to the guest. + Forward(tap, redirect netlink.Link, ifInfo Interface) (ForwardResult, error) +} + +type ForwardResult struct { + DNSServer net.IP + ResolvConfPath string +} + +const ( + dockerCustomNetDNS = "127.0.0.11" +) + +// localhostForwarders holds the ordered set of runtime probes; the first with non-nil return wins +var localhostForwarders = []func(containerID string) (LocalhostForwarder, error){ + // we detect docker only for now. + detectDocker, +} + +func DetectLocalhostForwarder(containerID string) (LocalhostForwarder, error) { + for _, detect := range localhostForwarders { + fwd, err := detect(containerID) + if err != nil { + return nil, err + } + if fwd != nil { + return fwd, nil + } + } + return nil, nil +} + +var ( + // dockerDNSTCPRe and dockerDNSUDPRe extract the tcp/udp ports that Docker's + // embedded DNS resolver (127.0.0.11) listens on. + dockerDNSTCPRe = regexp.MustCompile(`-p\s+tcp.*to-destination\s+127\.0\.0\.11:(\d+)`) + dockerDNSUDPRe = regexp.MustCompile(`-p\s+udp.*to-destination\s+127\.0\.0\.11:(\d+)`) +) + +type dockerLocalhost struct { + resolvConfPath string +} + +func (dockerLocalhost) Name() string { + return "docker" +} + +func detectDocker(containerID string) (LocalhostForwarder, error) { + containerBase := filepath.Join("/var/lib/docker/containers", containerID) + if _, err := os.Stat(containerBase); err != nil { + if os.IsNotExist(err) { + return nil, nil // not a Docker runtime + } + return nil, err + } + + resolvConfPath := filepath.Join(containerBase, "resolv.conf") + data, err := os.ReadFile(resolvConfPath) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("resolv.conf is missing: %w", err) + } + return nil, err + } + + // When we run container in custom user network + // Docker add 127.0.0.11 as the embedded DNS resolver + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == "nameserver" && fields[1] == dockerCustomNetDNS { + return dockerLocalhost{resolvConfPath: resolvConfPath}, nil + } + } + return nil, nil +} + +func (d dockerLocalhost) Forward(tap, redirect netlink.Link, ifInfo Interface) (ForwardResult, error) { + resolvIP, err := getLastSubnetIP(ifInfo.IP, ifInfo.Mask) + if err != nil { + return ForwardResult{}, fmt.Errorf("could not derive virtual resolver IP: %w", err) + } + // The guest reuses the container's MAC, so replies must be addressed to it. + guestMAC := redirect.Attrs().HardwareAddr + + path, err := exec.LookPath("iptables") + if err != nil { + return ForwardResult{}, err + } + + tcpPort, udpPort, err := dockerDNSPorts(path) + if err != nil { + return ForwardResult{}, err + } + netlog.Debugf("Docker DNS resolver: tcp/%s udp/%s", tcpPort, udpPort) + + if err := addDNATRule(path, resolvIP, "udp", udpPort); err != nil { + return ForwardResult{}, err + } + if err := addDNATRule(path, resolvIP, "tcp", tcpPort); err != nil { + return ForwardResult{}, err + } + netlog.Debug("Applied custom Docker iptables rules for NAT") + + loopback, err := netlink.LinkByName("lo") + if err != nil { + return ForwardResult{}, err + } + if err := addClsactQdisc(loopback); err != nil { + return ForwardResult{}, fmt.Errorf("addClsactQdisc(interface=%s) failed: %w", loopback.Attrs().Name, err) + } + if err := addTapToLoRedirect(tap, resolvIP); err != nil { + return ForwardResult{}, fmt.Errorf("addTapToLoRedirect(%s) failed: %w", tap.Attrs().Name, err) + } + if err := addARPFilter(tap, resolvIP); err != nil { + return ForwardResult{}, fmt.Errorf("addARPFilter(%s) failed: %w", tap.Attrs().Name, err) + } + if err := addLoToTapRedirect(tap, resolvIP, guestMAC); err != nil { + return ForwardResult{}, fmt.Errorf("addLoToTapRedirect(%s) failed: %w", tap.Attrs().Name, err) + } + + // FIXME: This adds resolvIP as a static IP on the tap in the host namespace + // just for ARP replies, which breaks with more than one tap in the netns. + // Alternatives: eBPF or ... ? + addr := &netlink.Addr{IPNet: &net.IPNet{IP: resolvIP, Mask: net.CIDRMask(32, 32)}} + if err := netlink.AddrReplace(tap, addr); err != nil { + return ForwardResult{}, err + } + + return ForwardResult{DNSServer: resolvIP, ResolvConfPath: d.resolvConfPath}, nil +} + +func dockerDNSPorts(path string) (tcpPort, udpPort string, err error) { + var stdout, stderr bytes.Buffer + cmd := exec.Cmd{ + Path: path, + Args: []string{path, "-t", "nat", "-S", "DOCKER_OUTPUT", "--wait", "1"}, + Stdout: &stdout, + Stderr: &stderr, + } + if err := cmd.Run(); err != nil { + if _, ok := err.(*exec.ExitError); ok { + return "", "", fmt.Errorf("iptables command %s failed: %s", cmd.String(), stderr.String()) + } + return "", "", err + } + + out := stdout.String() + if m := dockerDNSTCPRe.FindStringSubmatch(out); m != nil { + tcpPort = m[1] + } + if m := dockerDNSUDPRe.FindStringSubmatch(out); m != nil { + udpPort = m[1] + } + if tcpPort == "" || udpPort == "" { + return "", "", fmt.Errorf("could not find Docker DNS ports in DOCKER_OUTPUT chain") + } + return tcpPort, udpPort, nil +} + +func addDNATRule(path string, resolvIP net.IP, proto, port string) error { + var stdout, stderr bytes.Buffer + cmd := exec.Cmd{ + Path: path, + Args: []string{ + path, "-t", "nat", "-A", "PREROUTING", + "-i", "lo", + "-d", resolvIP.String(), + "-p", proto, + "-j", "DNAT", + "--to-destination", "127.0.0.11:" + port, + "--wait", "1", + }, + Stdout: &stdout, + Stderr: &stderr, + } + if err := cmd.Run(); err != nil { + if _, ok := err.(*exec.ExitError); ok { + netlog.Debugf("Iptables error %s", err.Error()) + return fmt.Errorf("iptables command %s failed: %s", cmd.String(), stderr.String()) + } + return err + } + return nil +} + +// getLastSubnetIP returns the last usable host of the subnet ip/mask belongs to +// (broadcast - 1). +// e.g. 172.23.0.5 + 255.255.0.0 -> 172.23.255.254 +func getLastSubnetIP(ip, mask string) (net.IP, error) { + ipv4 := net.ParseIP(ip).To4() + if ipv4 == nil { + return net.IP{}, fmt.Errorf("invalid IPv4 address %q", ip) + } + m := net.IPMask(net.ParseIP(mask).To4()) + if m == nil { + return net.IP{}, fmt.Errorf("invalid mask %q", mask) + } + + last := make(net.IP, net.IPv4len) + for i := range net.IPv4len { + last[i] = ipv4[i] | ^m[i] // broadcast + } + // broadcast - 1, with borrow + for i := net.IPv4len - 1; i >= 0; i-- { + if last[i]--; last[i] != 0xff { + break + } + } + return last, nil +} + +func uint16Ptr(v uint16) *uint16 { + return &v +} + +func addClsactQdisc(link netlink.Link) error { + clsact := &netlink.Clsact{ + QdiscAttrs: netlink.QdiscAttrs{ + LinkIndex: link.Attrs().Index, + Parent: netlink.HANDLE_CLSACT, + }, + } + return netlink.QdiscAdd((clsact)) +} + +func addARPFilter(source netlink.Link, resolvIP net.IP) error { + v4 := resolvIP.To4() + if v4 == nil { + return fmt.Errorf("addARPFilter: %q is not an IPv4 address", resolvIP) + } + + // tc filter add dev ingress protocol arp pref 50 \ + // u32 match u32 0xffffffff at 24 action pass + return netlink.FilterAdd(&netlink.U32{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: source.Attrs().Index, + Parent: netlink.MakeHandle(0xffff, 0), + Priority: 50, + Protocol: unix.ETH_P_ARP, + }, + Sel: &netlink.TcU32Sel{ + Nkeys: 1, + Flags: netlink.TC_U32_TERMINAL, + Keys: []netlink.TcU32Key{{ + Val: binary.BigEndian.Uint32(v4), + Mask: 0xffffffff, + Off: 24, // ARP target protocol address (TIP) + }}, + }, + Actions: []netlink.Action{ + &netlink.GenericAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_OK, // "action pass" + }, + }, + }, + }) +} + +func addTapToLoRedirect(source netlink.Link, resolvIP net.IP) error { + + // tc filter add dev ingress protocol ip pref x \ + // u32 match ip dst \ + // action skbedit ptype host pipe \ + // action mirred ingress redirect dev lo + + return netlink.FilterAdd(&netlink.Flower{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: source.Attrs().Index, + Parent: netlink.MakeHandle(0xffff, 0), + Priority: 100, + Protocol: unix.ETH_P_IP, + }, + DestIP: resolvIP, + Actions: []netlink.Action{ + &netlink.SkbEditAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_PIPE, + }, + PType: uint16Ptr(unix.PACKET_HOST), + }, &netlink.MirredAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_STOLEN, + }, + MirredAction: netlink.TCA_INGRESS_REDIR, + Ifindex: 1, // LOOPBACK + }, + }, + }) +} + +func addLoToTapRedirect(target netlink.Link, resolvIP net.IP, guestMAC net.HardwareAddr) error { + v4 := resolvIP.To4() + if v4 == nil { + return fmt.Errorf("addLoToTapRedirect: %q is not an IPv4 address", resolvIP) + } + + // tc filter add dev lo egress protocol ip pref x \ + // u32 match ip src \ + // action pedit ex munge eth dst set pipe \ + // action mirred egress redirect dev + + return netlink.FilterAdd(&netlink.Flower{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: 1, // LOOPBACK + Parent: netlink.HANDLE_MIN_EGRESS, + Priority: 100, + Protocol: unix.ETH_P_IP, + }, + SrcIP: resolvIP, + Actions: []netlink.Action{ + &netlink.PeditAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_PIPE, + }, + DstMacAddr: guestMAC, + }, + &netlink.MirredAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_STOLEN, + }, + MirredAction: netlink.TCA_EGRESS_REDIR, + Ifindex: target.Attrs().Index, + }, + }, + }) +} diff --git a/pkg/network/network.go b/pkg/network/network.go index 6fb18c6c..e6a14119 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -34,11 +34,13 @@ const ( var netlog = logrus.WithField("subsystem", "network") type UnikernelNetworkInfo struct { - TapDevice string - EthDevice Interface + TapDevice string + EthDevice Interface + DNSServer string + ResolvConfPath string } type Manager interface { - NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) + NetworkSetup(uid uint32, gid uint32, fwd LocalhostForwarder) (*UnikernelNetworkInfo, error) } type Interface struct { diff --git a/pkg/network/network_dynamic.go b/pkg/network/network_dynamic.go index 1d6a2237..c31b617e 100644 --- a/pkg/network/network_dynamic.go +++ b/pkg/network/network_dynamic.go @@ -32,7 +32,7 @@ type DynamicNetwork struct { // FIXME: CUrrently only one tap device per netns can provide functional networking. We need to find a proper way to handle networking // for multiple unikernels in the same pod/network namespace. // See: https://github.com/urunc-dev/urunc/issues/13 -func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) { +func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32, fwd LocalhostForwarder) (*UnikernelNetworkInfo, error) { tapIndex, err := getTapIndex() if err != nil { return nil, fmt.Errorf("getTapIndex failed: %w", err) @@ -62,8 +62,20 @@ func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkI return nil, fmt.Errorf("getInterfaceInfo(%s) failed: %w", redirectLink.Attrs().Name, err) } - return &UnikernelNetworkInfo{ + info := &UnikernelNetworkInfo{ TapDevice: newTapDevice.Attrs().Name, EthDevice: ifInfo, - }, nil + } + + if fwd != nil { + res, err := fwd.Forward(newTapDevice, redirectLink, ifInfo) + if err != nil { + return nil, fmt.Errorf("%s localhost forwarder failed: %w", fwd.Name(), err) + } + info.DNSServer = res.DNSServer.String() + info.ResolvConfPath = res.ResolvConfPath + netlog.Debugf("%s localhost forwarder applied: resolver IP %s", fwd.Name(), info.DNSServer) + } + + return info, nil } diff --git a/pkg/network/network_static.go b/pkg/network/network_static.go index fa46608c..cb5d13f8 100644 --- a/pkg/network/network_static.go +++ b/pkg/network/network_static.go @@ -88,7 +88,7 @@ func setNATRule(iface string, sourceIP string) error { return nil } -func (n StaticNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) { +func (n StaticNetwork) NetworkSetup(uid uint32, gid uint32, _ LocalhostForwarder) (*UnikernelNetworkInfo, error) { newTapName := strings.ReplaceAll(DefaultTap, "X", "0") addTCRules := false redirectLink, err := discoverContainerIface() diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index c6388e2c..ff26921f 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -45,12 +45,14 @@ type VMM interface { } type NetDevParams struct { - IP string // The veth device IP - Mask string // The veth device mask - Gateway string // The veth device gateway - MAC string // The MAC address of the guest network device - TapDev string // The tap device name - MTU int // The MTU value of the tap device + IP string // The veth device IP + Mask string // The veth device mask + Gateway string // The veth device gateway + MAC string // The MAC address of the guest network device + TapDev string // The tap device name + MTU int // The MTU value of the tap device + DNSServer string // DNSServer is the virtual resolver IP if a DNS fixup installed for the guest + ResolvConfPath string // ResolvConfPath is the host path of the resolv.conf the guest reads. Empty when not applicable. } type BlockDevParams struct { diff --git a/pkg/unikontainers/unikernels/linux.go b/pkg/unikontainers/unikernels/linux.go index d6bb8095..d656b78b 100644 --- a/pkg/unikontainers/unikernels/linux.go +++ b/pkg/unikontainers/unikernels/linux.go @@ -17,6 +17,7 @@ package unikernels import ( "fmt" "net" + "os" "path/filepath" "runtime" "strconv" @@ -215,6 +216,12 @@ func (l *Linux) Init(data types.UnikernelParams) error { } l.configureNetwork(data.Net) + // rewrite the guest's resolv.conf to point at virtual resolver if needed. + if data.Net.DNSServer != "" && data.Net.ResolvConfPath != "" { + if err := writeResolvConf(data.Net.ResolvConfPath, data.Net.DNSServer); err != nil { + return fmt.Errorf("failed to configure guest DNS: %w", err) + } + } l.Blk = data.Block l.RootFsType = data.Rootfs.Type l.Env = data.EnvVars @@ -271,6 +278,32 @@ func (l *Linux) configureNetwork(net types.NetDevParams) { l.Net.Mask = net.Mask } +func writeResolvConf(path, resolvIP string) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read %s: %w", path, err) + } + + lines := strings.Split(string(data), "\n") + out := make([]string, 0, len(lines)+1) + + // TODO: this assummes we have one nameserver that needs to rewrite + // which is the case for Docker. Could we need to preserve + // more than one nameserver in other cases ? + for _, line := range lines { + if strings.HasPrefix(strings.TrimSpace(line), "nameserver") { + out = append(out, "nameserver "+resolvIP) + continue + } + out = append(out, line) + } + + if err := os.WriteFile(path, []byte(strings.Join(out, "\n")), 0o644); err != nil { //nolint: gosec + return fmt.Errorf("failed to write %s: %w", path, err) + } + return nil +} + // setupUrunitConfig creates the urunit configuration file with environment variables. func (l *Linux) setupUrunitConfig(rfs types.RootfsParams) error { urunitConfig := l.buildUrunitConfig() diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index fc297f93..3236d936 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -199,7 +199,7 @@ func (u *Unikontainer) SetRunningState() error { return u.saveContainerState() } -func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { +func (u *Unikontainer) SetupNet(fwd network.LocalhostForwarder) (types.NetDevParams, error) { networkType := u.getNetworkType() uniklog.WithField("network type", networkType).Debug("Retrieved network type") netArgs := types.NetDevParams{} @@ -207,8 +207,7 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { if err != nil { return netArgs, fmt.Errorf("failed to create network manager for %s type: %v", networkType, err) } - - networkInfo, err := netManager.NetworkSetup(u.Spec.Process.User.UID, u.Spec.Process.User.GID) + networkInfo, err := netManager.NetworkSetup(u.Spec.Process.User.UID, u.Spec.Process.User.GID, fwd) if err != nil { // TODO: Handle this case better. We do not need to show an error // since there was no network in the container. Therefore, we @@ -226,6 +225,8 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { // virtual ethernet interface inside the namespace netArgs.MAC = networkInfo.EthDevice.MAC netArgs.MTU = networkInfo.EthDevice.MTU + netArgs.DNSServer = networkInfo.DNSServer + netArgs.ResolvConfPath = networkInfo.ResolvConfPath } return netArgs, nil @@ -401,8 +402,15 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { unikernelParams.CmdLine = strings.Fields(u.State.Annotations[annotCmdLine]) } + // Detect whether we need tap exposes services on the loopback that + // the guest needs reached. + fwd, err := network.DetectLocalhostForwarder(u.State.ID) + if err != nil { + uniklog.Errorf("failed to detect localhost forwarder: %v", err) + } + // handle network - netArgs, err := u.SetupNet() + netArgs, err := u.SetupNet(fwd) if err != nil { uniklog.Errorf("failed to setup network: %v", err) return err From a743f29a2234a2cce100ac084dfb54fa81e95381 Mon Sep 17 00:00:00 2001 From: Ali Mohamed Date: Sun, 28 Jun 2026 10:43:53 +0300 Subject: [PATCH 2/2] chore(contributors): add Ali Mohamed Signed-off-by: Ali Mohamed --- .github/contributors.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/contributors.yaml b/.github/contributors.yaml index 411f9f42..400b3b8d 100644 --- a/.github/contributors.yaml +++ b/.github/contributors.yaml @@ -107,3 +107,6 @@ users: Chennamma-Hotkar: name: Chennamma Hotkar email: channuhotkar@gmail.com + alimx07: + name: Ali Mohamed + email: amx746@gmail.com