diff --git a/.github/contributors.yaml b/.github/contributors.yaml index 411f9f42..400b3b8d 100644 --- a/.github/contributors.yaml +++ b/.github/contributors.yaml @@ -107,3 +107,6 @@ users: Chennamma-Hotkar: name: Chennamma Hotkar email: channuhotkar@gmail.com + alimx07: + name: Ali Mohamed + email: amx746@gmail.com diff --git a/pkg/network/localhost.go b/pkg/network/localhost.go new file mode 100644 index 00000000..5baeefd3 --- /dev/null +++ b/pkg/network/localhost.go @@ -0,0 +1,361 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package network + +import ( + "bytes" + "encoding/binary" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +type LocalhostForwarder interface { + Name() string + // Forward installs the host-side plumbing (tc filters + customRules) that exposes + // the loopback service to the guest. + Forward(tap, redirect netlink.Link, ifInfo Interface) (ForwardResult, error) +} + +type ForwardResult struct { + DNSServer net.IP + ResolvConfPath string +} + +const ( + dockerCustomNetDNS = "127.0.0.11" +) + +// localhostForwarders holds the ordered set of runtime probes; the first with non-nil return wins +var localhostForwarders = []func(containerID string) (LocalhostForwarder, error){ + // we detect docker only for now. + detectDocker, +} + +func DetectLocalhostForwarder(containerID string) (LocalhostForwarder, error) { + for _, detect := range localhostForwarders { + fwd, err := detect(containerID) + if err != nil { + return nil, err + } + if fwd != nil { + return fwd, nil + } + } + return nil, nil +} + +var ( + // dockerDNSTCPRe and dockerDNSUDPRe extract the tcp/udp ports that Docker's + // embedded DNS resolver (127.0.0.11) listens on. + dockerDNSTCPRe = regexp.MustCompile(`-p\s+tcp.*to-destination\s+127\.0\.0\.11:(\d+)`) + dockerDNSUDPRe = regexp.MustCompile(`-p\s+udp.*to-destination\s+127\.0\.0\.11:(\d+)`) +) + +type dockerLocalhost struct { + resolvConfPath string +} + +func (dockerLocalhost) Name() string { + return "docker" +} + +func detectDocker(containerID string) (LocalhostForwarder, error) { + containerBase := filepath.Join("/var/lib/docker/containers", containerID) + if _, err := os.Stat(containerBase); err != nil { + if os.IsNotExist(err) { + return nil, nil // not a Docker runtime + } + return nil, err + } + + resolvConfPath := filepath.Join(containerBase, "resolv.conf") + data, err := os.ReadFile(resolvConfPath) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("resolv.conf is missing: %w", err) + } + return nil, err + } + + // When we run container in custom user network + // Docker add 127.0.0.11 as the embedded DNS resolver + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == "nameserver" && fields[1] == dockerCustomNetDNS { + return dockerLocalhost{resolvConfPath: resolvConfPath}, nil + } + } + return nil, nil +} + +func (d dockerLocalhost) Forward(tap, redirect netlink.Link, ifInfo Interface) (ForwardResult, error) { + resolvIP, err := getLastSubnetIP(ifInfo.IP, ifInfo.Mask) + if err != nil { + return ForwardResult{}, fmt.Errorf("could not derive virtual resolver IP: %w", err) + } + // The guest reuses the container's MAC, so replies must be addressed to it. + guestMAC := redirect.Attrs().HardwareAddr + + path, err := exec.LookPath("iptables") + if err != nil { + return ForwardResult{}, err + } + + tcpPort, udpPort, err := dockerDNSPorts(path) + if err != nil { + return ForwardResult{}, err + } + netlog.Debugf("Docker DNS resolver: tcp/%s udp/%s", tcpPort, udpPort) + + if err := addDNATRule(path, resolvIP, "udp", udpPort); err != nil { + return ForwardResult{}, err + } + if err := addDNATRule(path, resolvIP, "tcp", tcpPort); err != nil { + return ForwardResult{}, err + } + netlog.Debug("Applied custom Docker iptables rules for NAT") + + loopback, err := netlink.LinkByName("lo") + if err != nil { + return ForwardResult{}, err + } + if err := addClsactQdisc(loopback); err != nil { + return ForwardResult{}, fmt.Errorf("addClsactQdisc(interface=%s) failed: %w", loopback.Attrs().Name, err) + } + if err := addTapToLoRedirect(tap, resolvIP); err != nil { + return ForwardResult{}, fmt.Errorf("addTapToLoRedirect(%s) failed: %w", tap.Attrs().Name, err) + } + if err := addARPFilter(tap, resolvIP); err != nil { + return ForwardResult{}, fmt.Errorf("addARPFilter(%s) failed: %w", tap.Attrs().Name, err) + } + if err := addLoToTapRedirect(tap, resolvIP, guestMAC); err != nil { + return ForwardResult{}, fmt.Errorf("addLoToTapRedirect(%s) failed: %w", tap.Attrs().Name, err) + } + + // FIXME: This adds resolvIP as a static IP on the tap in the host namespace + // just for ARP replies, which breaks with more than one tap in the netns. + // Alternatives: eBPF or ... ? + addr := &netlink.Addr{IPNet: &net.IPNet{IP: resolvIP, Mask: net.CIDRMask(32, 32)}} + if err := netlink.AddrReplace(tap, addr); err != nil { + return ForwardResult{}, err + } + + return ForwardResult{DNSServer: resolvIP, ResolvConfPath: d.resolvConfPath}, nil +} + +func dockerDNSPorts(path string) (tcpPort, udpPort string, err error) { + var stdout, stderr bytes.Buffer + cmd := exec.Cmd{ + Path: path, + Args: []string{path, "-t", "nat", "-S", "DOCKER_OUTPUT", "--wait", "1"}, + Stdout: &stdout, + Stderr: &stderr, + } + if err := cmd.Run(); err != nil { + if _, ok := err.(*exec.ExitError); ok { + return "", "", fmt.Errorf("iptables command %s failed: %s", cmd.String(), stderr.String()) + } + return "", "", err + } + + out := stdout.String() + if m := dockerDNSTCPRe.FindStringSubmatch(out); m != nil { + tcpPort = m[1] + } + if m := dockerDNSUDPRe.FindStringSubmatch(out); m != nil { + udpPort = m[1] + } + if tcpPort == "" || udpPort == "" { + return "", "", fmt.Errorf("could not find Docker DNS ports in DOCKER_OUTPUT chain") + } + return tcpPort, udpPort, nil +} + +func addDNATRule(path string, resolvIP net.IP, proto, port string) error { + var stdout, stderr bytes.Buffer + cmd := exec.Cmd{ + Path: path, + Args: []string{ + path, "-t", "nat", "-A", "PREROUTING", + "-i", "lo", + "-d", resolvIP.String(), + "-p", proto, + "-j", "DNAT", + "--to-destination", "127.0.0.11:" + port, + "--wait", "1", + }, + Stdout: &stdout, + Stderr: &stderr, + } + if err := cmd.Run(); err != nil { + if _, ok := err.(*exec.ExitError); ok { + netlog.Debugf("Iptables error %s", err.Error()) + return fmt.Errorf("iptables command %s failed: %s", cmd.String(), stderr.String()) + } + return err + } + return nil +} + +// getLastSubnetIP returns the last usable host of the subnet ip/mask belongs to +// (broadcast - 1). +// e.g. 172.23.0.5 + 255.255.0.0 -> 172.23.255.254 +func getLastSubnetIP(ip, mask string) (net.IP, error) { + ipv4 := net.ParseIP(ip).To4() + if ipv4 == nil { + return net.IP{}, fmt.Errorf("invalid IPv4 address %q", ip) + } + m := net.IPMask(net.ParseIP(mask).To4()) + if m == nil { + return net.IP{}, fmt.Errorf("invalid mask %q", mask) + } + + last := make(net.IP, net.IPv4len) + for i := range net.IPv4len { + last[i] = ipv4[i] | ^m[i] // broadcast + } + // broadcast - 1, with borrow + for i := net.IPv4len - 1; i >= 0; i-- { + if last[i]--; last[i] != 0xff { + break + } + } + return last, nil +} + +func uint16Ptr(v uint16) *uint16 { + return &v +} + +func addClsactQdisc(link netlink.Link) error { + clsact := &netlink.Clsact{ + QdiscAttrs: netlink.QdiscAttrs{ + LinkIndex: link.Attrs().Index, + Parent: netlink.HANDLE_CLSACT, + }, + } + return netlink.QdiscAdd((clsact)) +} + +func addARPFilter(source netlink.Link, resolvIP net.IP) error { + v4 := resolvIP.To4() + if v4 == nil { + return fmt.Errorf("addARPFilter: %q is not an IPv4 address", resolvIP) + } + + // tc filter add dev ingress protocol arp pref 50 \ + // u32 match u32 0xffffffff at 24 action pass + return netlink.FilterAdd(&netlink.U32{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: source.Attrs().Index, + Parent: netlink.MakeHandle(0xffff, 0), + Priority: 50, + Protocol: unix.ETH_P_ARP, + }, + Sel: &netlink.TcU32Sel{ + Nkeys: 1, + Flags: netlink.TC_U32_TERMINAL, + Keys: []netlink.TcU32Key{{ + Val: binary.BigEndian.Uint32(v4), + Mask: 0xffffffff, + Off: 24, // ARP target protocol address (TIP) + }}, + }, + Actions: []netlink.Action{ + &netlink.GenericAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_OK, // "action pass" + }, + }, + }, + }) +} + +func addTapToLoRedirect(source netlink.Link, resolvIP net.IP) error { + + // tc filter add dev ingress protocol ip pref x \ + // u32 match ip dst \ + // action skbedit ptype host pipe \ + // action mirred ingress redirect dev lo + + return netlink.FilterAdd(&netlink.Flower{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: source.Attrs().Index, + Parent: netlink.MakeHandle(0xffff, 0), + Priority: 100, + Protocol: unix.ETH_P_IP, + }, + DestIP: resolvIP, + Actions: []netlink.Action{ + &netlink.SkbEditAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_PIPE, + }, + PType: uint16Ptr(unix.PACKET_HOST), + }, &netlink.MirredAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_STOLEN, + }, + MirredAction: netlink.TCA_INGRESS_REDIR, + Ifindex: 1, // LOOPBACK + }, + }, + }) +} + +func addLoToTapRedirect(target netlink.Link, resolvIP net.IP, guestMAC net.HardwareAddr) error { + v4 := resolvIP.To4() + if v4 == nil { + return fmt.Errorf("addLoToTapRedirect: %q is not an IPv4 address", resolvIP) + } + + // tc filter add dev lo egress protocol ip pref x \ + // u32 match ip src \ + // action pedit ex munge eth dst set pipe \ + // action mirred egress redirect dev + + return netlink.FilterAdd(&netlink.Flower{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: 1, // LOOPBACK + Parent: netlink.HANDLE_MIN_EGRESS, + Priority: 100, + Protocol: unix.ETH_P_IP, + }, + SrcIP: resolvIP, + Actions: []netlink.Action{ + &netlink.PeditAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_PIPE, + }, + DstMacAddr: guestMAC, + }, + &netlink.MirredAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_STOLEN, + }, + MirredAction: netlink.TCA_EGRESS_REDIR, + Ifindex: target.Attrs().Index, + }, + }, + }) +} diff --git a/pkg/network/network.go b/pkg/network/network.go index 6fb18c6c..e6a14119 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -34,11 +34,13 @@ const ( var netlog = logrus.WithField("subsystem", "network") type UnikernelNetworkInfo struct { - TapDevice string - EthDevice Interface + TapDevice string + EthDevice Interface + DNSServer string + ResolvConfPath string } type Manager interface { - NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) + NetworkSetup(uid uint32, gid uint32, fwd LocalhostForwarder) (*UnikernelNetworkInfo, error) } type Interface struct { diff --git a/pkg/network/network_dynamic.go b/pkg/network/network_dynamic.go index 1d6a2237..c31b617e 100644 --- a/pkg/network/network_dynamic.go +++ b/pkg/network/network_dynamic.go @@ -32,7 +32,7 @@ type DynamicNetwork struct { // FIXME: CUrrently only one tap device per netns can provide functional networking. We need to find a proper way to handle networking // for multiple unikernels in the same pod/network namespace. // See: https://github.com/urunc-dev/urunc/issues/13 -func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) { +func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32, fwd LocalhostForwarder) (*UnikernelNetworkInfo, error) { tapIndex, err := getTapIndex() if err != nil { return nil, fmt.Errorf("getTapIndex failed: %w", err) @@ -62,8 +62,20 @@ func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkI return nil, fmt.Errorf("getInterfaceInfo(%s) failed: %w", redirectLink.Attrs().Name, err) } - return &UnikernelNetworkInfo{ + info := &UnikernelNetworkInfo{ TapDevice: newTapDevice.Attrs().Name, EthDevice: ifInfo, - }, nil + } + + if fwd != nil { + res, err := fwd.Forward(newTapDevice, redirectLink, ifInfo) + if err != nil { + return nil, fmt.Errorf("%s localhost forwarder failed: %w", fwd.Name(), err) + } + info.DNSServer = res.DNSServer.String() + info.ResolvConfPath = res.ResolvConfPath + netlog.Debugf("%s localhost forwarder applied: resolver IP %s", fwd.Name(), info.DNSServer) + } + + return info, nil } diff --git a/pkg/network/network_static.go b/pkg/network/network_static.go index fa46608c..cb5d13f8 100644 --- a/pkg/network/network_static.go +++ b/pkg/network/network_static.go @@ -88,7 +88,7 @@ func setNATRule(iface string, sourceIP string) error { return nil } -func (n StaticNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) { +func (n StaticNetwork) NetworkSetup(uid uint32, gid uint32, _ LocalhostForwarder) (*UnikernelNetworkInfo, error) { newTapName := strings.ReplaceAll(DefaultTap, "X", "0") addTCRules := false redirectLink, err := discoverContainerIface() diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index c6388e2c..ff26921f 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -45,12 +45,14 @@ type VMM interface { } type NetDevParams struct { - IP string // The veth device IP - Mask string // The veth device mask - Gateway string // The veth device gateway - MAC string // The MAC address of the guest network device - TapDev string // The tap device name - MTU int // The MTU value of the tap device + IP string // The veth device IP + Mask string // The veth device mask + Gateway string // The veth device gateway + MAC string // The MAC address of the guest network device + TapDev string // The tap device name + MTU int // The MTU value of the tap device + DNSServer string // DNSServer is the virtual resolver IP if a DNS fixup installed for the guest + ResolvConfPath string // ResolvConfPath is the host path of the resolv.conf the guest reads. Empty when not applicable. } type BlockDevParams struct { diff --git a/pkg/unikontainers/unikernels/linux.go b/pkg/unikontainers/unikernels/linux.go index d6bb8095..d656b78b 100644 --- a/pkg/unikontainers/unikernels/linux.go +++ b/pkg/unikontainers/unikernels/linux.go @@ -17,6 +17,7 @@ package unikernels import ( "fmt" "net" + "os" "path/filepath" "runtime" "strconv" @@ -215,6 +216,12 @@ func (l *Linux) Init(data types.UnikernelParams) error { } l.configureNetwork(data.Net) + // rewrite the guest's resolv.conf to point at virtual resolver if needed. + if data.Net.DNSServer != "" && data.Net.ResolvConfPath != "" { + if err := writeResolvConf(data.Net.ResolvConfPath, data.Net.DNSServer); err != nil { + return fmt.Errorf("failed to configure guest DNS: %w", err) + } + } l.Blk = data.Block l.RootFsType = data.Rootfs.Type l.Env = data.EnvVars @@ -271,6 +278,32 @@ func (l *Linux) configureNetwork(net types.NetDevParams) { l.Net.Mask = net.Mask } +func writeResolvConf(path, resolvIP string) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read %s: %w", path, err) + } + + lines := strings.Split(string(data), "\n") + out := make([]string, 0, len(lines)+1) + + // TODO: this assummes we have one nameserver that needs to rewrite + // which is the case for Docker. Could we need to preserve + // more than one nameserver in other cases ? + for _, line := range lines { + if strings.HasPrefix(strings.TrimSpace(line), "nameserver") { + out = append(out, "nameserver "+resolvIP) + continue + } + out = append(out, line) + } + + if err := os.WriteFile(path, []byte(strings.Join(out, "\n")), 0o644); err != nil { //nolint: gosec + return fmt.Errorf("failed to write %s: %w", path, err) + } + return nil +} + // setupUrunitConfig creates the urunit configuration file with environment variables. func (l *Linux) setupUrunitConfig(rfs types.RootfsParams) error { urunitConfig := l.buildUrunitConfig() diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index fc297f93..3236d936 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -199,7 +199,7 @@ func (u *Unikontainer) SetRunningState() error { return u.saveContainerState() } -func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { +func (u *Unikontainer) SetupNet(fwd network.LocalhostForwarder) (types.NetDevParams, error) { networkType := u.getNetworkType() uniklog.WithField("network type", networkType).Debug("Retrieved network type") netArgs := types.NetDevParams{} @@ -207,8 +207,7 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { if err != nil { return netArgs, fmt.Errorf("failed to create network manager for %s type: %v", networkType, err) } - - networkInfo, err := netManager.NetworkSetup(u.Spec.Process.User.UID, u.Spec.Process.User.GID) + networkInfo, err := netManager.NetworkSetup(u.Spec.Process.User.UID, u.Spec.Process.User.GID, fwd) if err != nil { // TODO: Handle this case better. We do not need to show an error // since there was no network in the container. Therefore, we @@ -226,6 +225,8 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { // virtual ethernet interface inside the namespace netArgs.MAC = networkInfo.EthDevice.MAC netArgs.MTU = networkInfo.EthDevice.MTU + netArgs.DNSServer = networkInfo.DNSServer + netArgs.ResolvConfPath = networkInfo.ResolvConfPath } return netArgs, nil @@ -401,8 +402,15 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { unikernelParams.CmdLine = strings.Fields(u.State.Annotations[annotCmdLine]) } + // Detect whether we need tap exposes services on the loopback that + // the guest needs reached. + fwd, err := network.DetectLocalhostForwarder(u.State.ID) + if err != nil { + uniklog.Errorf("failed to detect localhost forwarder: %v", err) + } + // handle network - netArgs, err := u.SetupNet() + netArgs, err := u.SetupNet(fwd) if err != nil { uniklog.Errorf("failed to setup network: %v", err) return err