//go:build linux // +build linux package integration import ( "bytes" "fmt" "os" "os/exec" "testing" "github.com/stretchr/testify/require" "github.com/tailscale/netlink" "golang.org/x/xerrors" "tailscale.com/tailcfg" "cdr.dev/slog" "github.com/coder/coder/v2/cryptorand" ) const ( client1Port = 48001 client1RouterPort = 48011 // used in easy and hard NAT client1RouterPortSTUN = 48201 // used in hard NAT client2Port = 48002 client2RouterPort = 48012 // used in easy and hard NAT client2RouterPortSTUN = 48101 // used in hard NAT ) type TestNetworking struct { Server TestNetworkingServer STUNs []TestNetworkingSTUN Client1 TestNetworkingClient Client2 TestNetworkingClient } type TestNetworkingServer struct { Process TestNetworkingProcess ListenAddr string } type TestNetworkingSTUN struct { Process TestNetworkingProcess IP string ListenAddr string } type TestNetworkingClient struct { Process TestNetworkingProcess // ServerAccessURL is the hostname and port that the client uses to access // the server over HTTP for coordination. ServerAccessURL string // DERPMap is the DERP map that the client uses. If nil, a basic DERP map // containing only a single DERP with `ServerAccessURL` is used with no // STUN servers. DERPMap *tailcfg.DERPMap } func (c TestNetworkingClient) ResolveDERPMap() (*tailcfg.DERPMap, error) { if c.DERPMap != nil { return c.DERPMap, nil } return basicDERPMap(c.ServerAccessURL) } type TestNetworkingProcess struct { // NetNS to enter. If nil, the current network namespace is used. NetNS *os.File } // SetupNetworkingLoopback creates a network namespace with a loopback interface // for all tests to share. This is the simplest networking setup. The network // namespace only exists for isolation on the host and doesn't serve any routing // purpose. func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking { // Create a single network namespace for all tests so we can have an // isolated loopback interface. netNSFile := createNetNS(t, uniqNetName(t)) var ( listenAddr = "127.0.0.1:8080" process = TestNetworkingProcess{ NetNS: netNSFile, } ) return TestNetworking{ Server: TestNetworkingServer{ Process: process, ListenAddr: listenAddr, }, Client1: TestNetworkingClient{ Process: process, ServerAccessURL: "http://" + listenAddr, }, Client2: TestNetworkingClient{ Process: process, ServerAccessURL: "http://" + listenAddr, }, } } func easyNAT(t *testing.T) fakeInternet { internet := createFakeInternet(t) _, err := commandInNetNS(internet.BridgeNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output() require.NoError(t, wrapExitErr(err), "enable IP forwarding in bridge NetNS") // Set up iptables masquerade rules to allow each router to NAT packets. leaves := []struct { fakeRouterLeaf clientPort int natPort int }{ {internet.Client1, client1Port, client1RouterPort}, {internet.Client2, client2Port, client2RouterPort}, } for _, leaf := range leaves { _, err := commandInNetNS(leaf.RouterNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output() require.NoError(t, wrapExitErr(err), "enable IP forwarding in router NetNS") // All non-UDP traffic should use regular masquerade e.g. for HTTP. _, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{ "-t", "nat", "-A", "POSTROUTING", // Every interface except loopback. "!", "-o", "lo", // Every protocol except UDP. "!", "-p", "udp", "-j", "MASQUERADE", }).Output() require.NoError(t, wrapExitErr(err), "add iptables non-UDP masquerade rule") // Outgoing traffic should get NATed to the router's IP. _, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{ "-t", "nat", "-A", "POSTROUTING", "-p", "udp", "--sport", fmt.Sprint(leaf.clientPort), "-j", "SNAT", "--to-source", fmt.Sprintf("%s:%d", leaf.RouterIP, leaf.natPort), }).Output() require.NoError(t, wrapExitErr(err), "add iptables SNAT rule") // Incoming traffic should be forwarded to the client's IP. _, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{ "-t", "nat", "-A", "PREROUTING", "-p", "udp", "--dport", fmt.Sprint(leaf.natPort), "-j", "DNAT", "--to-destination", fmt.Sprintf("%s:%d", leaf.ClientIP, leaf.clientPort), }).Output() require.NoError(t, wrapExitErr(err), "add iptables DNAT rule") } return internet } // SetupNetworkingEasyNAT creates a fake internet and sets up "easy NAT" // forwarding rules. // See createFakeInternet. // NAT is achieved through a single iptables masquerade rule. func SetupNetworkingEasyNAT(t *testing.T, _ slog.Logger) TestNetworking { return easyNAT(t).Net } // SetupNetworkingEasyNATWithSTUN does the same as SetupNetworkingEasyNAT, but // also creates a namespace and bridge address for a STUN server. func SetupNetworkingEasyNATWithSTUN(t *testing.T, _ slog.Logger) TestNetworking { internet := easyNAT(t) internet.Net.STUNs = []TestNetworkingSTUN{ prepareSTUNServer(t, &internet, 0), } return internet.Net } // hardNAT creates a fake internet with multiple STUN servers and sets up "hard // NAT" forwarding rules. If bothHard is false, only the first client will have // hard NAT rules, and the second client will have easy NAT rules. // //nolint:revive func hardNAT(t *testing.T, stunCount int, bothHard bool) fakeInternet { internet := createFakeInternet(t) internet.Net.STUNs = make([]TestNetworkingSTUN, stunCount) for i := 0; i < stunCount; i++ { internet.Net.STUNs[i] = prepareSTUNServer(t, &internet, i) } _, err := commandInNetNS(internet.BridgeNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output() require.NoError(t, wrapExitErr(err), "enable IP forwarding in bridge NetNS") // Set up iptables masquerade rules to allow each router to NAT packets. leaves := []struct { fakeRouterLeaf peerIP string clientPort int natPortPeer int natStartPortSTUN int }{ { fakeRouterLeaf: internet.Client1, peerIP: internet.Client2.RouterIP, clientPort: client1Port, natPortPeer: client1RouterPort, natStartPortSTUN: client1RouterPortSTUN, }, { fakeRouterLeaf: internet.Client2, // If peerIP is empty, we do easy NAT (even for STUN) peerIP: func() string { if bothHard { return internet.Client1.RouterIP } return "" }(), clientPort: client2Port, natPortPeer: client2RouterPort, natStartPortSTUN: client2RouterPortSTUN, }, } for _, leaf := range leaves { _, err := commandInNetNS(leaf.RouterNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output() require.NoError(t, wrapExitErr(err), "enable IP forwarding in router NetNS") // All non-UDP traffic should use regular masquerade e.g. for HTTP. iptablesMasqueradeNonUDP(t, leaf.RouterNetNS) // NAT from this client to its peer. iptablesNAT(t, leaf.RouterNetNS, leaf.ClientIP, leaf.clientPort, leaf.RouterIP, leaf.natPortPeer, leaf.peerIP) // NAT from this client to each STUN server. Only do this if we're doing // hard NAT, as the rule above will also touch STUN traffic in easy NAT. if leaf.peerIP != "" { for i, stun := range internet.Net.STUNs { natPort := leaf.natStartPortSTUN + i iptablesNAT(t, leaf.RouterNetNS, leaf.ClientIP, leaf.clientPort, leaf.RouterIP, natPort, stun.IP) } } } return internet } func SetupNetworkingHardNATEasyNATDirect(t *testing.T, _ slog.Logger) TestNetworking { return hardNAT(t, 2, false).Net } type vethPair struct { Outer string Inner string } type fakeRouterLeaf struct { // RouterIP is the IP address of the router on the bridge. RouterIP string // ClientIP is the IP address of the client on the router. ClientIP string // RouterNetNS is the router for this specific leaf. RouterNetNS *os.File // ClientNetNS is where the "user" is. ClientNetNS *os.File // Veth pair between the router and the bridge. OuterVethPair vethPair // Veth pair between the user and the router. InnerVethPair vethPair } type fakeInternet struct { Net TestNetworking NamePrefix string BridgeNetNS *os.File BridgeName string ServerNetNS *os.File ServerVethPair vethPair // between bridge and server NS Client1 fakeRouterLeaf Client2 fakeRouterLeaf } // createFakeInternet creates multiple namespaces with veth pairs between them // with the following topology: // // . veth ┌────────┐ veth // . ┌─────────────────┤ Bridge ├───────────────────┐ // . │ └───┬────┘ │ // . │ │ │ // . │10.0.0.1 veth│10.0.0.2 │10.0.0.3 // . ┌───────┴───────┐ ┌───────┴─────────┐ ┌────────┴────────┐ // . │ Server │ │ Client 1 router │ │ Client 2 router │ // . └───────────────┘ └───────┬─────────┘ └────────┬────────┘ // . │10.0.2.1 │10.0.3.1 // . veth│ veth│ // . │10.0.2.2 │10.0.3.2 // . ┌───────┴─────────┐ ┌────────┴────────┐ // . │ Client 1 │ │ Client 2 │ // . └─────────────────┘ └─────────────────┘ // // No iptables rules are created, so packets will not be forwarded out of the // box. Default routes are created from the edge namespaces (client1, client2) // to their respective routers, but no NAT rules are created. func createFakeInternet(t *testing.T) fakeInternet { t.Helper() const ( bridgePrefix = "10.0.0." serverIP = bridgePrefix + "1" client1Prefix = "10.0.2." client2Prefix = "10.0.3." ) var ( namePrefix = uniqNetName(t) + "_" router = fakeInternet{ NamePrefix: namePrefix, BridgeName: namePrefix + "b", } ) // Create bridge namespace and bridge interface. router.BridgeNetNS = createNetNS(t, router.BridgeName) err := createBridge(router.BridgeNetNS, router.BridgeName) require.NoError(t, err, "create bridge in netns") // Create server namespace and veth pair between bridge and server. router.ServerNetNS = createNetNS(t, namePrefix+"s") router.ServerVethPair = vethPair{ Outer: namePrefix + "b-s", Inner: namePrefix + "s-b", } err = joinBridge(joinBridgeOpts{ bridgeNetNS: router.BridgeNetNS, netNS: router.ServerNetNS, bridgeName: router.BridgeName, vethPair: router.ServerVethPair, ip: serverIP, }) require.NoError(t, err, "join bridge with server") leaves := []struct { leaf *fakeRouterLeaf routerName string clientName string routerBridgeIP string routerClientIP string clientIP string }{ { leaf: &router.Client1, routerName: "c1r", clientName: "c1", routerBridgeIP: bridgePrefix + "2", routerClientIP: client1Prefix + "1", clientIP: client1Prefix + "2", }, { leaf: &router.Client2, routerName: "c2r", clientName: "c2", routerBridgeIP: bridgePrefix + "3", routerClientIP: client2Prefix + "1", clientIP: client2Prefix + "2", }, } for _, leaf := range leaves { leaf.leaf.RouterIP = leaf.routerBridgeIP leaf.leaf.ClientIP = leaf.clientIP // Create two network namespaces for each leaf: one for the router and // one for the "client". leaf.leaf.RouterNetNS = createNetNS(t, namePrefix+leaf.routerName) leaf.leaf.ClientNetNS = createNetNS(t, namePrefix+leaf.clientName) // Join the bridge. leaf.leaf.OuterVethPair = vethPair{ Outer: namePrefix + "b-" + leaf.routerName, Inner: namePrefix + leaf.routerName + "-b", } err = joinBridge(joinBridgeOpts{ bridgeNetNS: router.BridgeNetNS, netNS: leaf.leaf.RouterNetNS, bridgeName: router.BridgeName, vethPair: leaf.leaf.OuterVethPair, ip: leaf.routerBridgeIP, }) require.NoError(t, err, "join bridge with router") // Create inner veth pair between the router and the client. leaf.leaf.InnerVethPair = vethPair{ Outer: namePrefix + leaf.routerName + "-" + leaf.clientName, Inner: namePrefix + leaf.clientName + "-" + leaf.routerName, } err = createVethPair(leaf.leaf.InnerVethPair.Outer, leaf.leaf.InnerVethPair.Inner) require.NoErrorf(t, err, "create veth pair %q <-> %q", leaf.leaf.InnerVethPair.Outer, leaf.leaf.InnerVethPair.Inner) // Move the network interfaces to the respective network namespaces. err = setVethNetNS(leaf.leaf.InnerVethPair.Outer, int(leaf.leaf.RouterNetNS.Fd())) require.NoErrorf(t, err, "set veth %q to NetNS", leaf.leaf.InnerVethPair.Outer) err = setVethNetNS(leaf.leaf.InnerVethPair.Inner, int(leaf.leaf.ClientNetNS.Fd())) require.NoErrorf(t, err, "set veth %q to NetNS", leaf.leaf.InnerVethPair.Inner) // Set router's "local" IP on the veth. err = setInterfaceIP(leaf.leaf.RouterNetNS, leaf.leaf.InnerVethPair.Outer, leaf.routerClientIP) require.NoErrorf(t, err, "set IP %q on interface %q", leaf.routerClientIP, leaf.leaf.InnerVethPair.Outer) // Set client's IP on the veth. err = setInterfaceIP(leaf.leaf.ClientNetNS, leaf.leaf.InnerVethPair.Inner, leaf.clientIP) require.NoErrorf(t, err, "set IP %q on interface %q", leaf.clientIP, leaf.leaf.InnerVethPair.Inner) // Bring up the interfaces. err = setInterfaceUp(leaf.leaf.RouterNetNS, leaf.leaf.InnerVethPair.Outer) require.NoErrorf(t, err, "bring up interface %q", leaf.leaf.OuterVethPair.Outer) err = setInterfaceUp(leaf.leaf.ClientNetNS, leaf.leaf.InnerVethPair.Inner) require.NoErrorf(t, err, "bring up interface %q", leaf.leaf.InnerVethPair.Inner) // We don't need to add a route from parent to peer since the kernel // already adds a default route for the /24. We DO need to add a default // route from peer to parent, however. err = addRouteInNetNS(leaf.leaf.ClientNetNS, []string{"default", "via", leaf.routerClientIP, "dev", leaf.leaf.InnerVethPair.Inner}) require.NoErrorf(t, err, "add peer default route to %q", leaf.leaf.InnerVethPair.Inner) } router.Net = TestNetworking{ Server: TestNetworkingServer{ Process: TestNetworkingProcess{NetNS: router.ServerNetNS}, ListenAddr: serverIP + ":8080", }, Client1: TestNetworkingClient{ Process: TestNetworkingProcess{NetNS: router.Client1.ClientNetNS}, ServerAccessURL: "http://" + serverIP + ":8080", }, Client2: TestNetworkingClient{ Process: TestNetworkingProcess{NetNS: router.Client2.ClientNetNS}, ServerAccessURL: "http://" + serverIP + ":8080", }, } return router } func uniqNetName(t *testing.T) string { t.Helper() netNSName := "cdr_" randStr, err := cryptorand.String(3) require.NoError(t, err, "generate random string for netns name") netNSName += randStr return netNSName } type joinBridgeOpts struct { bridgeNetNS *os.File netNS *os.File bridgeName string // This vethPair will be created and should not already exist. vethPair vethPair ip string } // joinBridge joins the given network namespace to the bridge. It creates a veth // pair between the specified NetNS and the bridge NetNS, sets the IP address on // the "child" veth, and brings up the interfaces. func joinBridge(opts joinBridgeOpts) error { // Create outer veth pair between the router and the bridge. err := createVethPair(opts.vethPair.Outer, opts.vethPair.Inner) if err != nil { return xerrors.Errorf("create veth pair %q <-> %q: %w", opts.vethPair.Outer, opts.vethPair.Inner, err) } // Move the network interfaces to the respective network namespaces. err = setVethNetNS(opts.vethPair.Outer, int(opts.bridgeNetNS.Fd())) if err != nil { return xerrors.Errorf("set veth %q to NetNS: %w", opts.vethPair.Outer, err) } err = setVethNetNS(opts.vethPair.Inner, int(opts.netNS.Fd())) if err != nil { return xerrors.Errorf("set veth %q to NetNS: %w", opts.vethPair.Inner, err) } // Connect the outer veth to the bridge. err = setInterfaceBridge(opts.bridgeNetNS, opts.vethPair.Outer, opts.bridgeName) if err != nil { return xerrors.Errorf("set interface %q master to %q: %w", opts.vethPair.Outer, opts.bridgeName, err) } // Set the bridge IP on the inner veth. err = setInterfaceIP(opts.netNS, opts.vethPair.Inner, opts.ip) if err != nil { return xerrors.Errorf("set IP %q on interface %q: %w", opts.ip, opts.vethPair.Inner, err) } // Bring up the interfaces. err = setInterfaceUp(opts.bridgeNetNS, opts.vethPair.Outer) if err != nil { return xerrors.Errorf("bring up interface %q: %w", opts.vethPair.Outer, err) } err = setInterfaceUp(opts.netNS, opts.vethPair.Inner) if err != nil { return xerrors.Errorf("bring up interface %q: %w", opts.vethPair.Inner, err) } return nil } // createNetNS creates a new network namespace with the given name. The returned // file is a file descriptor to the network namespace. // Note: all cleanup is handled for you, you do not need to call Close on the // returned file. func createNetNS(t *testing.T, name string) *os.File { // We use ip-netns here because it handles the process of creating a // disowned netns for us. // The only way to create a network namespace is by calling unshare(2) or // clone(2) with the CLONE_NEWNET flag, and as soon as the last process in a // network namespace exits, the namespace is destroyed. // However, if you create a bind mount of /proc/$PID/ns/net to a file, it // will keep the namespace alive until the mount is removed. // ip-netns does this for us. Without it, we would have to fork anyways. // Later, we will use nsenter to enter this network namespace. _, err := exec.Command("ip", "netns", "add", name).Output() require.NoError(t, wrapExitErr(err), "create network namespace via ip-netns") t.Cleanup(func() { _, _ = exec.Command("ip", "netns", "delete", name).Output() }) // Open /run/netns/$name to get a file descriptor to the network namespace. path := fmt.Sprintf("/run/netns/%s", name) file, err := os.OpenFile(path, os.O_RDONLY, 0) require.NoError(t, err, "open network namespace file") t.Cleanup(func() { _ = file.Close() }) // Exec "ip link set lo up" in the namespace to bring up loopback // networking. //nolint:gosec _, err = exec.Command("ip", "-netns", name, "link", "set", "lo", "up").Output() require.NoError(t, wrapExitErr(err), "bring up loopback interface in network namespace") return file } // createBridge creates a bridge in the given network namespace. The bridge is // automatically brought up. func createBridge(netNS *os.File, name string) error { // While it might be possible to create a bridge directly in a NetNS or move // an existing bridge to a NetNS, I couldn't figure out a way to do it. // Creating it directly within the NetNS is the simplest way. _, err := commandInNetNS(netNS, "ip", []string{"link", "add", name, "type", "bridge"}).Output() if err != nil { return xerrors.Errorf("create bridge %q in netns: %w", name, wrapExitErr(err)) } _, err = commandInNetNS(netNS, "ip", []string{"link", "set", name, "up"}).Output() if err != nil { return xerrors.Errorf("set bridge %q up in netns: %w", name, wrapExitErr(err)) } return nil } // setInterfaceBridge sets the master of the given interface to the specified // bridge. func setInterfaceBridge(netNS *os.File, ifaceName, bridgeName string) error { _, err := commandInNetNS(netNS, "ip", []string{"link", "set", ifaceName, "master", bridgeName}).Output() if err != nil { return xerrors.Errorf("set interface %q master to %q in netns: %w", ifaceName, bridgeName, wrapExitErr(err)) } return nil } // createVethPair creates a veth pair with the given names. func createVethPair(parentVethName, peerVethName string) error { linkAttrs := netlink.NewLinkAttrs() linkAttrs.Name = parentVethName veth := &netlink.Veth{ LinkAttrs: linkAttrs, PeerName: peerVethName, } err := netlink.LinkAdd(veth) if err != nil { return xerrors.Errorf("LinkAdd(type: veth, name: %q, peerName: %q): %w", parentVethName, peerVethName, err) } return nil } // setVethNetNS moves the veth interface to the specified network namespace. func setVethNetNS(vethName string, netNSFd int) error { veth, err := netlink.LinkByName(vethName) if err != nil { return xerrors.Errorf("LinkByName(%q): %w", vethName, err) } err = netlink.LinkSetNsFd(veth, netNSFd) if err != nil { return xerrors.Errorf("LinkSetNsFd(%q, %v): %w", vethName, netNSFd, err) } return nil } // setInterfaceIP sets the IP address on the given interface. It automatically // adds a /24 subnet mask. func setInterfaceIP(netNS *os.File, ifaceName, ip string) error { _, err := commandInNetNS(netNS, "ip", []string{"addr", "add", ip + "/24", "dev", ifaceName}).Output() if err != nil { return xerrors.Errorf("set IP %q on interface %q in netns: %w", ip, ifaceName, wrapExitErr(err)) } return nil } // setInterfaceUp brings the given interface up. func setInterfaceUp(netNS *os.File, ifaceName string) error { _, err := commandInNetNS(netNS, "ip", []string{"link", "set", ifaceName, "up"}).Output() if err != nil { return xerrors.Errorf("bring up interface %q in netns: %w", ifaceName, wrapExitErr(err)) } return nil } // addRouteInNetNS adds a route to the given network namespace. func addRouteInNetNS(netNS *os.File, route []string) error { _, err := commandInNetNS(netNS, "ip", append([]string{"route", "add"}, route...)).Output() if err != nil { return xerrors.Errorf("add route %q in netns: %w", route, wrapExitErr(err)) } return nil } // prepareSTUNServer creates a STUN server networking spec in a network // namespace and joins it to the bridge. It also sets up the DERP map for the // clients to use the STUN. func prepareSTUNServer(t *testing.T, internet *fakeInternet, number int) TestNetworkingSTUN { name := fmt.Sprintf("stn%d", number) stunNetNS := createNetNS(t, internet.NamePrefix+name) stun := TestNetworkingSTUN{ Process: TestNetworkingProcess{ NetNS: stunNetNS, }, } stun.IP = "10.0.0." + fmt.Sprint(64+number) err := joinBridge(joinBridgeOpts{ bridgeNetNS: internet.BridgeNetNS, netNS: stunNetNS, bridgeName: internet.BridgeName, vethPair: vethPair{ Outer: internet.NamePrefix + "b-" + name, Inner: internet.NamePrefix + name + "-b", }, ip: stun.IP, }) require.NoError(t, err, "join bridge with STUN server") stun.ListenAddr = stun.IP + ":3478" // Define custom DERP map. stunRegion := &tailcfg.DERPRegion{ RegionID: 10000 + number, RegionCode: name, RegionName: name, Nodes: []*tailcfg.DERPNode{ { Name: name + "a", RegionID: 1, IPv4: stun.IP, IPv6: "none", STUNPort: 3478, STUNOnly: true, }, }, } client1DERP, err := internet.Net.Client1.ResolveDERPMap() require.NoError(t, err, "resolve DERP map for client 1") client1DERP.Regions[stunRegion.RegionID] = stunRegion internet.Net.Client1.DERPMap = client1DERP client2DERP, err := internet.Net.Client2.ResolveDERPMap() require.NoError(t, err, "resolve DERP map for client 2") client2DERP.Regions[stunRegion.RegionID] = stunRegion internet.Net.Client2.DERPMap = client2DERP return stun } func iptablesMasqueradeNonUDP(t *testing.T, netNS *os.File) { t.Helper() _, err := commandInNetNS(netNS, "iptables", []string{ "-t", "nat", "-A", "POSTROUTING", // Every interface except loopback. "!", "-o", "lo", // Every protocol except UDP. "!", "-p", "udp", "-j", "MASQUERADE", }).Output() require.NoError(t, wrapExitErr(err), "add iptables non-UDP masquerade rule") } // iptablesNAT sets up iptables rules for NAT forwarding. If destIP is // specified, the forwarding rule will only apply to traffic to/from that IP // (mapvarydest). func iptablesNAT(t *testing.T, netNS *os.File, clientIP string, clientPort int, routerIP string, routerPort int, destIP string) { t.Helper() snatArgs := []string{ "-t", "nat", "-A", "POSTROUTING", "-p", "udp", "--sport", fmt.Sprint(clientPort), "-j", "SNAT", "--to-source", fmt.Sprintf("%s:%d", routerIP, routerPort), } if destIP != "" { // Insert `-d $destIP` after the --sport flag+value. newSnatArgs := append([]string{}, snatArgs[:8]...) newSnatArgs = append(newSnatArgs, "-d", destIP) newSnatArgs = append(newSnatArgs, snatArgs[8:]...) snatArgs = newSnatArgs } _, err := commandInNetNS(netNS, "iptables", snatArgs).Output() require.NoError(t, wrapExitErr(err), "add iptables SNAT rule") // Incoming traffic should be forwarded to the client's IP. dnatArgs := []string{ "-t", "nat", "-A", "PREROUTING", "-p", "udp", "--dport", fmt.Sprint(routerPort), "-j", "DNAT", "--to-destination", fmt.Sprintf("%s:%d", clientIP, clientPort), } if destIP != "" { // Insert `-s $destIP` before the --dport flag+value. newDnatArgs := append([]string{}, dnatArgs[:6]...) newDnatArgs = append(newDnatArgs, "-s", destIP) newDnatArgs = append(newDnatArgs, dnatArgs[6:]...) dnatArgs = newDnatArgs } _, err = commandInNetNS(netNS, "iptables", dnatArgs).Output() require.NoError(t, wrapExitErr(err), "add iptables DNAT rule") } func commandInNetNS(netNS *os.File, bin string, args []string) *exec.Cmd { //nolint:gosec cmd := exec.Command("nsenter", append([]string{"--net=/proc/self/fd/3", bin}, args...)...) cmd.ExtraFiles = []*os.File{netNS} return cmd } func wrapExitErr(err error) error { if err == nil { return nil } var exitErr *exec.ExitError if xerrors.As(err, &exitErr) { return xerrors.Errorf("output: %s\n\n%w", bytes.TrimSpace(exitErr.Stderr), exitErr) } return err }