chore: add easy NAT integration tests part 2 (#13312)

This commit is contained in:
Dean Sheather
2024-05-23 23:32:30 -07:00
committed by GitHub
parent 1b4ca00428
commit e5bb0a7a00
6 changed files with 622 additions and 228 deletions

View File

@ -77,7 +77,7 @@ func TestOwnerExec(t *testing.T) {
})
}
// nolint:tparallel,paralleltest -- subtests share a map, just run sequentially.
// nolint:tparallel,paralleltest // subtests share a map, just run sequentially.
func TestRolePermissions(t *testing.T) {
t.Parallel()
@ -557,7 +557,7 @@ func TestRolePermissions(t *testing.T) {
// nolint:tparallel,paralleltest
for _, c := range testCases {
c := c
// nolint:tparallel,paralleltest -- These share the same remainingPermissions map
// nolint:tparallel,paralleltest // These share the same remainingPermissions map
t.Run(c.Name, func(t *testing.T) {
remainingSubjs := make(map[string]struct{})
for _, subj := range requiredSubjects {
@ -600,7 +600,7 @@ func TestRolePermissions(t *testing.T) {
// Only run these if the tests on top passed. Otherwise, the error output is too noisy.
if passed {
for rtype, v := range remainingPermissions {
// nolint:tparallel,paralleltest -- Making a subtest for easier diagnosing failures.
// nolint:tparallel,paralleltest // Making a subtest for easier diagnosing failures.
t.Run(fmt.Sprintf("%s-AllActions", rtype), func(t *testing.T) {
if len(v) > 0 {
assert.Equal(t, map[policy.Action]bool{}, v, "remaining permissions should be empty for type %q", rtype)

View File

@ -41,12 +41,34 @@ import (
"github.com/coder/coder/v2/testutil"
)
// IDs used in tests.
var (
Client1ID = uuid.MustParse("00000000-0000-0000-0000-000000000001")
Client2ID = uuid.MustParse("00000000-0000-0000-0000-000000000002")
type ClientNumber int
const (
ClientNumber1 ClientNumber = 1
ClientNumber2 ClientNumber = 2
)
type Client struct {
Number ClientNumber
ID uuid.UUID
ListenPort uint16
ShouldRunTests bool
}
var Client1 = Client{
Number: ClientNumber1,
ID: uuid.MustParse("00000000-0000-0000-0000-000000000001"),
ListenPort: client1Port,
ShouldRunTests: true,
}
var Client2 = Client{
Number: ClientNumber2,
ID: uuid.MustParse("00000000-0000-0000-0000-000000000002"),
ListenPort: client2Port,
ShouldRunTests: false,
}
type TestTopology struct {
Name string
// SetupNetworking creates interfaces and network namespaces for the test.
@ -59,12 +81,12 @@ type TestTopology struct {
Server ServerStarter
// StartClient gets called in each client subprocess. It's expected to
// create the tailnet.Conn and ensure connectivity to it's peer.
StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID) *tailnet.Conn
StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me Client, peer Client) *tailnet.Conn
// RunTests is the main test function. It's called in each of the client
// subprocesses. If tests can only run once, they should check the client ID
// and return early if it's not the expected one.
RunTests func(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID, conn *tailnet.Conn)
RunTests func(t *testing.T, logger slog.Logger, serverURL *url.URL, conn *tailnet.Conn, me Client, peer Client)
}
type ServerStarter interface {
@ -264,13 +286,14 @@ http {
// StartClientDERP creates a client connection to the server for coordination
// and creates a tailnet.Conn which will only use DERP to connect to the peer.
func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn {
return startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)},
DERPMap: basicDERPMap(t, serverURL),
func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn {
return startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(me.ID), 128)},
DERPMap: derpMap,
BlockEndpoints: true,
Logger: logger,
DERPForceWebSockets: false,
ListenPort: me.ListenPort,
// These tests don't have internet connection, so we need to force
// magicsock to do anything.
ForceNetworkUp: true,
@ -279,13 +302,14 @@ func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, myID,
// StartClientDERPWebSockets does the same thing as StartClientDERP but will
// only use DERP WebSocket fallback.
func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn {
return startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)},
DERPMap: basicDERPMap(t, serverURL),
func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn {
return startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(me.ID), 128)},
DERPMap: derpMap,
BlockEndpoints: true,
Logger: logger,
DERPForceWebSockets: true,
ListenPort: me.ListenPort,
// These tests don't have internet connection, so we need to force
// magicsock to do anything.
ForceNetworkUp: true,
@ -295,20 +319,21 @@ func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.
// StartClientDirect does the same thing as StartClientDERP but disables
// BlockEndpoints (which enables Direct connections), and waits for a direct
// connection to be established between the two peers.
func StartClientDirect(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn {
conn := startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)},
DERPMap: basicDERPMap(t, serverURL),
func StartClientDirect(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn {
conn := startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(me.ID), 128)},
DERPMap: derpMap,
BlockEndpoints: false,
Logger: logger,
DERPForceWebSockets: true,
ListenPort: me.ListenPort,
// These tests don't have internet connection, so we need to force
// magicsock to do anything.
ForceNetworkUp: true,
})
// Wait for direct connection to be established.
peerIP := tailnet.IPFromUUID(peerID)
peerIP := tailnet.IPFromUUID(peer.ID)
require.Eventually(t, func() bool {
t.Log("attempting ping to peer to judge direct connection")
ctx := testutil.Context(t, testutil.WaitShort)
@ -332,8 +357,8 @@ type ClientStarter struct {
Options *tailnet.Options
}
func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID, options *tailnet.Options) *tailnet.Conn {
u, err := serverURL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/coordinate", myID.String()))
func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, me, peer Client, options *tailnet.Options) *tailnet.Conn {
u, err := serverURL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/coordinate", me.ID.String()))
require.NoError(t, err)
//nolint:bodyclose
ws, _, err := websocket.Dial(context.Background(), u.String(), nil)
@ -357,7 +382,7 @@ func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, my
_ = conn.Close()
})
coordination := tailnet.NewRemoteCoordination(logger, coord, conn, peerID)
coordination := tailnet.NewRemoteCoordination(logger, coord, conn, peer.ID)
t.Cleanup(func() {
_ = coordination.Close()
})
@ -365,10 +390,17 @@ func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, my
return conn
}
func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap {
func basicDERPMap(serverURLStr string) (*tailcfg.DERPMap, error) {
serverURL, err := url.Parse(serverURLStr)
if err != nil {
return nil, xerrors.Errorf("parse server URL %q: %w", serverURLStr, err)
}
portStr := serverURL.Port()
port, err := strconv.Atoi(portStr)
require.NoError(t, err, "parse server port")
if err != nil {
return nil, xerrors.Errorf("parse port %q: %w", portStr, err)
}
hostname := serverURL.Hostname()
ipv4 := ""
@ -399,7 +431,7 @@ func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap {
},
},
},
}
}, nil
}
// ExecBackground starts a subprocess with the given flags and returns a

View File

@ -4,19 +4,27 @@
package integration_test
import (
"context"
"encoding/json"
"flag"
"fmt"
"net"
"net/http"
"net/url"
"os"
"os/signal"
"path/filepath"
"runtime"
"strconv"
"syscall"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"tailscale.com/net/stun/stuntest"
"tailscale.com/tailcfg"
"tailscale.com/types/nettype"
"cdr.dev/slog"
"cdr.dev/slog/sloggers/slogtest"
@ -30,17 +38,19 @@ const runTestEnv = "CODER_TAILNET_TESTS"
var (
isSubprocess = flag.Bool("subprocess", false, "Signifies that this is a test subprocess")
testID = flag.String("test-name", "", "Which test is being run")
role = flag.String("role", "", "The role of the test subprocess: server, client")
role = flag.String("role", "", "The role of the test subprocess: server, stun, client")
// Role: server
serverListenAddr = flag.String("server-listen-addr", "", "The address to listen on for the server")
// Role: stun
stunListenAddr = flag.String("stun-listen-addr", "", "The address to listen on for the STUN server")
// Role: client
clientName = flag.String("client-name", "", "The name of the client for logs")
clientServerURL = flag.String("client-server-url", "", "The url to connect to the server")
clientMyID = flag.String("client-id", "", "The id of the client")
clientPeerID = flag.String("client-peer-id", "", "The id of the other client")
clientRunTests = flag.Bool("client-run-tests", false, "Run the tests in the client subprocess")
clientName = flag.String("client-name", "", "The name of the client for logs")
clientNumber = flag.Int("client-number", 0, "The number of the client")
clientServerURL = flag.String("client-server-url", "", "The url to connect to the server")
clientDERPMapPath = flag.String("client-derp-map-path", "", "The path to the DERP map file to use on this client")
)
func TestMain(m *testing.M) {
@ -87,7 +97,7 @@ var topologies = []integration.TestTopology{
// endpoints to connect as routing is enabled between client 1 and
// client 2.
Name: "EasyNATDirect",
SetupNetworking: integration.SetupNetworkingEasyNAT,
SetupNetworking: integration.SetupNetworkingEasyNATWithSTUN,
Server: integration.SimpleServerOptions{},
StartClient: integration.StartClientDirect,
RunTests: integration.TestSuite,
@ -143,17 +153,41 @@ func TestIntegration(t *testing.T) {
log := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
networking := topo.SetupNetworking(t, log)
// Fork the three child processes.
// Useful for debugging network namespaces by avoiding cleanup.
// t.Cleanup(func() {
// time.Sleep(time.Minute * 15)
// })
closeServer := startServerSubprocess(t, topo.Name, networking)
closeSTUN := func() error { return nil }
if networking.STUN.ListenAddr != "" {
closeSTUN = startSTUNSubprocess(t, topo.Name, networking)
}
// Write the DERP maps to a file.
tempDir := t.TempDir()
client1DERPMapPath := filepath.Join(tempDir, "client1-derp-map.json")
client1DERPMap, err := networking.Client1.ResolveDERPMap()
require.NoError(t, err, "resolve client 1 DERP map")
err = writeDERPMapToFile(client1DERPMapPath, client1DERPMap)
require.NoError(t, err, "write client 1 DERP map")
client2DERPMapPath := filepath.Join(tempDir, "client2-derp-map.json")
client2DERPMap, err := networking.Client2.ResolveDERPMap()
require.NoError(t, err, "resolve client 2 DERP map")
err = writeDERPMapToFile(client2DERPMapPath, client2DERPMap)
require.NoError(t, err, "write client 2 DERP map")
// client1 runs the tests.
client1ErrCh, _ := startClientSubprocess(t, topo.Name, networking, 1)
_, closeClient2 := startClientSubprocess(t, topo.Name, networking, 2)
client1ErrCh, _ := startClientSubprocess(t, topo.Name, networking, integration.Client1, client1DERPMapPath)
_, closeClient2 := startClientSubprocess(t, topo.Name, networking, integration.Client2, client2DERPMapPath)
// Wait for client1 to exit.
require.NoError(t, <-client1ErrCh, "client 1 exited")
// Close client2 and the server.
require.NoError(t, closeClient2(), "client 2 exited")
require.NoError(t, closeSTUN(), "stun exited")
require.NoError(t, closeServer(), "server exited")
})
}
@ -169,10 +203,11 @@ func handleTestSubprocess(t *testing.T) {
}
}
require.NotEmptyf(t, topo.Name, "unknown test topology %q", *testID)
require.Contains(t, []string{"server", "stun", "client"}, *role, "unknown role %q", *role)
testName := topo.Name + "/"
if *role == "server" {
testName += "server"
if *role == "server" || *role == "stun" {
testName += *role
} else {
testName += *clientName
}
@ -185,27 +220,44 @@ func handleTestSubprocess(t *testing.T) {
topo.Server.StartServer(t, logger, *serverListenAddr)
// no exit
case "stun":
launchSTUNServer(t, *stunListenAddr)
// no exit
case "client":
logger = logger.Named(*clientName)
if *clientNumber != int(integration.ClientNumber1) && *clientNumber != int(integration.ClientNumber2) {
t.Fatalf("invalid client number %d", clientNumber)
}
me, peer := integration.Client1, integration.Client2
if *clientNumber == int(integration.ClientNumber2) {
me, peer = peer, me
}
serverURL, err := url.Parse(*clientServerURL)
require.NoErrorf(t, err, "parse server url %q", *clientServerURL)
myID, err := uuid.Parse(*clientMyID)
require.NoErrorf(t, err, "parse client id %q", *clientMyID)
peerID, err := uuid.Parse(*clientPeerID)
require.NoErrorf(t, err, "parse peer id %q", *clientPeerID)
// Load the DERP map.
var derpMap tailcfg.DERPMap
derpMapPath := *clientDERPMapPath
f, err := os.Open(derpMapPath)
require.NoErrorf(t, err, "open DERP map %q", derpMapPath)
err = json.NewDecoder(f).Decode(&derpMap)
_ = f.Close()
require.NoErrorf(t, err, "decode DERP map %q", derpMapPath)
waitForServerAvailable(t, serverURL)
conn := topo.StartClient(t, logger, serverURL, myID, peerID)
conn := topo.StartClient(t, logger, serverURL, &derpMap, me, peer)
if *clientRunTests {
if me.ShouldRunTests {
// Wait for connectivity.
peerIP := tailnet.IPFromUUID(peerID)
peerIP := tailnet.IPFromUUID(peer.ID)
if !conn.AwaitReachable(testutil.Context(t, testutil.WaitLong), peerIP) {
t.Fatalf("peer %v did not become reachable", peerIP)
}
topo.RunTests(t, logger, serverURL, myID, peerID, conn)
topo.RunTests(t, logger, serverURL, conn, me, peer)
// then exit
return
}
@ -218,6 +270,23 @@ func handleTestSubprocess(t *testing.T) {
})
}
type forcedAddrPacketListener struct {
addr string
}
var _ nettype.PacketListener = forcedAddrPacketListener{}
func (ln forcedAddrPacketListener) ListenPacket(ctx context.Context, network, _ string) (net.PacketConn, error) {
return nettype.Std{}.ListenPacket(ctx, network, ln.addr)
}
func launchSTUNServer(t *testing.T, listenAddr string) {
ln := forcedAddrPacketListener{addr: listenAddr}
addr, cleanup := stuntest.ServeWithPacketListener(t, ln)
t.Cleanup(cleanup)
assert.Equal(t, listenAddr, addr.String(), "listen address should match forced addr")
}
func waitForServerAvailable(t *testing.T, serverURL *url.URL) {
const delay = 100 * time.Millisecond
const reqTimeout = 2 * time.Second
@ -247,29 +316,32 @@ func waitForServerAvailable(t *testing.T, serverURL *url.URL) {
}
func startServerSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) func() error {
_, closeFn := startSubprocess(t, "server", networking.ProcessServer.NetNS, []string{
_, closeFn := startSubprocess(t, "server", networking.Server.Process.NetNS, []string{
"--subprocess",
"--test-name=" + topologyName,
"--role=server",
"--server-listen-addr=" + networking.ServerListenAddr,
"--server-listen-addr=" + networking.Server.ListenAddr,
})
return closeFn
}
func startClientSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking, clientNumber int) (<-chan error, func() error) {
require.True(t, clientNumber == 1 || clientNumber == 2)
func startSTUNSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) func() error {
_, closeFn := startSubprocess(t, "stun", networking.STUN.Process.NetNS, []string{
"--subprocess",
"--test-name=" + topologyName,
"--role=stun",
"--stun-listen-addr=" + networking.STUN.ListenAddr,
})
return closeFn
}
func startClientSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking, me integration.Client, derpMapPath string) (<-chan error, func() error) {
var (
clientName = fmt.Sprintf("client%d", clientNumber)
myID = integration.Client1ID
peerID = integration.Client2ID
accessURL = networking.ServerAccessURLClient1
netNS = networking.ProcessClient1.NetNS
clientName = fmt.Sprintf("client%d", me.Number)
clientProcessConfig = networking.Client1
)
if clientNumber == 2 {
myID, peerID = peerID, myID
accessURL = networking.ServerAccessURLClient2
netNS = networking.ProcessClient2.NetNS
if me.Number == integration.ClientNumber2 {
clientProcessConfig = networking.Client2
}
flags := []string{
@ -277,15 +349,12 @@ func startClientSubprocess(t *testing.T, topologyName string, networking integra
"--test-name=" + topologyName,
"--role=client",
"--client-name=" + clientName,
"--client-server-url=" + accessURL,
"--client-id=" + myID.String(),
"--client-peer-id=" + peerID.String(),
}
if clientNumber == 1 {
flags = append(flags, "--client-run-tests")
"--client-number=" + strconv.Itoa(int(me.Number)),
"--client-server-url=" + clientProcessConfig.ServerAccessURL,
"--client-derp-map-path=" + derpMapPath,
}
return startSubprocess(t, clientName, netNS, flags)
return startSubprocess(t, clientName, clientProcessConfig.Process.NetNS, flags)
}
// startSubprocess launches the test binary with the same flags as the test, but
@ -295,6 +364,22 @@ func startClientSubprocess(t *testing.T, topologyName string, networking integra
func startSubprocess(t *testing.T, processName string, netNS *os.File, flags []string) (<-chan error, func() error) {
name := os.Args[0]
// Always use verbose mode since it gets piped to the parent test anyways.
args := append(os.Args[1:], append([]string{"-test.v=true"}, flags...)...)
args := append(os.Args[1:], append([]string{"-test.v=true"}, flags...)...) //nolint:gocritic
return integration.ExecBackground(t, processName, netNS, name, args)
}
func writeDERPMapToFile(path string, derpMap *tailcfg.DERPMap) error {
f, err := os.Create(path)
if err != nil {
return err
}
defer f.Close()
enc := json.NewEncoder(f)
enc.SetIndent("", " ")
err = enc.Encode(derpMap)
if err != nil {
return err
}
return nil
}

View File

@ -13,27 +13,55 @@ import (
"github.com/stretchr/testify/require"
"github.com/tailscale/netlink"
"golang.org/x/xerrors"
"tailscale.com/tailcfg"
"cdr.dev/slog"
"github.com/coder/coder/v2/cryptorand"
)
type TestNetworking struct {
// ServerListenAddr is the IP address and port that the server listens on,
// passed to StartServer.
ServerListenAddr string
// ServerAccessURLClient1 is the hostname and port that the first client
// uses to access the server.
ServerAccessURLClient1 string
// ServerAccessURLClient2 is the hostname and port that the second client
// uses to access the server.
ServerAccessURLClient2 string
const (
client1Port = 48001
client1RouterPort = 48011
client2Port = 48002
client2RouterPort = 48012
)
// Networking settings for each subprocess.
ProcessServer TestNetworkingProcess
ProcessClient1 TestNetworkingProcess
ProcessClient2 TestNetworkingProcess
type TestNetworking struct {
Server TestNetworkingServer
STUN TestNetworkingSTUN
Client1 TestNetworkingClient
Client2 TestNetworkingClient
}
type TestNetworkingServer struct {
Process TestNetworkingProcess
ListenAddr string
}
type TestNetworkingSTUN struct {
Process TestNetworkingProcess
// If empty, no STUN subprocess is launched.
ListenAddr string
}
type TestNetworkingClient struct {
Process TestNetworkingProcess
// ServerAccessURL is the hostname and port that the client uses to access
// the server over HTTP for coordination.
ServerAccessURL string
// DERPMap is the DERP map that the client uses. If nil, a basic DERP map
// containing only a single DERP with `ServerAccessURL` is used with no
// STUN servers.
DERPMap *tailcfg.DERPMap
}
func (c TestNetworkingClient) ResolveDERPMap() (*tailcfg.DERPMap, error) {
if c.DERPMap != nil {
return c.DERPMap, nil
}
return basicDERPMap(c.ServerAccessURL)
}
type TestNetworkingProcess struct {
@ -46,14 +74,9 @@ type TestNetworkingProcess struct {
// namespace only exists for isolation on the host and doesn't serve any routing
// purpose.
func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking {
netNSName := "codertest_netns_"
randStr, err := cryptorand.String(4)
require.NoError(t, err, "generate random string for netns name")
netNSName += randStr
// Create a single network namespace for all tests so we can have an
// isolated loopback interface.
netNSFile := createNetNS(t, netNSName)
netNSFile := createNetNS(t, uniqNetName(t))
var (
listenAddr = "127.0.0.1:8080"
@ -62,176 +85,323 @@ func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking {
}
)
return TestNetworking{
ServerListenAddr: listenAddr,
ServerAccessURLClient1: "http://" + listenAddr,
ServerAccessURLClient2: "http://" + listenAddr,
ProcessServer: process,
ProcessClient1: process,
ProcessClient2: process,
Server: TestNetworkingServer{
Process: process,
ListenAddr: listenAddr,
},
Client1: TestNetworkingClient{
Process: process,
ServerAccessURL: "http://" + listenAddr,
},
Client2: TestNetworkingClient{
Process: process,
ServerAccessURL: "http://" + listenAddr,
},
}
}
// SetupNetworkingEasyNAT creates a network namespace with a router that NATs
// packets between two clients and a server.
// See createFakeRouter for the full topology.
func easyNAT(t *testing.T) fakeInternet {
internet := createFakeInternet(t)
_, err := commandInNetNS(internet.BridgeNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output()
require.NoError(t, wrapExitErr(err), "enable IP forwarding in bridge NetNS")
// Set up iptables masquerade rules to allow each router to NAT packets.
leaves := []struct {
fakeRouterLeaf
clientPort int
natPort int
}{
{internet.Client1, client1Port, client1RouterPort},
{internet.Client2, client2Port, client2RouterPort},
}
for _, leaf := range leaves {
_, err := commandInNetNS(leaf.RouterNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output()
require.NoError(t, wrapExitErr(err), "enable IP forwarding in router NetNS")
// All non-UDP traffic should use regular masquerade e.g. for HTTP.
_, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{
"-t", "nat",
"-A", "POSTROUTING",
// Every interface except loopback.
"!", "-o", "lo",
// Every protocol except UDP.
"!", "-p", "udp",
"-j", "MASQUERADE",
}).Output()
require.NoError(t, wrapExitErr(err), "add iptables non-UDP masquerade rule")
// Outgoing traffic should get NATed to the router's IP.
_, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{
"-t", "nat",
"-A", "POSTROUTING",
"-p", "udp",
"--sport", fmt.Sprint(leaf.clientPort),
"-j", "SNAT",
"--to-source", fmt.Sprintf("%s:%d", leaf.RouterIP, leaf.natPort),
}).Output()
require.NoError(t, wrapExitErr(err), "add iptables SNAT rule")
// Incoming traffic should be forwarded to the client's IP.
_, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{
"-t", "nat",
"-A", "PREROUTING",
"-p", "udp",
"--dport", fmt.Sprint(leaf.natPort),
"-j", "DNAT",
"--to-destination", fmt.Sprintf("%s:%d", leaf.ClientIP, leaf.clientPort),
}).Output()
require.NoError(t, wrapExitErr(err), "add iptables DNAT rule")
}
return internet
}
// SetupNetworkingEasyNAT creates a fake internet and sets up "easy NAT"
// forwarding rules.
// See createFakeInternet.
// NAT is achieved through a single iptables masquerade rule.
func SetupNetworkingEasyNAT(t *testing.T, _ slog.Logger) TestNetworking {
router := createFakeRouter(t)
// Set up iptables masquerade rules to allow the router to NAT packets
// between the Three Kingdoms.
_, err := commandInNetNS(router.RouterNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output()
require.NoError(t, wrapExitErr(err), "enable IP forwarding in router NetNS")
_, err = commandInNetNS(router.RouterNetNS, "iptables", []string{
"-t", "nat",
"-A", "POSTROUTING",
// Every interface except loopback.
"!", "-o", "lo",
"-j", "MASQUERADE",
}).Output()
require.NoError(t, wrapExitErr(err), "add iptables masquerade rule")
return router.Net
return easyNAT(t).Net
}
type fakeRouter struct {
// SetupNetworkingEasyNATWithSTUN does the same as SetupNetworkingEasyNAT, but
// also creates a namespace and bridge address for a STUN server.
func SetupNetworkingEasyNATWithSTUN(t *testing.T, _ slog.Logger) TestNetworking {
internet := easyNAT(t)
// Create another network namespace for the STUN server.
stunNetNS := createNetNS(t, internet.NamePrefix+"stun")
internet.Net.STUN.Process = TestNetworkingProcess{
NetNS: stunNetNS,
}
const ip = "10.0.0.64"
err := joinBridge(joinBridgeOpts{
bridgeNetNS: internet.BridgeNetNS,
netNS: stunNetNS,
bridgeName: internet.BridgeName,
vethPair: vethPair{
Outer: internet.NamePrefix + "b-stun",
Inner: internet.NamePrefix + "stun-b",
},
ip: ip,
})
require.NoError(t, err, "join bridge with STUN server")
internet.Net.STUN.ListenAddr = ip + ":3478"
// Define custom DERP map.
stunRegion := &tailcfg.DERPRegion{
RegionID: 10000,
RegionCode: "stun0",
RegionName: "STUN0",
Nodes: []*tailcfg.DERPNode{
{
Name: "stun0a",
RegionID: 1,
IPv4: ip,
IPv6: "none",
STUNPort: 3478,
STUNOnly: true,
},
},
}
client1DERP, err := internet.Net.Client1.ResolveDERPMap()
require.NoError(t, err, "resolve DERP map for client 1")
client1DERP.Regions[stunRegion.RegionID] = stunRegion
internet.Net.Client1.DERPMap = client1DERP
client2DERP, err := internet.Net.Client2.ResolveDERPMap()
require.NoError(t, err, "resolve DERP map for client 2")
client2DERP.Regions[stunRegion.RegionID] = stunRegion
internet.Net.Client2.DERPMap = client2DERP
return internet.Net
}
type vethPair struct {
Outer string
Inner string
}
type fakeRouterLeaf struct {
// RouterIP is the IP address of the router on the bridge.
RouterIP string
// ClientIP is the IP address of the client on the router.
ClientIP string
// RouterNetNS is the router for this specific leaf.
RouterNetNS *os.File
// ClientNetNS is where the "user" is.
ClientNetNS *os.File
// Veth pair between the router and the bridge.
OuterVethPair vethPair
// Veth pair between the user and the router.
InnerVethPair vethPair
}
type fakeInternet struct {
Net TestNetworking
RouterNetNS *os.File
RouterVeths struct {
Server string
Client1 string
Client2 string
}
ServerNetNS *os.File
ServerVeth string
Client1NetNS *os.File
Client1Veth string
Client2NetNS *os.File
Client2Veth string
NamePrefix string
BridgeNetNS *os.File
BridgeName string
ServerNetNS *os.File
ServerVethPair vethPair // between bridge and server NS
Client1 fakeRouterLeaf
Client2 fakeRouterLeaf
}
// fakeRouter creates multiple namespaces with veth pairs between them with
// the following topology:
// createFakeInternet creates multiple namespaces with veth pairs between them
// with the following topology:
//
// namespaces:
// - router
// - server
// - client1
// - client2
//
// veth pairs:
// - router-server (10.0.1.1) <-> server-router (10.0.1.2)
// - router-client1 (10.0.2.1) <-> client1-router (10.0.2.2)
// - router-client2 (10.0.3.1) <-> client2-router (10.0.3.2)
// . veth ┌────────┐ veth
// . ┌─────────────────┤ Bridge ├───────────────────┐
// . │ └───┬────┘ │
// . │ │ │
// . │10.0.0.1 veth│10.0.0.2 │10.0.0.3
// . ┌───────┴───────┐ ┌───────┴─────────┐ ┌────────┴────────┐
// . │ Server │ │ Client 1 router │ │ Client 2 router │
// . └───────────────┘ └───────┬─────────┘ └────────┬────────┘
// . │10.0.2.1 │10.0.3.1
// . veth│ veth│
// . │10.0.2.2 │10.0.3.2
// . ┌───────┴─────────┐ ┌────────┴────────┐
// . │ Client 1 │ │ Client 2 │
// . └─────────────────┘ └─────────────────┘
//
// No iptables rules are created, so packets will not be forwarded out of the
// box. Routes are created between all namespaces based on the veth pairs,
// however.
func createFakeRouter(t *testing.T) fakeRouter {
// box. Default routes are created from the edge namespaces (client1, client2)
// to their respective routers, but no NAT rules are created.
func createFakeInternet(t *testing.T) fakeInternet {
t.Helper()
const (
routerServerPrefix = "10.0.1."
routerServerIP = routerServerPrefix + "1"
serverIP = routerServerPrefix + "2"
routerClient1Prefix = "10.0.2."
routerClient1IP = routerClient1Prefix + "1"
client1IP = routerClient1Prefix + "2"
routerClient2Prefix = "10.0.3."
routerClient2IP = routerClient2Prefix + "1"
client2IP = routerClient2Prefix + "2"
bridgePrefix = "10.0.0."
serverIP = bridgePrefix + "1"
client1Prefix = "10.0.2."
client2Prefix = "10.0.3."
)
var (
namePrefix = uniqNetName(t) + "_"
router = fakeInternet{
NamePrefix: namePrefix,
BridgeName: namePrefix + "b",
}
)
prefix := uniqNetName(t) + "_"
router := fakeRouter{}
router.RouterVeths.Server = prefix + "r-s"
router.RouterVeths.Client1 = prefix + "r-c1"
router.RouterVeths.Client2 = prefix + "r-c2"
router.ServerVeth = prefix + "s-r"
router.Client1Veth = prefix + "c1-r"
router.Client2Veth = prefix + "c2-r"
// Create bridge namespace and bridge interface.
router.BridgeNetNS = createNetNS(t, router.BridgeName)
err := createBridge(router.BridgeNetNS, router.BridgeName)
require.NoError(t, err, "create bridge in netns")
// Create namespaces.
router.RouterNetNS = createNetNS(t, prefix+"r")
serverNS := createNetNS(t, prefix+"s")
client1NS := createNetNS(t, prefix+"c1")
client2NS := createNetNS(t, prefix+"c2")
// Create server namespace and veth pair between bridge and server.
router.ServerNetNS = createNetNS(t, namePrefix+"s")
router.ServerVethPair = vethPair{
Outer: namePrefix + "b-s",
Inner: namePrefix + "s-b",
}
err = joinBridge(joinBridgeOpts{
bridgeNetNS: router.BridgeNetNS,
netNS: router.ServerNetNS,
bridgeName: router.BridgeName,
vethPair: router.ServerVethPair,
ip: serverIP,
})
require.NoError(t, err, "join bridge with server")
vethPairs := []struct {
parentName string
peerName string
parentNS *os.File
peerNS *os.File
parentIP string
peerIP string
leaves := []struct {
leaf *fakeRouterLeaf
routerName string
clientName string
routerBridgeIP string
routerClientIP string
clientIP string
}{
{
parentName: router.RouterVeths.Server,
peerName: router.ServerVeth,
parentNS: router.RouterNetNS,
peerNS: serverNS,
parentIP: routerServerIP,
peerIP: serverIP,
leaf: &router.Client1,
routerName: "c1r",
clientName: "c1",
routerBridgeIP: bridgePrefix + "2",
routerClientIP: client1Prefix + "1",
clientIP: client1Prefix + "2",
},
{
parentName: router.RouterVeths.Client1,
peerName: router.Client1Veth,
parentNS: router.RouterNetNS,
peerNS: client1NS,
parentIP: routerClient1IP,
peerIP: client1IP,
},
{
parentName: router.RouterVeths.Client2,
peerName: router.Client2Veth,
parentNS: router.RouterNetNS,
peerNS: client2NS,
parentIP: routerClient2IP,
peerIP: client2IP,
leaf: &router.Client2,
routerName: "c2r",
clientName: "c2",
routerBridgeIP: bridgePrefix + "3",
routerClientIP: client2Prefix + "1",
clientIP: client2Prefix + "2",
},
}
for _, vethPair := range vethPairs {
err := createVethPair(vethPair.parentName, vethPair.peerName)
require.NoErrorf(t, err, "create veth pair %q <-> %q", vethPair.parentName, vethPair.peerName)
for _, leaf := range leaves {
leaf.leaf.RouterIP = leaf.routerBridgeIP
leaf.leaf.ClientIP = leaf.clientIP
// Move the veth interfaces to the respective network namespaces.
err = setVethNetNS(vethPair.parentName, int(vethPair.parentNS.Fd()))
require.NoErrorf(t, err, "set veth %q to NetNS", vethPair.parentName)
err = setVethNetNS(vethPair.peerName, int(vethPair.peerNS.Fd()))
require.NoErrorf(t, err, "set veth %q to NetNS", vethPair.peerName)
// Create two network namespaces for each leaf: one for the router and
// one for the "client".
leaf.leaf.RouterNetNS = createNetNS(t, namePrefix+leaf.routerName)
leaf.leaf.ClientNetNS = createNetNS(t, namePrefix+leaf.clientName)
// Set IP addresses on the interfaces.
err = setInterfaceIP(vethPair.parentNS, vethPair.parentName, vethPair.parentIP)
require.NoErrorf(t, err, "set IP %q on interface %q", vethPair.parentIP, vethPair.parentName)
err = setInterfaceIP(vethPair.peerNS, vethPair.peerName, vethPair.peerIP)
require.NoErrorf(t, err, "set IP %q on interface %q", vethPair.peerIP, vethPair.peerName)
// Join the bridge.
leaf.leaf.OuterVethPair = vethPair{
Outer: namePrefix + "b-" + leaf.routerName,
Inner: namePrefix + leaf.routerName + "-b",
}
err = joinBridge(joinBridgeOpts{
bridgeNetNS: router.BridgeNetNS,
netNS: leaf.leaf.RouterNetNS,
bridgeName: router.BridgeName,
vethPair: leaf.leaf.OuterVethPair,
ip: leaf.routerBridgeIP,
})
require.NoError(t, err, "join bridge with router")
// Bring up both interfaces.
err = setInterfaceUp(vethPair.parentNS, vethPair.parentName)
require.NoErrorf(t, err, "bring up interface %q", vethPair.parentName)
err = setInterfaceUp(vethPair.peerNS, vethPair.peerName)
require.NoErrorf(t, err, "bring up interface %q", vethPair.parentName)
// Create inner veth pair between the router and the client.
leaf.leaf.InnerVethPair = vethPair{
Outer: namePrefix + leaf.routerName + "-" + leaf.clientName,
Inner: namePrefix + leaf.clientName + "-" + leaf.routerName,
}
err = createVethPair(leaf.leaf.InnerVethPair.Outer, leaf.leaf.InnerVethPair.Inner)
require.NoErrorf(t, err, "create veth pair %q <-> %q", leaf.leaf.InnerVethPair.Outer, leaf.leaf.InnerVethPair.Inner)
// Move the network interfaces to the respective network namespaces.
err = setVethNetNS(leaf.leaf.InnerVethPair.Outer, int(leaf.leaf.RouterNetNS.Fd()))
require.NoErrorf(t, err, "set veth %q to NetNS", leaf.leaf.InnerVethPair.Outer)
err = setVethNetNS(leaf.leaf.InnerVethPair.Inner, int(leaf.leaf.ClientNetNS.Fd()))
require.NoErrorf(t, err, "set veth %q to NetNS", leaf.leaf.InnerVethPair.Inner)
// Set router's "local" IP on the veth.
err = setInterfaceIP(leaf.leaf.RouterNetNS, leaf.leaf.InnerVethPair.Outer, leaf.routerClientIP)
require.NoErrorf(t, err, "set IP %q on interface %q", leaf.routerClientIP, leaf.leaf.InnerVethPair.Outer)
// Set client's IP on the veth.
err = setInterfaceIP(leaf.leaf.ClientNetNS, leaf.leaf.InnerVethPair.Inner, leaf.clientIP)
require.NoErrorf(t, err, "set IP %q on interface %q", leaf.clientIP, leaf.leaf.InnerVethPair.Inner)
// Bring up the interfaces.
err = setInterfaceUp(leaf.leaf.RouterNetNS, leaf.leaf.InnerVethPair.Outer)
require.NoErrorf(t, err, "bring up interface %q", leaf.leaf.OuterVethPair.Outer)
err = setInterfaceUp(leaf.leaf.ClientNetNS, leaf.leaf.InnerVethPair.Inner)
require.NoErrorf(t, err, "bring up interface %q", leaf.leaf.InnerVethPair.Inner)
// We don't need to add a route from parent to peer since the kernel
// already adds a default route for the /24. We DO need to add a default
// route from peer to parent, however.
err = addRouteInNetNS(vethPair.peerNS, []string{"default", "via", vethPair.parentIP, "dev", vethPair.peerName})
require.NoErrorf(t, err, "add peer default route to %q", vethPair.peerName)
err = addRouteInNetNS(leaf.leaf.ClientNetNS, []string{"default", "via", leaf.routerClientIP, "dev", leaf.leaf.InnerVethPair.Inner})
require.NoErrorf(t, err, "add peer default route to %q", leaf.leaf.InnerVethPair.Inner)
}
router.Net = TestNetworking{
ServerListenAddr: serverIP + ":8080",
ServerAccessURLClient1: "http://" + serverIP + ":8080",
ServerAccessURLClient2: "http://" + serverIP + ":8080",
ProcessServer: TestNetworkingProcess{
NetNS: serverNS,
Server: TestNetworkingServer{
Process: TestNetworkingProcess{NetNS: router.ServerNetNS},
ListenAddr: serverIP + ":8080",
},
ProcessClient1: TestNetworkingProcess{
NetNS: client1NS,
Client1: TestNetworkingClient{
Process: TestNetworkingProcess{NetNS: router.Client1.ClientNetNS},
ServerAccessURL: "http://" + serverIP + ":8080",
},
ProcessClient2: TestNetworkingProcess{
NetNS: client2NS,
Client2: TestNetworkingClient{
Process: TestNetworkingProcess{NetNS: router.Client2.ClientNetNS},
ServerAccessURL: "http://" + serverIP + ":8080",
},
}
return router
@ -246,6 +416,60 @@ func uniqNetName(t *testing.T) string {
return netNSName
}
type joinBridgeOpts struct {
bridgeNetNS *os.File
netNS *os.File
bridgeName string
// This vethPair will be created and should not already exist.
vethPair vethPair
ip string
}
// joinBridge joins the given network namespace to the bridge. It creates a veth
// pair between the specified NetNS and the bridge NetNS, sets the IP address on
// the "child" veth, and brings up the interfaces.
func joinBridge(opts joinBridgeOpts) error {
// Create outer veth pair between the router and the bridge.
err := createVethPair(opts.vethPair.Outer, opts.vethPair.Inner)
if err != nil {
return xerrors.Errorf("create veth pair %q <-> %q: %w", opts.vethPair.Outer, opts.vethPair.Inner, err)
}
// Move the network interfaces to the respective network namespaces.
err = setVethNetNS(opts.vethPair.Outer, int(opts.bridgeNetNS.Fd()))
if err != nil {
return xerrors.Errorf("set veth %q to NetNS: %w", opts.vethPair.Outer, err)
}
err = setVethNetNS(opts.vethPair.Inner, int(opts.netNS.Fd()))
if err != nil {
return xerrors.Errorf("set veth %q to NetNS: %w", opts.vethPair.Inner, err)
}
// Connect the outer veth to the bridge.
err = setInterfaceBridge(opts.bridgeNetNS, opts.vethPair.Outer, opts.bridgeName)
if err != nil {
return xerrors.Errorf("set interface %q master to %q: %w", opts.vethPair.Outer, opts.bridgeName, err)
}
// Set the bridge IP on the inner veth.
err = setInterfaceIP(opts.netNS, opts.vethPair.Inner, opts.ip)
if err != nil {
return xerrors.Errorf("set IP %q on interface %q: %w", opts.ip, opts.vethPair.Inner, err)
}
// Bring up the interfaces.
err = setInterfaceUp(opts.bridgeNetNS, opts.vethPair.Outer)
if err != nil {
return xerrors.Errorf("bring up interface %q: %w", opts.vethPair.Outer, err)
}
err = setInterfaceUp(opts.netNS, opts.vethPair.Inner)
if err != nil {
return xerrors.Errorf("bring up interface %q: %w", opts.vethPair.Inner, err)
}
return nil
}
// createNetNS creates a new network namespace with the given name. The returned
// file is a file descriptor to the network namespace.
// Note: all cleanup is handled for you, you do not need to call Close on the
@ -283,18 +507,48 @@ func createNetNS(t *testing.T, name string) *os.File {
return file
}
// createBridge creates a bridge in the given network namespace. The bridge is
// automatically brought up.
func createBridge(netNS *os.File, name string) error {
// While it might be possible to create a bridge directly in a NetNS or move
// an existing bridge to a NetNS, I couldn't figure out a way to do it.
// Creating it directly within the NetNS is the simplest way.
_, err := commandInNetNS(netNS, "ip", []string{"link", "add", name, "type", "bridge"}).Output()
if err != nil {
return xerrors.Errorf("create bridge %q in netns: %w", name, wrapExitErr(err))
}
_, err = commandInNetNS(netNS, "ip", []string{"link", "set", name, "up"}).Output()
if err != nil {
return xerrors.Errorf("set bridge %q up in netns: %w", name, wrapExitErr(err))
}
return nil
}
// setInterfaceBridge sets the master of the given interface to the specified
// bridge.
func setInterfaceBridge(netNS *os.File, ifaceName, bridgeName string) error {
_, err := commandInNetNS(netNS, "ip", []string{"link", "set", ifaceName, "master", bridgeName}).Output()
if err != nil {
return xerrors.Errorf("set interface %q master to %q in netns: %w", ifaceName, bridgeName, wrapExitErr(err))
}
return nil
}
// createVethPair creates a veth pair with the given names.
func createVethPair(parentVethName, peerVethName string) error {
vethLinkAttrs := netlink.NewLinkAttrs()
vethLinkAttrs.Name = parentVethName
linkAttrs := netlink.NewLinkAttrs()
linkAttrs.Name = parentVethName
veth := &netlink.Veth{
LinkAttrs: vethLinkAttrs,
LinkAttrs: linkAttrs,
PeerName: peerVethName,
}
err := netlink.LinkAdd(veth)
if err != nil {
return xerrors.Errorf("LinkAdd(name: %q, peerName: %q): %w", parentVethName, peerVethName, err)
return xerrors.Errorf("LinkAdd(type: veth, name: %q, peerName: %q): %w", parentVethName, peerVethName, err)
}
return nil

View File

@ -0,0 +1,24 @@
#!/bin/bash
set -euo pipefail
if [[ $(id -u) -ne 0 ]]; then
echo "Please run with sudo"
exit 1
fi
to_delete=$(ip netns list | grep -o 'cdr_.*_.*' | cut -d' ' -f1)
echo "Will delete:"
for ns in $to_delete; do
echo "- $ns"
done
read -p "Continue? [y/N] " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
exit 1
fi
for ns in $to_delete; do
ip netns delete "$ns"
done

View File

@ -7,7 +7,6 @@ import (
"net/url"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"cdr.dev/slog"
@ -17,12 +16,12 @@ import (
// TODO: instead of reusing one conn for each suite, maybe we should make a new
// one for each subtest?
func TestSuite(t *testing.T, _ slog.Logger, _ *url.URL, _, peerID uuid.UUID, conn *tailnet.Conn) {
func TestSuite(t *testing.T, _ slog.Logger, _ *url.URL, conn *tailnet.Conn, _, peer Client) {
t.Parallel()
t.Run("Connectivity", func(t *testing.T) {
t.Parallel()
peerIP := tailnet.IPFromUUID(peerID)
peerIP := tailnet.IPFromUUID(peer.ID)
_, _, _, err := conn.Ping(testutil.Context(t, testutil.WaitLong), peerIP)
require.NoError(t, err, "ping peer")
})