mirror of
https://github.com/coder/coder.git
synced 2025-07-13 21:36:50 +00:00
chore: add easy NAT integration tests part 2 (#13312)
This commit is contained in:
@ -77,7 +77,7 @@ func TestOwnerExec(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// nolint:tparallel,paralleltest -- subtests share a map, just run sequentially.
|
||||
// nolint:tparallel,paralleltest // subtests share a map, just run sequentially.
|
||||
func TestRolePermissions(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@ -557,7 +557,7 @@ func TestRolePermissions(t *testing.T) {
|
||||
// nolint:tparallel,paralleltest
|
||||
for _, c := range testCases {
|
||||
c := c
|
||||
// nolint:tparallel,paralleltest -- These share the same remainingPermissions map
|
||||
// nolint:tparallel,paralleltest // These share the same remainingPermissions map
|
||||
t.Run(c.Name, func(t *testing.T) {
|
||||
remainingSubjs := make(map[string]struct{})
|
||||
for _, subj := range requiredSubjects {
|
||||
@ -600,7 +600,7 @@ func TestRolePermissions(t *testing.T) {
|
||||
// Only run these if the tests on top passed. Otherwise, the error output is too noisy.
|
||||
if passed {
|
||||
for rtype, v := range remainingPermissions {
|
||||
// nolint:tparallel,paralleltest -- Making a subtest for easier diagnosing failures.
|
||||
// nolint:tparallel,paralleltest // Making a subtest for easier diagnosing failures.
|
||||
t.Run(fmt.Sprintf("%s-AllActions", rtype), func(t *testing.T) {
|
||||
if len(v) > 0 {
|
||||
assert.Equal(t, map[policy.Action]bool{}, v, "remaining permissions should be empty for type %q", rtype)
|
||||
|
@ -41,12 +41,34 @@ import (
|
||||
"github.com/coder/coder/v2/testutil"
|
||||
)
|
||||
|
||||
// IDs used in tests.
|
||||
var (
|
||||
Client1ID = uuid.MustParse("00000000-0000-0000-0000-000000000001")
|
||||
Client2ID = uuid.MustParse("00000000-0000-0000-0000-000000000002")
|
||||
type ClientNumber int
|
||||
|
||||
const (
|
||||
ClientNumber1 ClientNumber = 1
|
||||
ClientNumber2 ClientNumber = 2
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
Number ClientNumber
|
||||
ID uuid.UUID
|
||||
ListenPort uint16
|
||||
ShouldRunTests bool
|
||||
}
|
||||
|
||||
var Client1 = Client{
|
||||
Number: ClientNumber1,
|
||||
ID: uuid.MustParse("00000000-0000-0000-0000-000000000001"),
|
||||
ListenPort: client1Port,
|
||||
ShouldRunTests: true,
|
||||
}
|
||||
|
||||
var Client2 = Client{
|
||||
Number: ClientNumber2,
|
||||
ID: uuid.MustParse("00000000-0000-0000-0000-000000000002"),
|
||||
ListenPort: client2Port,
|
||||
ShouldRunTests: false,
|
||||
}
|
||||
|
||||
type TestTopology struct {
|
||||
Name string
|
||||
// SetupNetworking creates interfaces and network namespaces for the test.
|
||||
@ -59,12 +81,12 @@ type TestTopology struct {
|
||||
Server ServerStarter
|
||||
// StartClient gets called in each client subprocess. It's expected to
|
||||
// create the tailnet.Conn and ensure connectivity to it's peer.
|
||||
StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID) *tailnet.Conn
|
||||
StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me Client, peer Client) *tailnet.Conn
|
||||
|
||||
// RunTests is the main test function. It's called in each of the client
|
||||
// subprocesses. If tests can only run once, they should check the client ID
|
||||
// and return early if it's not the expected one.
|
||||
RunTests func(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID, conn *tailnet.Conn)
|
||||
RunTests func(t *testing.T, logger slog.Logger, serverURL *url.URL, conn *tailnet.Conn, me Client, peer Client)
|
||||
}
|
||||
|
||||
type ServerStarter interface {
|
||||
@ -264,13 +286,14 @@ http {
|
||||
|
||||
// StartClientDERP creates a client connection to the server for coordination
|
||||
// and creates a tailnet.Conn which will only use DERP to connect to the peer.
|
||||
func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn {
|
||||
return startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{
|
||||
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)},
|
||||
DERPMap: basicDERPMap(t, serverURL),
|
||||
func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn {
|
||||
return startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{
|
||||
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(me.ID), 128)},
|
||||
DERPMap: derpMap,
|
||||
BlockEndpoints: true,
|
||||
Logger: logger,
|
||||
DERPForceWebSockets: false,
|
||||
ListenPort: me.ListenPort,
|
||||
// These tests don't have internet connection, so we need to force
|
||||
// magicsock to do anything.
|
||||
ForceNetworkUp: true,
|
||||
@ -279,13 +302,14 @@ func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, myID,
|
||||
|
||||
// StartClientDERPWebSockets does the same thing as StartClientDERP but will
|
||||
// only use DERP WebSocket fallback.
|
||||
func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn {
|
||||
return startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{
|
||||
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)},
|
||||
DERPMap: basicDERPMap(t, serverURL),
|
||||
func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn {
|
||||
return startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{
|
||||
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(me.ID), 128)},
|
||||
DERPMap: derpMap,
|
||||
BlockEndpoints: true,
|
||||
Logger: logger,
|
||||
DERPForceWebSockets: true,
|
||||
ListenPort: me.ListenPort,
|
||||
// These tests don't have internet connection, so we need to force
|
||||
// magicsock to do anything.
|
||||
ForceNetworkUp: true,
|
||||
@ -295,20 +319,21 @@ func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.
|
||||
// StartClientDirect does the same thing as StartClientDERP but disables
|
||||
// BlockEndpoints (which enables Direct connections), and waits for a direct
|
||||
// connection to be established between the two peers.
|
||||
func StartClientDirect(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn {
|
||||
conn := startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{
|
||||
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)},
|
||||
DERPMap: basicDERPMap(t, serverURL),
|
||||
func StartClientDirect(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn {
|
||||
conn := startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{
|
||||
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(me.ID), 128)},
|
||||
DERPMap: derpMap,
|
||||
BlockEndpoints: false,
|
||||
Logger: logger,
|
||||
DERPForceWebSockets: true,
|
||||
ListenPort: me.ListenPort,
|
||||
// These tests don't have internet connection, so we need to force
|
||||
// magicsock to do anything.
|
||||
ForceNetworkUp: true,
|
||||
})
|
||||
|
||||
// Wait for direct connection to be established.
|
||||
peerIP := tailnet.IPFromUUID(peerID)
|
||||
peerIP := tailnet.IPFromUUID(peer.ID)
|
||||
require.Eventually(t, func() bool {
|
||||
t.Log("attempting ping to peer to judge direct connection")
|
||||
ctx := testutil.Context(t, testutil.WaitShort)
|
||||
@ -332,8 +357,8 @@ type ClientStarter struct {
|
||||
Options *tailnet.Options
|
||||
}
|
||||
|
||||
func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID, options *tailnet.Options) *tailnet.Conn {
|
||||
u, err := serverURL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/coordinate", myID.String()))
|
||||
func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, me, peer Client, options *tailnet.Options) *tailnet.Conn {
|
||||
u, err := serverURL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/coordinate", me.ID.String()))
|
||||
require.NoError(t, err)
|
||||
//nolint:bodyclose
|
||||
ws, _, err := websocket.Dial(context.Background(), u.String(), nil)
|
||||
@ -357,7 +382,7 @@ func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, my
|
||||
_ = conn.Close()
|
||||
})
|
||||
|
||||
coordination := tailnet.NewRemoteCoordination(logger, coord, conn, peerID)
|
||||
coordination := tailnet.NewRemoteCoordination(logger, coord, conn, peer.ID)
|
||||
t.Cleanup(func() {
|
||||
_ = coordination.Close()
|
||||
})
|
||||
@ -365,10 +390,17 @@ func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, my
|
||||
return conn
|
||||
}
|
||||
|
||||
func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap {
|
||||
func basicDERPMap(serverURLStr string) (*tailcfg.DERPMap, error) {
|
||||
serverURL, err := url.Parse(serverURLStr)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("parse server URL %q: %w", serverURLStr, err)
|
||||
}
|
||||
|
||||
portStr := serverURL.Port()
|
||||
port, err := strconv.Atoi(portStr)
|
||||
require.NoError(t, err, "parse server port")
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("parse port %q: %w", portStr, err)
|
||||
}
|
||||
|
||||
hostname := serverURL.Hostname()
|
||||
ipv4 := ""
|
||||
@ -399,7 +431,7 @@ func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap {
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ExecBackground starts a subprocess with the given flags and returns a
|
||||
|
@ -4,19 +4,27 @@
|
||||
package integration_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"tailscale.com/net/stun/stuntest"
|
||||
"tailscale.com/tailcfg"
|
||||
"tailscale.com/types/nettype"
|
||||
|
||||
"cdr.dev/slog"
|
||||
"cdr.dev/slog/sloggers/slogtest"
|
||||
@ -30,17 +38,19 @@ const runTestEnv = "CODER_TAILNET_TESTS"
|
||||
var (
|
||||
isSubprocess = flag.Bool("subprocess", false, "Signifies that this is a test subprocess")
|
||||
testID = flag.String("test-name", "", "Which test is being run")
|
||||
role = flag.String("role", "", "The role of the test subprocess: server, client")
|
||||
role = flag.String("role", "", "The role of the test subprocess: server, stun, client")
|
||||
|
||||
// Role: server
|
||||
serverListenAddr = flag.String("server-listen-addr", "", "The address to listen on for the server")
|
||||
|
||||
// Role: stun
|
||||
stunListenAddr = flag.String("stun-listen-addr", "", "The address to listen on for the STUN server")
|
||||
|
||||
// Role: client
|
||||
clientName = flag.String("client-name", "", "The name of the client for logs")
|
||||
clientServerURL = flag.String("client-server-url", "", "The url to connect to the server")
|
||||
clientMyID = flag.String("client-id", "", "The id of the client")
|
||||
clientPeerID = flag.String("client-peer-id", "", "The id of the other client")
|
||||
clientRunTests = flag.Bool("client-run-tests", false, "Run the tests in the client subprocess")
|
||||
clientName = flag.String("client-name", "", "The name of the client for logs")
|
||||
clientNumber = flag.Int("client-number", 0, "The number of the client")
|
||||
clientServerURL = flag.String("client-server-url", "", "The url to connect to the server")
|
||||
clientDERPMapPath = flag.String("client-derp-map-path", "", "The path to the DERP map file to use on this client")
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
@ -87,7 +97,7 @@ var topologies = []integration.TestTopology{
|
||||
// endpoints to connect as routing is enabled between client 1 and
|
||||
// client 2.
|
||||
Name: "EasyNATDirect",
|
||||
SetupNetworking: integration.SetupNetworkingEasyNAT,
|
||||
SetupNetworking: integration.SetupNetworkingEasyNATWithSTUN,
|
||||
Server: integration.SimpleServerOptions{},
|
||||
StartClient: integration.StartClientDirect,
|
||||
RunTests: integration.TestSuite,
|
||||
@ -143,17 +153,41 @@ func TestIntegration(t *testing.T) {
|
||||
log := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
|
||||
networking := topo.SetupNetworking(t, log)
|
||||
|
||||
// Fork the three child processes.
|
||||
// Useful for debugging network namespaces by avoiding cleanup.
|
||||
// t.Cleanup(func() {
|
||||
// time.Sleep(time.Minute * 15)
|
||||
// })
|
||||
|
||||
closeServer := startServerSubprocess(t, topo.Name, networking)
|
||||
|
||||
closeSTUN := func() error { return nil }
|
||||
if networking.STUN.ListenAddr != "" {
|
||||
closeSTUN = startSTUNSubprocess(t, topo.Name, networking)
|
||||
}
|
||||
|
||||
// Write the DERP maps to a file.
|
||||
tempDir := t.TempDir()
|
||||
client1DERPMapPath := filepath.Join(tempDir, "client1-derp-map.json")
|
||||
client1DERPMap, err := networking.Client1.ResolveDERPMap()
|
||||
require.NoError(t, err, "resolve client 1 DERP map")
|
||||
err = writeDERPMapToFile(client1DERPMapPath, client1DERPMap)
|
||||
require.NoError(t, err, "write client 1 DERP map")
|
||||
client2DERPMapPath := filepath.Join(tempDir, "client2-derp-map.json")
|
||||
client2DERPMap, err := networking.Client2.ResolveDERPMap()
|
||||
require.NoError(t, err, "resolve client 2 DERP map")
|
||||
err = writeDERPMapToFile(client2DERPMapPath, client2DERPMap)
|
||||
require.NoError(t, err, "write client 2 DERP map")
|
||||
|
||||
// client1 runs the tests.
|
||||
client1ErrCh, _ := startClientSubprocess(t, topo.Name, networking, 1)
|
||||
_, closeClient2 := startClientSubprocess(t, topo.Name, networking, 2)
|
||||
client1ErrCh, _ := startClientSubprocess(t, topo.Name, networking, integration.Client1, client1DERPMapPath)
|
||||
_, closeClient2 := startClientSubprocess(t, topo.Name, networking, integration.Client2, client2DERPMapPath)
|
||||
|
||||
// Wait for client1 to exit.
|
||||
require.NoError(t, <-client1ErrCh, "client 1 exited")
|
||||
|
||||
// Close client2 and the server.
|
||||
require.NoError(t, closeClient2(), "client 2 exited")
|
||||
require.NoError(t, closeSTUN(), "stun exited")
|
||||
require.NoError(t, closeServer(), "server exited")
|
||||
})
|
||||
}
|
||||
@ -169,10 +203,11 @@ func handleTestSubprocess(t *testing.T) {
|
||||
}
|
||||
}
|
||||
require.NotEmptyf(t, topo.Name, "unknown test topology %q", *testID)
|
||||
require.Contains(t, []string{"server", "stun", "client"}, *role, "unknown role %q", *role)
|
||||
|
||||
testName := topo.Name + "/"
|
||||
if *role == "server" {
|
||||
testName += "server"
|
||||
if *role == "server" || *role == "stun" {
|
||||
testName += *role
|
||||
} else {
|
||||
testName += *clientName
|
||||
}
|
||||
@ -185,27 +220,44 @@ func handleTestSubprocess(t *testing.T) {
|
||||
topo.Server.StartServer(t, logger, *serverListenAddr)
|
||||
// no exit
|
||||
|
||||
case "stun":
|
||||
launchSTUNServer(t, *stunListenAddr)
|
||||
// no exit
|
||||
|
||||
case "client":
|
||||
logger = logger.Named(*clientName)
|
||||
if *clientNumber != int(integration.ClientNumber1) && *clientNumber != int(integration.ClientNumber2) {
|
||||
t.Fatalf("invalid client number %d", clientNumber)
|
||||
}
|
||||
me, peer := integration.Client1, integration.Client2
|
||||
if *clientNumber == int(integration.ClientNumber2) {
|
||||
me, peer = peer, me
|
||||
}
|
||||
|
||||
serverURL, err := url.Parse(*clientServerURL)
|
||||
require.NoErrorf(t, err, "parse server url %q", *clientServerURL)
|
||||
myID, err := uuid.Parse(*clientMyID)
|
||||
require.NoErrorf(t, err, "parse client id %q", *clientMyID)
|
||||
peerID, err := uuid.Parse(*clientPeerID)
|
||||
require.NoErrorf(t, err, "parse peer id %q", *clientPeerID)
|
||||
|
||||
// Load the DERP map.
|
||||
var derpMap tailcfg.DERPMap
|
||||
derpMapPath := *clientDERPMapPath
|
||||
f, err := os.Open(derpMapPath)
|
||||
require.NoErrorf(t, err, "open DERP map %q", derpMapPath)
|
||||
err = json.NewDecoder(f).Decode(&derpMap)
|
||||
_ = f.Close()
|
||||
require.NoErrorf(t, err, "decode DERP map %q", derpMapPath)
|
||||
|
||||
waitForServerAvailable(t, serverURL)
|
||||
|
||||
conn := topo.StartClient(t, logger, serverURL, myID, peerID)
|
||||
conn := topo.StartClient(t, logger, serverURL, &derpMap, me, peer)
|
||||
|
||||
if *clientRunTests {
|
||||
if me.ShouldRunTests {
|
||||
// Wait for connectivity.
|
||||
peerIP := tailnet.IPFromUUID(peerID)
|
||||
peerIP := tailnet.IPFromUUID(peer.ID)
|
||||
if !conn.AwaitReachable(testutil.Context(t, testutil.WaitLong), peerIP) {
|
||||
t.Fatalf("peer %v did not become reachable", peerIP)
|
||||
}
|
||||
|
||||
topo.RunTests(t, logger, serverURL, myID, peerID, conn)
|
||||
topo.RunTests(t, logger, serverURL, conn, me, peer)
|
||||
// then exit
|
||||
return
|
||||
}
|
||||
@ -218,6 +270,23 @@ func handleTestSubprocess(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
type forcedAddrPacketListener struct {
|
||||
addr string
|
||||
}
|
||||
|
||||
var _ nettype.PacketListener = forcedAddrPacketListener{}
|
||||
|
||||
func (ln forcedAddrPacketListener) ListenPacket(ctx context.Context, network, _ string) (net.PacketConn, error) {
|
||||
return nettype.Std{}.ListenPacket(ctx, network, ln.addr)
|
||||
}
|
||||
|
||||
func launchSTUNServer(t *testing.T, listenAddr string) {
|
||||
ln := forcedAddrPacketListener{addr: listenAddr}
|
||||
addr, cleanup := stuntest.ServeWithPacketListener(t, ln)
|
||||
t.Cleanup(cleanup)
|
||||
assert.Equal(t, listenAddr, addr.String(), "listen address should match forced addr")
|
||||
}
|
||||
|
||||
func waitForServerAvailable(t *testing.T, serverURL *url.URL) {
|
||||
const delay = 100 * time.Millisecond
|
||||
const reqTimeout = 2 * time.Second
|
||||
@ -247,29 +316,32 @@ func waitForServerAvailable(t *testing.T, serverURL *url.URL) {
|
||||
}
|
||||
|
||||
func startServerSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) func() error {
|
||||
_, closeFn := startSubprocess(t, "server", networking.ProcessServer.NetNS, []string{
|
||||
_, closeFn := startSubprocess(t, "server", networking.Server.Process.NetNS, []string{
|
||||
"--subprocess",
|
||||
"--test-name=" + topologyName,
|
||||
"--role=server",
|
||||
"--server-listen-addr=" + networking.ServerListenAddr,
|
||||
"--server-listen-addr=" + networking.Server.ListenAddr,
|
||||
})
|
||||
return closeFn
|
||||
}
|
||||
|
||||
func startClientSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking, clientNumber int) (<-chan error, func() error) {
|
||||
require.True(t, clientNumber == 1 || clientNumber == 2)
|
||||
func startSTUNSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) func() error {
|
||||
_, closeFn := startSubprocess(t, "stun", networking.STUN.Process.NetNS, []string{
|
||||
"--subprocess",
|
||||
"--test-name=" + topologyName,
|
||||
"--role=stun",
|
||||
"--stun-listen-addr=" + networking.STUN.ListenAddr,
|
||||
})
|
||||
return closeFn
|
||||
}
|
||||
|
||||
func startClientSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking, me integration.Client, derpMapPath string) (<-chan error, func() error) {
|
||||
var (
|
||||
clientName = fmt.Sprintf("client%d", clientNumber)
|
||||
myID = integration.Client1ID
|
||||
peerID = integration.Client2ID
|
||||
accessURL = networking.ServerAccessURLClient1
|
||||
netNS = networking.ProcessClient1.NetNS
|
||||
clientName = fmt.Sprintf("client%d", me.Number)
|
||||
clientProcessConfig = networking.Client1
|
||||
)
|
||||
if clientNumber == 2 {
|
||||
myID, peerID = peerID, myID
|
||||
accessURL = networking.ServerAccessURLClient2
|
||||
netNS = networking.ProcessClient2.NetNS
|
||||
if me.Number == integration.ClientNumber2 {
|
||||
clientProcessConfig = networking.Client2
|
||||
}
|
||||
|
||||
flags := []string{
|
||||
@ -277,15 +349,12 @@ func startClientSubprocess(t *testing.T, topologyName string, networking integra
|
||||
"--test-name=" + topologyName,
|
||||
"--role=client",
|
||||
"--client-name=" + clientName,
|
||||
"--client-server-url=" + accessURL,
|
||||
"--client-id=" + myID.String(),
|
||||
"--client-peer-id=" + peerID.String(),
|
||||
}
|
||||
if clientNumber == 1 {
|
||||
flags = append(flags, "--client-run-tests")
|
||||
"--client-number=" + strconv.Itoa(int(me.Number)),
|
||||
"--client-server-url=" + clientProcessConfig.ServerAccessURL,
|
||||
"--client-derp-map-path=" + derpMapPath,
|
||||
}
|
||||
|
||||
return startSubprocess(t, clientName, netNS, flags)
|
||||
return startSubprocess(t, clientName, clientProcessConfig.Process.NetNS, flags)
|
||||
}
|
||||
|
||||
// startSubprocess launches the test binary with the same flags as the test, but
|
||||
@ -295,6 +364,22 @@ func startClientSubprocess(t *testing.T, topologyName string, networking integra
|
||||
func startSubprocess(t *testing.T, processName string, netNS *os.File, flags []string) (<-chan error, func() error) {
|
||||
name := os.Args[0]
|
||||
// Always use verbose mode since it gets piped to the parent test anyways.
|
||||
args := append(os.Args[1:], append([]string{"-test.v=true"}, flags...)...)
|
||||
args := append(os.Args[1:], append([]string{"-test.v=true"}, flags...)...) //nolint:gocritic
|
||||
return integration.ExecBackground(t, processName, netNS, name, args)
|
||||
}
|
||||
|
||||
func writeDERPMapToFile(path string, derpMap *tailcfg.DERPMap) error {
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
enc := json.NewEncoder(f)
|
||||
enc.SetIndent("", " ")
|
||||
err = enc.Encode(derpMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
@ -13,27 +13,55 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/tailscale/netlink"
|
||||
"golang.org/x/xerrors"
|
||||
"tailscale.com/tailcfg"
|
||||
|
||||
"cdr.dev/slog"
|
||||
|
||||
"github.com/coder/coder/v2/cryptorand"
|
||||
)
|
||||
|
||||
type TestNetworking struct {
|
||||
// ServerListenAddr is the IP address and port that the server listens on,
|
||||
// passed to StartServer.
|
||||
ServerListenAddr string
|
||||
// ServerAccessURLClient1 is the hostname and port that the first client
|
||||
// uses to access the server.
|
||||
ServerAccessURLClient1 string
|
||||
// ServerAccessURLClient2 is the hostname and port that the second client
|
||||
// uses to access the server.
|
||||
ServerAccessURLClient2 string
|
||||
const (
|
||||
client1Port = 48001
|
||||
client1RouterPort = 48011
|
||||
client2Port = 48002
|
||||
client2RouterPort = 48012
|
||||
)
|
||||
|
||||
// Networking settings for each subprocess.
|
||||
ProcessServer TestNetworkingProcess
|
||||
ProcessClient1 TestNetworkingProcess
|
||||
ProcessClient2 TestNetworkingProcess
|
||||
type TestNetworking struct {
|
||||
Server TestNetworkingServer
|
||||
STUN TestNetworkingSTUN
|
||||
Client1 TestNetworkingClient
|
||||
Client2 TestNetworkingClient
|
||||
}
|
||||
|
||||
type TestNetworkingServer struct {
|
||||
Process TestNetworkingProcess
|
||||
ListenAddr string
|
||||
}
|
||||
|
||||
type TestNetworkingSTUN struct {
|
||||
Process TestNetworkingProcess
|
||||
// If empty, no STUN subprocess is launched.
|
||||
ListenAddr string
|
||||
}
|
||||
|
||||
type TestNetworkingClient struct {
|
||||
Process TestNetworkingProcess
|
||||
// ServerAccessURL is the hostname and port that the client uses to access
|
||||
// the server over HTTP for coordination.
|
||||
ServerAccessURL string
|
||||
// DERPMap is the DERP map that the client uses. If nil, a basic DERP map
|
||||
// containing only a single DERP with `ServerAccessURL` is used with no
|
||||
// STUN servers.
|
||||
DERPMap *tailcfg.DERPMap
|
||||
}
|
||||
|
||||
func (c TestNetworkingClient) ResolveDERPMap() (*tailcfg.DERPMap, error) {
|
||||
if c.DERPMap != nil {
|
||||
return c.DERPMap, nil
|
||||
}
|
||||
|
||||
return basicDERPMap(c.ServerAccessURL)
|
||||
}
|
||||
|
||||
type TestNetworkingProcess struct {
|
||||
@ -46,14 +74,9 @@ type TestNetworkingProcess struct {
|
||||
// namespace only exists for isolation on the host and doesn't serve any routing
|
||||
// purpose.
|
||||
func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking {
|
||||
netNSName := "codertest_netns_"
|
||||
randStr, err := cryptorand.String(4)
|
||||
require.NoError(t, err, "generate random string for netns name")
|
||||
netNSName += randStr
|
||||
|
||||
// Create a single network namespace for all tests so we can have an
|
||||
// isolated loopback interface.
|
||||
netNSFile := createNetNS(t, netNSName)
|
||||
netNSFile := createNetNS(t, uniqNetName(t))
|
||||
|
||||
var (
|
||||
listenAddr = "127.0.0.1:8080"
|
||||
@ -62,176 +85,323 @@ func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking {
|
||||
}
|
||||
)
|
||||
return TestNetworking{
|
||||
ServerListenAddr: listenAddr,
|
||||
ServerAccessURLClient1: "http://" + listenAddr,
|
||||
ServerAccessURLClient2: "http://" + listenAddr,
|
||||
ProcessServer: process,
|
||||
ProcessClient1: process,
|
||||
ProcessClient2: process,
|
||||
Server: TestNetworkingServer{
|
||||
Process: process,
|
||||
ListenAddr: listenAddr,
|
||||
},
|
||||
Client1: TestNetworkingClient{
|
||||
Process: process,
|
||||
ServerAccessURL: "http://" + listenAddr,
|
||||
},
|
||||
Client2: TestNetworkingClient{
|
||||
Process: process,
|
||||
ServerAccessURL: "http://" + listenAddr,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// SetupNetworkingEasyNAT creates a network namespace with a router that NATs
|
||||
// packets between two clients and a server.
|
||||
// See createFakeRouter for the full topology.
|
||||
func easyNAT(t *testing.T) fakeInternet {
|
||||
internet := createFakeInternet(t)
|
||||
|
||||
_, err := commandInNetNS(internet.BridgeNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output()
|
||||
require.NoError(t, wrapExitErr(err), "enable IP forwarding in bridge NetNS")
|
||||
|
||||
// Set up iptables masquerade rules to allow each router to NAT packets.
|
||||
leaves := []struct {
|
||||
fakeRouterLeaf
|
||||
clientPort int
|
||||
natPort int
|
||||
}{
|
||||
{internet.Client1, client1Port, client1RouterPort},
|
||||
{internet.Client2, client2Port, client2RouterPort},
|
||||
}
|
||||
for _, leaf := range leaves {
|
||||
_, err := commandInNetNS(leaf.RouterNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output()
|
||||
require.NoError(t, wrapExitErr(err), "enable IP forwarding in router NetNS")
|
||||
|
||||
// All non-UDP traffic should use regular masquerade e.g. for HTTP.
|
||||
_, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{
|
||||
"-t", "nat",
|
||||
"-A", "POSTROUTING",
|
||||
// Every interface except loopback.
|
||||
"!", "-o", "lo",
|
||||
// Every protocol except UDP.
|
||||
"!", "-p", "udp",
|
||||
"-j", "MASQUERADE",
|
||||
}).Output()
|
||||
require.NoError(t, wrapExitErr(err), "add iptables non-UDP masquerade rule")
|
||||
|
||||
// Outgoing traffic should get NATed to the router's IP.
|
||||
_, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{
|
||||
"-t", "nat",
|
||||
"-A", "POSTROUTING",
|
||||
"-p", "udp",
|
||||
"--sport", fmt.Sprint(leaf.clientPort),
|
||||
"-j", "SNAT",
|
||||
"--to-source", fmt.Sprintf("%s:%d", leaf.RouterIP, leaf.natPort),
|
||||
}).Output()
|
||||
require.NoError(t, wrapExitErr(err), "add iptables SNAT rule")
|
||||
|
||||
// Incoming traffic should be forwarded to the client's IP.
|
||||
_, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{
|
||||
"-t", "nat",
|
||||
"-A", "PREROUTING",
|
||||
"-p", "udp",
|
||||
"--dport", fmt.Sprint(leaf.natPort),
|
||||
"-j", "DNAT",
|
||||
"--to-destination", fmt.Sprintf("%s:%d", leaf.ClientIP, leaf.clientPort),
|
||||
}).Output()
|
||||
require.NoError(t, wrapExitErr(err), "add iptables DNAT rule")
|
||||
}
|
||||
|
||||
return internet
|
||||
}
|
||||
|
||||
// SetupNetworkingEasyNAT creates a fake internet and sets up "easy NAT"
|
||||
// forwarding rules.
|
||||
// See createFakeInternet.
|
||||
// NAT is achieved through a single iptables masquerade rule.
|
||||
func SetupNetworkingEasyNAT(t *testing.T, _ slog.Logger) TestNetworking {
|
||||
router := createFakeRouter(t)
|
||||
|
||||
// Set up iptables masquerade rules to allow the router to NAT packets
|
||||
// between the Three Kingdoms.
|
||||
_, err := commandInNetNS(router.RouterNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output()
|
||||
require.NoError(t, wrapExitErr(err), "enable IP forwarding in router NetNS")
|
||||
_, err = commandInNetNS(router.RouterNetNS, "iptables", []string{
|
||||
"-t", "nat",
|
||||
"-A", "POSTROUTING",
|
||||
// Every interface except loopback.
|
||||
"!", "-o", "lo",
|
||||
"-j", "MASQUERADE",
|
||||
}).Output()
|
||||
require.NoError(t, wrapExitErr(err), "add iptables masquerade rule")
|
||||
|
||||
return router.Net
|
||||
return easyNAT(t).Net
|
||||
}
|
||||
|
||||
type fakeRouter struct {
|
||||
// SetupNetworkingEasyNATWithSTUN does the same as SetupNetworkingEasyNAT, but
|
||||
// also creates a namespace and bridge address for a STUN server.
|
||||
func SetupNetworkingEasyNATWithSTUN(t *testing.T, _ slog.Logger) TestNetworking {
|
||||
internet := easyNAT(t)
|
||||
|
||||
// Create another network namespace for the STUN server.
|
||||
stunNetNS := createNetNS(t, internet.NamePrefix+"stun")
|
||||
internet.Net.STUN.Process = TestNetworkingProcess{
|
||||
NetNS: stunNetNS,
|
||||
}
|
||||
|
||||
const ip = "10.0.0.64"
|
||||
err := joinBridge(joinBridgeOpts{
|
||||
bridgeNetNS: internet.BridgeNetNS,
|
||||
netNS: stunNetNS,
|
||||
bridgeName: internet.BridgeName,
|
||||
vethPair: vethPair{
|
||||
Outer: internet.NamePrefix + "b-stun",
|
||||
Inner: internet.NamePrefix + "stun-b",
|
||||
},
|
||||
ip: ip,
|
||||
})
|
||||
require.NoError(t, err, "join bridge with STUN server")
|
||||
internet.Net.STUN.ListenAddr = ip + ":3478"
|
||||
|
||||
// Define custom DERP map.
|
||||
stunRegion := &tailcfg.DERPRegion{
|
||||
RegionID: 10000,
|
||||
RegionCode: "stun0",
|
||||
RegionName: "STUN0",
|
||||
Nodes: []*tailcfg.DERPNode{
|
||||
{
|
||||
Name: "stun0a",
|
||||
RegionID: 1,
|
||||
IPv4: ip,
|
||||
IPv6: "none",
|
||||
STUNPort: 3478,
|
||||
STUNOnly: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
client1DERP, err := internet.Net.Client1.ResolveDERPMap()
|
||||
require.NoError(t, err, "resolve DERP map for client 1")
|
||||
client1DERP.Regions[stunRegion.RegionID] = stunRegion
|
||||
internet.Net.Client1.DERPMap = client1DERP
|
||||
client2DERP, err := internet.Net.Client2.ResolveDERPMap()
|
||||
require.NoError(t, err, "resolve DERP map for client 2")
|
||||
client2DERP.Regions[stunRegion.RegionID] = stunRegion
|
||||
internet.Net.Client2.DERPMap = client2DERP
|
||||
|
||||
return internet.Net
|
||||
}
|
||||
|
||||
type vethPair struct {
|
||||
Outer string
|
||||
Inner string
|
||||
}
|
||||
|
||||
type fakeRouterLeaf struct {
|
||||
// RouterIP is the IP address of the router on the bridge.
|
||||
RouterIP string
|
||||
// ClientIP is the IP address of the client on the router.
|
||||
ClientIP string
|
||||
// RouterNetNS is the router for this specific leaf.
|
||||
RouterNetNS *os.File
|
||||
// ClientNetNS is where the "user" is.
|
||||
ClientNetNS *os.File
|
||||
// Veth pair between the router and the bridge.
|
||||
OuterVethPair vethPair
|
||||
// Veth pair between the user and the router.
|
||||
InnerVethPair vethPair
|
||||
}
|
||||
|
||||
type fakeInternet struct {
|
||||
Net TestNetworking
|
||||
|
||||
RouterNetNS *os.File
|
||||
RouterVeths struct {
|
||||
Server string
|
||||
Client1 string
|
||||
Client2 string
|
||||
}
|
||||
ServerNetNS *os.File
|
||||
ServerVeth string
|
||||
Client1NetNS *os.File
|
||||
Client1Veth string
|
||||
Client2NetNS *os.File
|
||||
Client2Veth string
|
||||
NamePrefix string
|
||||
BridgeNetNS *os.File
|
||||
BridgeName string
|
||||
ServerNetNS *os.File
|
||||
ServerVethPair vethPair // between bridge and server NS
|
||||
Client1 fakeRouterLeaf
|
||||
Client2 fakeRouterLeaf
|
||||
}
|
||||
|
||||
// fakeRouter creates multiple namespaces with veth pairs between them with
|
||||
// the following topology:
|
||||
// createFakeInternet creates multiple namespaces with veth pairs between them
|
||||
// with the following topology:
|
||||
//
|
||||
// namespaces:
|
||||
// - router
|
||||
// - server
|
||||
// - client1
|
||||
// - client2
|
||||
//
|
||||
// veth pairs:
|
||||
// - router-server (10.0.1.1) <-> server-router (10.0.1.2)
|
||||
// - router-client1 (10.0.2.1) <-> client1-router (10.0.2.2)
|
||||
// - router-client2 (10.0.3.1) <-> client2-router (10.0.3.2)
|
||||
// . veth ┌────────┐ veth
|
||||
// . ┌─────────────────┤ Bridge ├───────────────────┐
|
||||
// . │ └───┬────┘ │
|
||||
// . │ │ │
|
||||
// . │10.0.0.1 veth│10.0.0.2 │10.0.0.3
|
||||
// . ┌───────┴───────┐ ┌───────┴─────────┐ ┌────────┴────────┐
|
||||
// . │ Server │ │ Client 1 router │ │ Client 2 router │
|
||||
// . └───────────────┘ └───────┬─────────┘ └────────┬────────┘
|
||||
// . │10.0.2.1 │10.0.3.1
|
||||
// . veth│ veth│
|
||||
// . │10.0.2.2 │10.0.3.2
|
||||
// . ┌───────┴─────────┐ ┌────────┴────────┐
|
||||
// . │ Client 1 │ │ Client 2 │
|
||||
// . └─────────────────┘ └─────────────────┘
|
||||
//
|
||||
// No iptables rules are created, so packets will not be forwarded out of the
|
||||
// box. Routes are created between all namespaces based on the veth pairs,
|
||||
// however.
|
||||
func createFakeRouter(t *testing.T) fakeRouter {
|
||||
// box. Default routes are created from the edge namespaces (client1, client2)
|
||||
// to their respective routers, but no NAT rules are created.
|
||||
func createFakeInternet(t *testing.T) fakeInternet {
|
||||
t.Helper()
|
||||
const (
|
||||
routerServerPrefix = "10.0.1."
|
||||
routerServerIP = routerServerPrefix + "1"
|
||||
serverIP = routerServerPrefix + "2"
|
||||
routerClient1Prefix = "10.0.2."
|
||||
routerClient1IP = routerClient1Prefix + "1"
|
||||
client1IP = routerClient1Prefix + "2"
|
||||
routerClient2Prefix = "10.0.3."
|
||||
routerClient2IP = routerClient2Prefix + "1"
|
||||
client2IP = routerClient2Prefix + "2"
|
||||
bridgePrefix = "10.0.0."
|
||||
serverIP = bridgePrefix + "1"
|
||||
client1Prefix = "10.0.2."
|
||||
client2Prefix = "10.0.3."
|
||||
)
|
||||
var (
|
||||
namePrefix = uniqNetName(t) + "_"
|
||||
router = fakeInternet{
|
||||
NamePrefix: namePrefix,
|
||||
BridgeName: namePrefix + "b",
|
||||
}
|
||||
)
|
||||
|
||||
prefix := uniqNetName(t) + "_"
|
||||
router := fakeRouter{}
|
||||
router.RouterVeths.Server = prefix + "r-s"
|
||||
router.RouterVeths.Client1 = prefix + "r-c1"
|
||||
router.RouterVeths.Client2 = prefix + "r-c2"
|
||||
router.ServerVeth = prefix + "s-r"
|
||||
router.Client1Veth = prefix + "c1-r"
|
||||
router.Client2Veth = prefix + "c2-r"
|
||||
// Create bridge namespace and bridge interface.
|
||||
router.BridgeNetNS = createNetNS(t, router.BridgeName)
|
||||
err := createBridge(router.BridgeNetNS, router.BridgeName)
|
||||
require.NoError(t, err, "create bridge in netns")
|
||||
|
||||
// Create namespaces.
|
||||
router.RouterNetNS = createNetNS(t, prefix+"r")
|
||||
serverNS := createNetNS(t, prefix+"s")
|
||||
client1NS := createNetNS(t, prefix+"c1")
|
||||
client2NS := createNetNS(t, prefix+"c2")
|
||||
// Create server namespace and veth pair between bridge and server.
|
||||
router.ServerNetNS = createNetNS(t, namePrefix+"s")
|
||||
router.ServerVethPair = vethPair{
|
||||
Outer: namePrefix + "b-s",
|
||||
Inner: namePrefix + "s-b",
|
||||
}
|
||||
err = joinBridge(joinBridgeOpts{
|
||||
bridgeNetNS: router.BridgeNetNS,
|
||||
netNS: router.ServerNetNS,
|
||||
bridgeName: router.BridgeName,
|
||||
vethPair: router.ServerVethPair,
|
||||
ip: serverIP,
|
||||
})
|
||||
require.NoError(t, err, "join bridge with server")
|
||||
|
||||
vethPairs := []struct {
|
||||
parentName string
|
||||
peerName string
|
||||
parentNS *os.File
|
||||
peerNS *os.File
|
||||
parentIP string
|
||||
peerIP string
|
||||
leaves := []struct {
|
||||
leaf *fakeRouterLeaf
|
||||
routerName string
|
||||
clientName string
|
||||
routerBridgeIP string
|
||||
routerClientIP string
|
||||
clientIP string
|
||||
}{
|
||||
{
|
||||
parentName: router.RouterVeths.Server,
|
||||
peerName: router.ServerVeth,
|
||||
parentNS: router.RouterNetNS,
|
||||
peerNS: serverNS,
|
||||
parentIP: routerServerIP,
|
||||
peerIP: serverIP,
|
||||
leaf: &router.Client1,
|
||||
routerName: "c1r",
|
||||
clientName: "c1",
|
||||
routerBridgeIP: bridgePrefix + "2",
|
||||
routerClientIP: client1Prefix + "1",
|
||||
clientIP: client1Prefix + "2",
|
||||
},
|
||||
{
|
||||
parentName: router.RouterVeths.Client1,
|
||||
peerName: router.Client1Veth,
|
||||
parentNS: router.RouterNetNS,
|
||||
peerNS: client1NS,
|
||||
parentIP: routerClient1IP,
|
||||
peerIP: client1IP,
|
||||
},
|
||||
{
|
||||
parentName: router.RouterVeths.Client2,
|
||||
peerName: router.Client2Veth,
|
||||
parentNS: router.RouterNetNS,
|
||||
peerNS: client2NS,
|
||||
parentIP: routerClient2IP,
|
||||
peerIP: client2IP,
|
||||
leaf: &router.Client2,
|
||||
routerName: "c2r",
|
||||
clientName: "c2",
|
||||
routerBridgeIP: bridgePrefix + "3",
|
||||
routerClientIP: client2Prefix + "1",
|
||||
clientIP: client2Prefix + "2",
|
||||
},
|
||||
}
|
||||
|
||||
for _, vethPair := range vethPairs {
|
||||
err := createVethPair(vethPair.parentName, vethPair.peerName)
|
||||
require.NoErrorf(t, err, "create veth pair %q <-> %q", vethPair.parentName, vethPair.peerName)
|
||||
for _, leaf := range leaves {
|
||||
leaf.leaf.RouterIP = leaf.routerBridgeIP
|
||||
leaf.leaf.ClientIP = leaf.clientIP
|
||||
|
||||
// Move the veth interfaces to the respective network namespaces.
|
||||
err = setVethNetNS(vethPair.parentName, int(vethPair.parentNS.Fd()))
|
||||
require.NoErrorf(t, err, "set veth %q to NetNS", vethPair.parentName)
|
||||
err = setVethNetNS(vethPair.peerName, int(vethPair.peerNS.Fd()))
|
||||
require.NoErrorf(t, err, "set veth %q to NetNS", vethPair.peerName)
|
||||
// Create two network namespaces for each leaf: one for the router and
|
||||
// one for the "client".
|
||||
leaf.leaf.RouterNetNS = createNetNS(t, namePrefix+leaf.routerName)
|
||||
leaf.leaf.ClientNetNS = createNetNS(t, namePrefix+leaf.clientName)
|
||||
|
||||
// Set IP addresses on the interfaces.
|
||||
err = setInterfaceIP(vethPair.parentNS, vethPair.parentName, vethPair.parentIP)
|
||||
require.NoErrorf(t, err, "set IP %q on interface %q", vethPair.parentIP, vethPair.parentName)
|
||||
err = setInterfaceIP(vethPair.peerNS, vethPair.peerName, vethPair.peerIP)
|
||||
require.NoErrorf(t, err, "set IP %q on interface %q", vethPair.peerIP, vethPair.peerName)
|
||||
// Join the bridge.
|
||||
leaf.leaf.OuterVethPair = vethPair{
|
||||
Outer: namePrefix + "b-" + leaf.routerName,
|
||||
Inner: namePrefix + leaf.routerName + "-b",
|
||||
}
|
||||
err = joinBridge(joinBridgeOpts{
|
||||
bridgeNetNS: router.BridgeNetNS,
|
||||
netNS: leaf.leaf.RouterNetNS,
|
||||
bridgeName: router.BridgeName,
|
||||
vethPair: leaf.leaf.OuterVethPair,
|
||||
ip: leaf.routerBridgeIP,
|
||||
})
|
||||
require.NoError(t, err, "join bridge with router")
|
||||
|
||||
// Bring up both interfaces.
|
||||
err = setInterfaceUp(vethPair.parentNS, vethPair.parentName)
|
||||
require.NoErrorf(t, err, "bring up interface %q", vethPair.parentName)
|
||||
err = setInterfaceUp(vethPair.peerNS, vethPair.peerName)
|
||||
require.NoErrorf(t, err, "bring up interface %q", vethPair.parentName)
|
||||
// Create inner veth pair between the router and the client.
|
||||
leaf.leaf.InnerVethPair = vethPair{
|
||||
Outer: namePrefix + leaf.routerName + "-" + leaf.clientName,
|
||||
Inner: namePrefix + leaf.clientName + "-" + leaf.routerName,
|
||||
}
|
||||
err = createVethPair(leaf.leaf.InnerVethPair.Outer, leaf.leaf.InnerVethPair.Inner)
|
||||
require.NoErrorf(t, err, "create veth pair %q <-> %q", leaf.leaf.InnerVethPair.Outer, leaf.leaf.InnerVethPair.Inner)
|
||||
|
||||
// Move the network interfaces to the respective network namespaces.
|
||||
err = setVethNetNS(leaf.leaf.InnerVethPair.Outer, int(leaf.leaf.RouterNetNS.Fd()))
|
||||
require.NoErrorf(t, err, "set veth %q to NetNS", leaf.leaf.InnerVethPair.Outer)
|
||||
err = setVethNetNS(leaf.leaf.InnerVethPair.Inner, int(leaf.leaf.ClientNetNS.Fd()))
|
||||
require.NoErrorf(t, err, "set veth %q to NetNS", leaf.leaf.InnerVethPair.Inner)
|
||||
|
||||
// Set router's "local" IP on the veth.
|
||||
err = setInterfaceIP(leaf.leaf.RouterNetNS, leaf.leaf.InnerVethPair.Outer, leaf.routerClientIP)
|
||||
require.NoErrorf(t, err, "set IP %q on interface %q", leaf.routerClientIP, leaf.leaf.InnerVethPair.Outer)
|
||||
// Set client's IP on the veth.
|
||||
err = setInterfaceIP(leaf.leaf.ClientNetNS, leaf.leaf.InnerVethPair.Inner, leaf.clientIP)
|
||||
require.NoErrorf(t, err, "set IP %q on interface %q", leaf.clientIP, leaf.leaf.InnerVethPair.Inner)
|
||||
|
||||
// Bring up the interfaces.
|
||||
err = setInterfaceUp(leaf.leaf.RouterNetNS, leaf.leaf.InnerVethPair.Outer)
|
||||
require.NoErrorf(t, err, "bring up interface %q", leaf.leaf.OuterVethPair.Outer)
|
||||
err = setInterfaceUp(leaf.leaf.ClientNetNS, leaf.leaf.InnerVethPair.Inner)
|
||||
require.NoErrorf(t, err, "bring up interface %q", leaf.leaf.InnerVethPair.Inner)
|
||||
|
||||
// We don't need to add a route from parent to peer since the kernel
|
||||
// already adds a default route for the /24. We DO need to add a default
|
||||
// route from peer to parent, however.
|
||||
err = addRouteInNetNS(vethPair.peerNS, []string{"default", "via", vethPair.parentIP, "dev", vethPair.peerName})
|
||||
require.NoErrorf(t, err, "add peer default route to %q", vethPair.peerName)
|
||||
err = addRouteInNetNS(leaf.leaf.ClientNetNS, []string{"default", "via", leaf.routerClientIP, "dev", leaf.leaf.InnerVethPair.Inner})
|
||||
require.NoErrorf(t, err, "add peer default route to %q", leaf.leaf.InnerVethPair.Inner)
|
||||
}
|
||||
|
||||
router.Net = TestNetworking{
|
||||
ServerListenAddr: serverIP + ":8080",
|
||||
ServerAccessURLClient1: "http://" + serverIP + ":8080",
|
||||
ServerAccessURLClient2: "http://" + serverIP + ":8080",
|
||||
ProcessServer: TestNetworkingProcess{
|
||||
NetNS: serverNS,
|
||||
Server: TestNetworkingServer{
|
||||
Process: TestNetworkingProcess{NetNS: router.ServerNetNS},
|
||||
ListenAddr: serverIP + ":8080",
|
||||
},
|
||||
ProcessClient1: TestNetworkingProcess{
|
||||
NetNS: client1NS,
|
||||
Client1: TestNetworkingClient{
|
||||
Process: TestNetworkingProcess{NetNS: router.Client1.ClientNetNS},
|
||||
ServerAccessURL: "http://" + serverIP + ":8080",
|
||||
},
|
||||
ProcessClient2: TestNetworkingProcess{
|
||||
NetNS: client2NS,
|
||||
Client2: TestNetworkingClient{
|
||||
Process: TestNetworkingProcess{NetNS: router.Client2.ClientNetNS},
|
||||
ServerAccessURL: "http://" + serverIP + ":8080",
|
||||
},
|
||||
}
|
||||
return router
|
||||
@ -246,6 +416,60 @@ func uniqNetName(t *testing.T) string {
|
||||
return netNSName
|
||||
}
|
||||
|
||||
type joinBridgeOpts struct {
|
||||
bridgeNetNS *os.File
|
||||
netNS *os.File
|
||||
bridgeName string
|
||||
// This vethPair will be created and should not already exist.
|
||||
vethPair vethPair
|
||||
ip string
|
||||
}
|
||||
|
||||
// joinBridge joins the given network namespace to the bridge. It creates a veth
|
||||
// pair between the specified NetNS and the bridge NetNS, sets the IP address on
|
||||
// the "child" veth, and brings up the interfaces.
|
||||
func joinBridge(opts joinBridgeOpts) error {
|
||||
// Create outer veth pair between the router and the bridge.
|
||||
err := createVethPair(opts.vethPair.Outer, opts.vethPair.Inner)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("create veth pair %q <-> %q: %w", opts.vethPair.Outer, opts.vethPair.Inner, err)
|
||||
}
|
||||
|
||||
// Move the network interfaces to the respective network namespaces.
|
||||
err = setVethNetNS(opts.vethPair.Outer, int(opts.bridgeNetNS.Fd()))
|
||||
if err != nil {
|
||||
return xerrors.Errorf("set veth %q to NetNS: %w", opts.vethPair.Outer, err)
|
||||
}
|
||||
err = setVethNetNS(opts.vethPair.Inner, int(opts.netNS.Fd()))
|
||||
if err != nil {
|
||||
return xerrors.Errorf("set veth %q to NetNS: %w", opts.vethPair.Inner, err)
|
||||
}
|
||||
|
||||
// Connect the outer veth to the bridge.
|
||||
err = setInterfaceBridge(opts.bridgeNetNS, opts.vethPair.Outer, opts.bridgeName)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("set interface %q master to %q: %w", opts.vethPair.Outer, opts.bridgeName, err)
|
||||
}
|
||||
|
||||
// Set the bridge IP on the inner veth.
|
||||
err = setInterfaceIP(opts.netNS, opts.vethPair.Inner, opts.ip)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("set IP %q on interface %q: %w", opts.ip, opts.vethPair.Inner, err)
|
||||
}
|
||||
|
||||
// Bring up the interfaces.
|
||||
err = setInterfaceUp(opts.bridgeNetNS, opts.vethPair.Outer)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("bring up interface %q: %w", opts.vethPair.Outer, err)
|
||||
}
|
||||
err = setInterfaceUp(opts.netNS, opts.vethPair.Inner)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("bring up interface %q: %w", opts.vethPair.Inner, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// createNetNS creates a new network namespace with the given name. The returned
|
||||
// file is a file descriptor to the network namespace.
|
||||
// Note: all cleanup is handled for you, you do not need to call Close on the
|
||||
@ -283,18 +507,48 @@ func createNetNS(t *testing.T, name string) *os.File {
|
||||
return file
|
||||
}
|
||||
|
||||
// createBridge creates a bridge in the given network namespace. The bridge is
|
||||
// automatically brought up.
|
||||
func createBridge(netNS *os.File, name string) error {
|
||||
// While it might be possible to create a bridge directly in a NetNS or move
|
||||
// an existing bridge to a NetNS, I couldn't figure out a way to do it.
|
||||
// Creating it directly within the NetNS is the simplest way.
|
||||
_, err := commandInNetNS(netNS, "ip", []string{"link", "add", name, "type", "bridge"}).Output()
|
||||
if err != nil {
|
||||
return xerrors.Errorf("create bridge %q in netns: %w", name, wrapExitErr(err))
|
||||
}
|
||||
|
||||
_, err = commandInNetNS(netNS, "ip", []string{"link", "set", name, "up"}).Output()
|
||||
if err != nil {
|
||||
return xerrors.Errorf("set bridge %q up in netns: %w", name, wrapExitErr(err))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// setInterfaceBridge sets the master of the given interface to the specified
|
||||
// bridge.
|
||||
func setInterfaceBridge(netNS *os.File, ifaceName, bridgeName string) error {
|
||||
_, err := commandInNetNS(netNS, "ip", []string{"link", "set", ifaceName, "master", bridgeName}).Output()
|
||||
if err != nil {
|
||||
return xerrors.Errorf("set interface %q master to %q in netns: %w", ifaceName, bridgeName, wrapExitErr(err))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// createVethPair creates a veth pair with the given names.
|
||||
func createVethPair(parentVethName, peerVethName string) error {
|
||||
vethLinkAttrs := netlink.NewLinkAttrs()
|
||||
vethLinkAttrs.Name = parentVethName
|
||||
linkAttrs := netlink.NewLinkAttrs()
|
||||
linkAttrs.Name = parentVethName
|
||||
veth := &netlink.Veth{
|
||||
LinkAttrs: vethLinkAttrs,
|
||||
LinkAttrs: linkAttrs,
|
||||
PeerName: peerVethName,
|
||||
}
|
||||
|
||||
err := netlink.LinkAdd(veth)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("LinkAdd(name: %q, peerName: %q): %w", parentVethName, peerVethName, err)
|
||||
return xerrors.Errorf("LinkAdd(type: veth, name: %q, peerName: %q): %w", parentVethName, peerVethName, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
24
tailnet/test/integration/remove_test_ns.sh
Executable file
24
tailnet/test/integration/remove_test_ns.sh
Executable file
@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $(id -u) -ne 0 ]]; then
|
||||
echo "Please run with sudo"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
to_delete=$(ip netns list | grep -o 'cdr_.*_.*' | cut -d' ' -f1)
|
||||
echo "Will delete:"
|
||||
for ns in $to_delete; do
|
||||
echo "- $ns"
|
||||
done
|
||||
|
||||
read -p "Continue? [y/N] " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for ns in $to_delete; do
|
||||
ip netns delete "$ns"
|
||||
done
|
@ -7,7 +7,6 @@ import (
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"cdr.dev/slog"
|
||||
@ -17,12 +16,12 @@ import (
|
||||
|
||||
// TODO: instead of reusing one conn for each suite, maybe we should make a new
|
||||
// one for each subtest?
|
||||
func TestSuite(t *testing.T, _ slog.Logger, _ *url.URL, _, peerID uuid.UUID, conn *tailnet.Conn) {
|
||||
func TestSuite(t *testing.T, _ slog.Logger, _ *url.URL, conn *tailnet.Conn, _, peer Client) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("Connectivity", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
peerIP := tailnet.IPFromUUID(peerID)
|
||||
peerIP := tailnet.IPFromUUID(peer.ID)
|
||||
_, _, _, err := conn.Ping(testutil.Context(t, testutil.WaitLong), peerIP)
|
||||
require.NoError(t, err, "ping peer")
|
||||
})
|
||||
|
Reference in New Issue
Block a user