synced/coder
1
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2025-03-16 23:40:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(devtunnel): support geodistributed tunnels (#2711)

Browse Source
This commit is contained in:
Colin Adler
2022-06-30 19:11:13 -05:00
committed by GitHub
parent ae59f166fd
commit 482feef373
4 changed files with 189 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
coderd/devtunnel/servers.go Normal file
View File

@ -0,0 +1,95 @@
package devtunnel
import (
"runtime"
"sync"
"time"
"github.com/go-ping/ping"
"golang.org/x/exp/slices"
"golang.org/x/sync/errgroup"
"github.com/coder/coder/cryptorand"
)
type Region struct {
ID int
LocationName string
Nodes []Node
}
type Node struct {
ID int `json:"id"`
HostnameHTTPS string `json:"hostname_https"`
HostnameWireguard string `json:"hostname_wireguard"`
WireguardPort uint16 `json:"wireguard_port"`
AvgLatency time.Duration `json:"avg_latency"`
}
var Regions = []Region{
{
ID: 1,
LocationName: "US East Pittsburgh",
Nodes: []Node{
{
ID: 1,
HostnameHTTPS: "pit-1.try.coder.app",
HostnameWireguard: "pit-1.try.coder.app",
WireguardPort: 55551,
},
},
},
}
func FindClosestNode() (Node, error) {
nodes := []Node{}
for _, region := range Regions {
// Pick a random node from each region.
i, err := cryptorand.Intn(len(region.Nodes))
if err != nil {
return Node{}, err
}
nodes = append(nodes, region.Nodes[i])
}
var (
nodesMu sync.Mutex
eg = errgroup.Group{}
)
for i, node := range nodes {
i, node := i, node
eg.Go(func() error {
pinger, err := ping.NewPinger(node.HostnameHTTPS)
if err != nil {
return err
}
if runtime.GOOS == "windows" {
pinger.SetPrivileged(true)
}
pinger.Count = 5
err = pinger.Run()
if err != nil {
return err
}
nodesMu.Lock()
nodes[i].AvgLatency = pinger.Statistics().AvgRtt
nodesMu.Unlock()
return nil
})
}
err := eg.Wait()
if err != nil {
return Node{}, err
}
slices.SortFunc(nodes, func(i, j Node) bool {
return i.AvgLatency < j.AvgLatency
})
return nodes[0], nil
}

128
coderd/devtunnel/tunnel.go
View File

@ -23,14 +23,14 @@ import (
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
"cdr.dev/slog"
"github.com/coder/coder/cryptorand"
)
const (
EndpointWireguard = "wg-tunnel-udp.coder.app"
EndpointHTTPS = "wg-tunnel.coder.app"
var (
v0EndpointHTTPS = "wg-tunnel.coder.app"
ServerPublicKey = "+KNSMwed/IlqoesvTMSBNsHFaKVLrmmaCkn0bxIhUg0="
ServerUUID = "fcad0000-0000-4000-8000-000000000001"
v0ServerPublicKey = "+KNSMwed/IlqoesvTMSBNsHFaKVLrmmaCkn0bxIhUg0="
v0ServerIP = netip.AddrFrom16(uuid.MustParse("fcad0000-0000-4000-8000-000000000001"))
)
type Tunnel struct {
@ -39,25 +39,31 @@ type Tunnel struct {
}
type Config struct {
Version int `json:"version"`
ID uuid.UUID `json:"id"`
PrivateKey device.NoisePrivateKey `json:"private_key"`
PublicKey device.NoisePublicKey `json:"public_key"`
Tunnel Node `json:"tunnel"`
}
type configExt struct {
Version int `json:"-"`
ID uuid.UUID `json:"id"`
PrivateKey device.NoisePrivateKey `json:"-"`
PublicKey device.NoisePublicKey `json:"public_key"`
Tunnel Node `json:"-"`
}
// NewWithConfig calls New with the given config. For documentation, see New.
func NewWithConfig(ctx context.Context, logger slog.Logger, cfg Config) (*Tunnel, <-chan error, error) {
routineEnd, err := startUpdateRoutine(ctx, logger, cfg)
server, routineEnd, err := startUpdateRoutine(ctx, logger, cfg)
if err != nil {
return nil, nil, xerrors.Errorf("start update routine: %w", err)
}
tun, tnet, err := netstack.CreateNetTUN(
[]netip.Addr{netip.AddrFrom16(cfg.ID)},
[]netip.Addr{server.ClientIP},
[]netip.Addr{netip.AddrFrom4([4]byte{1, 1, 1, 1})},
1280,
)
@ -65,7 +71,7 @@ func NewWithConfig(ctx context.Context, logger slog.Logger, cfg Config) (*Tunnel
return nil, nil, xerrors.Errorf("create net TUN: %w", err)
}
wgip, err := net.ResolveIPAddr("ip", EndpointWireguard)
wgip, err := net.ResolveIPAddr("ip", cfg.Tunnel.HostnameWireguard)
if err != nil {
return nil, nil, xerrors.Errorf("resolve endpoint: %w", err)
}
@ -73,13 +79,14 @@ func NewWithConfig(ctx context.Context, logger slog.Logger, cfg Config) (*Tunnel
dev := device.NewDevice(tun, conn.NewDefaultBind(), device.NewLogger(device.LogLevelSilent, ""))
err = dev.IpcSet(fmt.Sprintf(`private_key=%s
public_key=%s
endpoint=%s:55555
endpoint=%s:%d
persistent_keepalive_interval=21
allowed_ip=%s/128`,
hex.EncodeToString(cfg.PrivateKey[:]),
encodeBase64ToHex(ServerPublicKey),
server.ServerPublicKey,
wgip.IP.String(),
netip.AddrFrom16(uuid.MustParse(ServerUUID)).String(),
cfg.Tunnel.WireguardPort,
server.ServerIP.String(),
))
if err != nil {
return nil, nil, xerrors.Errorf("configure wireguard ipc: %w", err)
@ -110,7 +117,7 @@ allowed_ip=%s/128`,
}()
return &Tunnel{
URL: fmt.Sprintf("https://%s.%s", cfg.ID, EndpointHTTPS),
URL: fmt.Sprintf("https://%s", server.Hostname),
Listener: wgListen,
}, ch, nil
}
@ -129,11 +136,11 @@ func New(ctx context.Context, logger slog.Logger) (*Tunnel, <-chan error, error)
return NewWithConfig(ctx, logger, cfg)
}
func startUpdateRoutine(ctx context.Context, logger slog.Logger, cfg Config) (<-chan struct{}, error) {
func startUpdateRoutine(ctx context.Context, logger slog.Logger, cfg Config) (ServerResponse, <-chan struct{}, error) {
// Ensure we send the first config before spawning in the background.
_, err := sendConfigToServer(ctx, cfg)
res, err := sendConfigToServer(ctx, cfg)
if err != nil {
return nil, xerrors.Errorf("send config to server: %w", err)
return ServerResponse{}, nil, xerrors.Errorf("send config to server: %w", err)
}
endCh := make(chan struct{})
@ -156,29 +163,67 @@ func startUpdateRoutine(ctx context.Context, logger slog.Logger, cfg Config) (<-
}
}
}()
return endCh, nil
return res, endCh, nil
}
func sendConfigToServer(ctx context.Context, cfg Config) (created bool, err error) {
type ServerResponse struct {
Hostname string `json:"hostname"`
ServerIP netip.Addr `json:"server_ip"`
ServerPublicKey string `json:"server_public_key"` // hex
ClientIP netip.Addr `json:"client_ip"`
}
func sendConfigToServer(ctx context.Context, cfg Config) (ServerResponse, error) {
raw, err := json.Marshal(configExt(cfg))
if err != nil {
return false, xerrors.Errorf("marshal config: %w", err)
return ServerResponse{}, xerrors.Errorf("marshal config: %w", err)
}
req, err := http.NewRequestWithContext(ctx, "POST", "https://"+EndpointHTTPS+"/tun", bytes.NewReader(raw))
if err != nil {
return false, xerrors.Errorf("new request: %w", err)
var req *http.Request
switch cfg.Version {
case 0:
req, err = http.NewRequestWithContext(ctx, "POST", "https://"+v0EndpointHTTPS+"/tun", bytes.NewReader(raw))
if err != nil {
return ServerResponse{}, xerrors.Errorf("new request: %w", err)
}
case 1:
req, err = http.NewRequestWithContext(ctx, "POST", "https://"+cfg.Tunnel.HostnameHTTPS+"/tun", bytes.NewReader(raw))
if err != nil {
return ServerResponse{}, xerrors.Errorf("new request: %w", err)
}
default:
return ServerResponse{}, xerrors.Errorf("unknown config version: %d", cfg.Version)
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return false, xerrors.Errorf("do request: %w", err)
return ServerResponse{}, xerrors.Errorf("do request: %w", err)
}
defer res.Body.Close()
var resp ServerResponse
switch cfg.Version {
case 0:
_, _ = io.Copy(io.Discard, res.Body)
resp.Hostname = fmt.Sprintf("%s.%s", cfg.ID, v0EndpointHTTPS)
resp.ServerIP = v0ServerIP
resp.ServerPublicKey = encodeBase64ToHex(v0ServerPublicKey)
resp.ClientIP = netip.AddrFrom16(cfg.ID)
case 1:
err := json.NewDecoder(res.Body).Decode(&resp)
if err != nil {
return ServerResponse{}, xerrors.Errorf("decode response: %w", err)
}
default:
_, _ = io.Copy(io.Discard, res.Body)
return ServerResponse{}, xerrors.Errorf("unknown config version: %d", cfg.Version)
}
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
return res.StatusCode == http.StatusCreated, nil
return resp, nil
}
func cfgPath() (string, error) {
@ -227,6 +272,15 @@ func readOrGenerateConfig() (Config, error) {
return Config{}, xerrors.Errorf("unmarshal config: %w", err)
}
if cfg.Version == 0 {
cfg.Tunnel = Node{
ID: 0,
HostnameHTTPS: "wg-tunnel.coder.app",
HostnameWireguard: "wg-tunnel-udp.coder.app",
WireguardPort: 55555,
}
}
return cfg, nil
}
@ -235,25 +289,25 @@ func GenerateConfig() (Config, error) {
if err != nil {
return Config{}, xerrors.Errorf("generate private key: %w", err)
}
pub := priv.PublicKey()
node, err := FindClosestNode()
if err != nil {
region := Regions[0]
n, _ := cryptorand.Intn(len(region.Nodes))
node = region.Nodes[n]
_, _ = fmt.Println("Error picking closest dev tunnel:", err)
_, _ = fmt.Println("Defaulting to", Regions[0].LocationName)
}
return Config{
ID: newUUID(),
Version: 1,
PrivateKey: device.NoisePrivateKey(priv),
PublicKey: device.NoisePublicKey(pub),
Tunnel: node,
}, nil
}
func newUUID() uuid.UUID {
u := uuid.New()
// 0xfc is the IPV6 prefix for internal networks.
u[0] = 0xfc
u[1] = 0xca
return u
}
func writeConfig(cfg Config) error {
cfgFi, err := cfgPath()
if err != nil {

1
go.mod
View File

@ -67,6 +67,7 @@ require (
github.com/go-chi/chi/v5 v5.0.7
github.com/go-chi/httprate v0.5.3
github.com/go-chi/render v1.0.1
github.com/go-ping/ping v1.1.0
github.com/go-playground/validator/v10 v10.11.0
github.com/gofrs/flock v0.8.1
github.com/gohugoio/hugo v0.101.0

2
go.sum
View File

@ -689,6 +689,8 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/go-ping/ping v1.1.0 h1:3MCGhVX4fyEUuhsfwPrsEdQw6xspHkv5zHsiSoDFZYw=
github.com/go-ping/ping v1.1.0/go.mod h1:xIFjORFzTxqIV/tDVGO4eDy/bLuSyawEeojSm3GfRGk=
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=