chore: Implement workspace proxy going away (graceful shutdown) (#7459)

* chore: Implement workspace proxy going away

When a workspace proxy shuts down, the health status of that
proxy should immediately be updated. This is purely a courtesy
and technically not required

enterprise/coderd/coderd.go

@@ -113,6 +113,7 @@ func New(ctx context.Context, options *Options) (*API, error) {
 				)
 				r.Post("/issue-signed-app-token", api.workspaceProxyIssueSignedAppToken)
 				r.Post("/register", api.workspaceProxyRegister)
+				r.Post("/goingaway", api.workspaceProxyGoingAway)
 			})
 			r.Route("/{workspaceproxy}", func(r chi.Router) {
 				r.Use(
@@ -239,7 +240,7 @@ func New(ctx context.Context, options *Options) (*API, error) {
 	if api.AGPL.Experiments.Enabled(codersdk.ExperimentMoons) {
 		// Proxy health is a moon feature.
 		api.ProxyHealth, err = proxyhealth.New(&proxyhealth.Options{
-			Interval:   time.Minute * 1,
+			Interval:   options.ProxyHealthInterval,
 			DB:         api.Database,
 			Logger:     options.Logger.Named("proxyhealth"),
 			Client:     api.HTTPClient,

enterprise/coderd/coderdenttest/coderdenttest.go

@@ -48,6 +48,7 @@ type Options struct {
 	EntitlementsUpdateInterval time.Duration
 	SCIMAPIKey                 []byte
 	UserWorkspaceQuota         int
+	ProxyHealthInterval        time.Duration
 }
 
 // New constructs a codersdk client connected to an in-memory Enterprise API instance.
@@ -74,6 +75,7 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c
 		Options:                    oop,
 		EntitlementsUpdateInterval: options.EntitlementsUpdateInterval,
 		Keys:                       Keys,
+		ProxyHealthInterval:        options.ProxyHealthInterval,
 	})
 	assert.NoError(t, err)
 	setHandler(coderAPI.AGPL.RootHandler)

enterprise/coderd/workspaceproxy.go

@@ -68,11 +68,7 @@ func (api *API) regions(rw http.ResponseWriter, r *http.Request) {
 				continue
 			}
 
-			health, ok := proxyHealth[proxy.ID]
-			if !ok {
-				health.Status = proxyhealth.Unknown
-			}
-
+			health := proxyHealth[proxy.ID]
 			regions = append(regions, codersdk.Region{
 				ID:               proxy.ID,
 				Name:             proxy.Name,
@@ -423,7 +419,7 @@ func (api *API) workspaceProxyRegister(rw http.ResponseWriter, r *http.Request)
 		//	Log:     api.Logger,
 		//	Request: r,
 		//	Action:  database.AuditActionWrite,
-		//})
+		// })
 	)
 	// aReq.Old = proxy
 	// defer commitAudit()
@@ -473,6 +469,33 @@ func (api *API) workspaceProxyRegister(rw http.ResponseWriter, r *http.Request)
 	go api.forceWorkspaceProxyHealthUpdate(api.ctx)
 }
 
+// workspaceProxyGoingAway is used to tell coderd that the workspace proxy is
+// shutting down and going away. The main purpose of this function is for the
+// health status of the workspace proxy to be more quickly updated when we know
+// that the proxy is going to be unhealthy. This does not delete the workspace
+// or cause any other side effects.
+// If the workspace proxy comes back online, even without a register, it will
+// be found healthy again by the normal checks.
+// @Summary Workspace proxy going away
+// @ID workspace-proxy-going-away
+// @Security CoderSessionToken
+// @Produce json
+// @Tags Enterprise
+// @Success 201 {object} codersdk.Response
+// @Router /workspaceproxies/me/goingaway [post]
+// @x-apidocgen {"skip": true}
+func (api *API) workspaceProxyGoingAway(rw http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	// Force a health update to happen immediately. The proxy should
+	// not return a successful response if it is going away.
+	go api.forceWorkspaceProxyHealthUpdate(api.ctx)
+
+	httpapi.Write(ctx, rw, http.StatusOK, codersdk.Response{
+		Message: "OK",
+	})
+}
+
 // reconnectingPTYSignedToken issues a signed app token for use when connecting
 // to the reconnecting PTY websocket on an external workspace proxy. This is set
 // by the client as a query parameter when connecting.
@@ -588,6 +611,9 @@ func convertProxies(p []database.WorkspaceProxy, statuses map[uuid.UUID]proxyhea
 }
 
 func convertProxy(p database.WorkspaceProxy, status proxyhealth.ProxyStatus) codersdk.WorkspaceProxy {
+	if status.Status == "" {
+		status.Status = proxyhealth.Unknown
+	}
 	return codersdk.WorkspaceProxy{
 		ID:               p.ID,
 		Name:             p.Name,

enterprise/coderd/workspaceproxy_test.go

@@ -7,6 +7,7 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"testing"
+	"time"
 
 	"github.com/google/uuid"
 	"github.com/moby/moby/pkg/namesgenerator"
@@ -172,6 +173,69 @@ func TestRegions(t *testing.T) {
 		require.Error(t, err)
 		require.Empty(t, regions)
 	})
+
+	t.Run("GoingAway", func(t *testing.T) {
+		t.Parallel()
+
+		dv := coderdtest.DeploymentValues(t)
+		dv.Experiments = []string{
+			string(codersdk.ExperimentMoons),
+			"*",
+		}
+
+		db, pubsub := dbtestutil.NewDB(t)
+
+		ctx := testutil.Context(t, testutil.WaitLong)
+
+		client, closer, api := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
+			Options: &coderdtest.Options{
+				AppHostname:      appHostname,
+				Database:         db,
+				Pubsub:           pubsub,
+				DeploymentValues: dv,
+			},
+			// The interval is set to 1 hour so the proxy health
+			// check will never happen manually. All checks will be
+			// forced updates.
+			ProxyHealthInterval: time.Hour,
+		})
+		t.Cleanup(func() {
+			_ = closer.Close()
+		})
+		_ = coderdtest.CreateFirstUser(t, client)
+		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
+			Features: license.Features{
+				codersdk.FeatureWorkspaceProxy: 1,
+			},
+		})
+
+		const proxyName = "testproxy"
+		proxy := coderdenttest.NewWorkspaceProxy(t, api, client, &coderdenttest.ProxyOptions{
+			Name: proxyName,
+		})
+		_ = proxy
+
+		require.Eventuallyf(t, func() bool {
+			proxy, err := client.WorkspaceProxyByName(ctx, proxyName)
+			if err != nil {
+				// We are testing the going away, not the initial healthy.
+				// Just force an update to change this to healthy.
+				_ = api.ProxyHealth.ForceUpdate(ctx)
+				return false
+			}
+			return proxy.Status.Status == codersdk.ProxyHealthy
+		}, testutil.WaitShort, testutil.IntervalFast, "proxy never became healthy")
+
+		_ = proxy.Close()
+		// The proxy should tell the primary on close that is is no longer healthy.
+		require.Eventuallyf(t, func() bool {
+			proxy, err := client.WorkspaceProxyByName(ctx, proxyName)
+			if err != nil {
+				return false
+			}
+			return proxy.Status.Status == codersdk.ProxyUnhealthy
+		}, testutil.WaitShort, testutil.IntervalFast, "proxy never became unhealthy after close")
+	})
 }
 
 func TestWorkspaceProxyCRUD(t *testing.T) {