fix: Improve coder server shutdown procedure (#3246)

* fix: Improve `coder server` shutdown procedure

This commit improves the `coder server` shutdown procedure so that all
triggers for shutdown do so in a graceful way without skipping any
steps.

We also improve cancellation and shutdown of services by ensuring
resources are cleaned up at the end.

Notable changes:
- We wrap `cmd.Context()` to allow us to control cancellation better
- We attempt graceful shutdown of the http server (`server.Shutdown`)
  because it's less abrupt (compared to `shutdownConns`)
- All exit paths share the same shutdown procedure (except for early
  exit)
- `provisionerd`s are now shutdown concurrently instead of one at a
  time, the also now get a new context for shutdown because
  `cmd.Context()` may be cancelled
- Resources created by `newProvisionerDaemon` are cleaned up
- Lifecycle `Executor` exits its goroutine on context cancellation

Fixes #3245

coderd/autobuild/executor/lifecycle_executor.go

@@ -54,15 +54,27 @@ func (e *Executor) WithStatsChannel(ch chan<- Stats) *Executor {
 // its channel is closed.
 func (e *Executor) Run() {
 	go func() {
-		for t := range e.tick {
-			stats := e.runOnce(t)
-			if stats.Error != nil {
-				e.log.Error(e.ctx, "error running once", slog.Error(stats.Error))
+		for {
+			select {
+			case <-e.ctx.Done():
+				return
+			case t, ok := <-e.tick:
+				if !ok {
+					return
+				}
+				stats := e.runOnce(t)
+				if stats.Error != nil {
+					e.log.Error(e.ctx, "error running once", slog.Error(stats.Error))
+				}
+				if e.statsCh != nil {
+					select {
+					case <-e.ctx.Done():
+						return
+					case e.statsCh <- stats:
+					}
+				}
+				e.log.Debug(e.ctx, "run stats", slog.F("elapsed", stats.Elapsed), slog.F("transitions", stats.Transitions))
 			}
-			if e.statsCh != nil {
-				e.statsCh <- stats
-			}
-			e.log.Debug(e.ctx, "run stats", slog.F("elapsed", stats.Elapsed), slog.F("transitions", stats.Transitions))
 		}
 	}()
 }