feat(agent): add devcontainer autostart support (#17076)

This change adds support for devcontainer autostart in workspaces. The
preconditions for utilizing this feature are:

1. The `coder_devcontainer` resource must be defined in Terraform
2. By the time the startup scripts have completed,
	- The `@devcontainers/cli` tool must be installed
	- The given workspace folder must contain a devcontainer configuration

Example Terraform:

```tf
resource "coder_devcontainer" "coder" {
  agent_id         = coder_agent.main.id
  workspace_folder = "/home/coder/coder"
  config_path      = ".devcontainer/devcontainer.json" # (optional)
}
```

Closes #16423

agent/agent.go

@@ -1075,7 +1075,7 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 		//
 		// An example is VS Code Remote, which must know the directory
 		// before initializing a connection.
-		manifest.Directory, err = expandDirectory(manifest.Directory)
+		manifest.Directory, err = expandPathToAbs(manifest.Directory)
 		if err != nil {
 			return xerrors.Errorf("expand directory: %w", err)
 		}
@@ -1115,16 +1115,35 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context,
 				}
 			}
 
-			err = a.scriptRunner.Init(manifest.Scripts, aAPI.ScriptCompleted)
+			var (
+				scripts          = manifest.Scripts
+				scriptRunnerOpts []agentscripts.InitOption
+			)
+			if a.experimentalDevcontainersEnabled {
+				var dcScripts []codersdk.WorkspaceAgentScript
+				scripts, dcScripts = agentcontainers.ExtractAndInitializeDevcontainerScripts(a.logger, expandPathToAbs, manifest.Devcontainers, scripts)
+				// See ExtractAndInitializeDevcontainerScripts for motivation
+				// behind running dcScripts as post start scripts.
+				scriptRunnerOpts = append(scriptRunnerOpts, agentscripts.WithPostStartScripts(dcScripts...))
+			}
+			err = a.scriptRunner.Init(scripts, aAPI.ScriptCompleted, scriptRunnerOpts...)
 			if err != nil {
 				return xerrors.Errorf("init script runner: %w", err)
 			}
 			err = a.trackGoroutine(func() {
 				start := time.Now()
-				// here we use the graceful context because the script runner is not directly tied
+				// Here we use the graceful context because the script runner is
-				// to the agent API.
+				// not directly tied to the agent API.
+				//
+				// First we run the start scripts to ensure the workspace has
+				// been initialized and then the post start scripts which may
+				// depend on the workspace start scripts.
+				//
+				// Measure the time immediately after the start scripts have
+				// finished (both start and post start). For instance, an
+				// autostarted devcontainer will be included in this time.
 				err := a.scriptRunner.Execute(a.gracefulCtx, agentscripts.ExecuteStartScripts)
-				// Measure the time immediately after the script has finished
+				err = errors.Join(err, a.scriptRunner.Execute(a.gracefulCtx, agentscripts.ExecutePostStartScripts))
 				dur := time.Since(start).Seconds()
 				if err != nil {
 					a.logger.Warn(ctx, "startup script(s) failed", slog.Error(err))
@@ -1851,30 +1870,29 @@ func userHomeDir() (string, error) {
 	return u.HomeDir, nil
 }
 
-// expandDirectory converts a directory path to an absolute path.
+// expandPathToAbs converts a path to an absolute path. It primarily resolves
-// It primarily resolves the home directory and any environment
+// the home directory and any environment variables that may be set.
-// variables that may be set
+func expandPathToAbs(path string) (string, error) {
-func expandDirectory(dir string) (string, error) {
+	if path == "" {
-	if dir == "" {
 		return "", nil
 	}
-	if dir[0] == '~' {
+	if path[0] == '~' {
 		home, err := userHomeDir()
 		if err != nil {
 			return "", err
 		}
-		dir = filepath.Join(home, dir[1:])
+		path = filepath.Join(home, path[1:])
 	}
-	dir = os.ExpandEnv(dir)
+	path = os.ExpandEnv(path)
 
-	if !filepath.IsAbs(dir) {
+	if !filepath.IsAbs(path) {
 		home, err := userHomeDir()
 		if err != nil {
 			return "", err
 		}
-		dir = filepath.Join(home, dir)
+		path = filepath.Join(home, path)
 	}
-	return dir, nil
+	return path, nil
 }
 
 // EnvAgentSubsystem is the environment variable used to denote the

agent/agent_test.go

@@ -1937,6 +1937,134 @@ func TestAgent_ReconnectingPTYContainer(t *testing.T) {
 	require.ErrorIs(t, tr.ReadUntil(ctx, nil), io.EOF)
 }
 
+// This tests end-to-end functionality of auto-starting a devcontainer.
+// It runs "devcontainer up" which creates a real Docker container. As
+// such, it does not run by default in CI.
+//
+// You can run it manually as follows:
+//
+// CODER_TEST_USE_DOCKER=1 go test -count=1 ./agent -run TestAgent_DevcontainerAutostart
+func TestAgent_DevcontainerAutostart(t *testing.T) {
+	t.Parallel()
+	if os.Getenv("CODER_TEST_USE_DOCKER") != "1" {
+		t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test")
+	}
+
+	ctx := testutil.Context(t, testutil.WaitLong)
+
+	// Connect to Docker
+	pool, err := dockertest.NewPool("")
+	require.NoError(t, err, "Could not connect to docker")
+
+	// Prepare temporary devcontainer for test (mywork).
+	devcontainerID := uuid.New()
+	tempWorkspaceFolder := t.TempDir()
+	tempWorkspaceFolder = filepath.Join(tempWorkspaceFolder, "mywork")
+	t.Logf("Workspace folder: %s", tempWorkspaceFolder)
+	devcontainerPath := filepath.Join(tempWorkspaceFolder, ".devcontainer")
+	err = os.MkdirAll(devcontainerPath, 0o755)
+	require.NoError(t, err, "create devcontainer directory")
+	devcontainerFile := filepath.Join(devcontainerPath, "devcontainer.json")
+	err = os.WriteFile(devcontainerFile, []byte(`{
+        "name": "mywork",
+        "image": "busybox:latest",
+        "cmd": ["sleep", "infinity"]
+    }`), 0o600)
+	require.NoError(t, err, "write devcontainer.json")
+
+	manifest := agentsdk.Manifest{
+		// Set up pre-conditions for auto-starting a devcontainer, the script
+		// is expected to be prepared by the provisioner normally.
+		Devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+			{
+				ID:              devcontainerID,
+				Name:            "test",
+				WorkspaceFolder: tempWorkspaceFolder,
+			},
+		},
+		Scripts: []codersdk.WorkspaceAgentScript{
+			{
+				ID:          devcontainerID,
+				LogSourceID: agentsdk.ExternalLogSourceID,
+				RunOnStart:  true,
+				Script:      "echo this-will-be-replaced",
+				DisplayName: "Dev Container (test)",
+			},
+		},
+	}
+	// nolint: dogsled
+	conn, _, _, _, _ := setupAgent(t, manifest, 0, func(_ *agenttest.Client, o *agent.Options) {
+		o.ExperimentalDevcontainersEnabled = true
+	})
+
+	t.Logf("Waiting for container with label: devcontainer.local_folder=%s", tempWorkspaceFolder)
+
+	var container docker.APIContainers
+	require.Eventually(t, func() bool {
+		containers, err := pool.Client.ListContainers(docker.ListContainersOptions{All: true})
+		if err != nil {
+			t.Logf("Error listing containers: %v", err)
+			return false
+		}
+
+		for _, c := range containers {
+			t.Logf("Found container: %s with labels: %v", c.ID[:12], c.Labels)
+			if labelValue, ok := c.Labels["devcontainer.local_folder"]; ok {
+				if labelValue == tempWorkspaceFolder {
+					t.Logf("Found matching container: %s", c.ID[:12])
+					container = c
+					return true
+				}
+			}
+		}
+
+		return false
+	}, testutil.WaitSuperLong, testutil.IntervalMedium, "no container with workspace folder label found")
+
+	t.Cleanup(func() {
+		// We can't rely on pool here because the container is not
+		// managed by it (it is managed by @devcontainer/cli).
+		err := pool.Client.RemoveContainer(docker.RemoveContainerOptions{
+			ID:            container.ID,
+			RemoveVolumes: true,
+			Force:         true,
+		})
+		assert.NoError(t, err, "remove container")
+	})
+
+	containerInfo, err := pool.Client.InspectContainer(container.ID)
+	require.NoError(t, err, "inspect container")
+	t.Logf("Container state: status: %v", containerInfo.State.Status)
+	require.True(t, containerInfo.State.Running, "container should be running")
+
+	ac, err := conn.ReconnectingPTY(ctx, uuid.New(), 80, 80, "", func(opts *workspacesdk.AgentReconnectingPTYInit) {
+		opts.Container = container.ID
+	})
+	require.NoError(t, err, "failed to create ReconnectingPTY")
+	defer ac.Close()
+
+	// Use terminal reader so we can see output in case somethin goes wrong.
+	tr := testutil.NewTerminalReader(t, ac)
+
+	require.NoError(t, tr.ReadUntil(ctx, func(line string) bool {
+		return strings.Contains(line, "#") || strings.Contains(line, "$")
+	}), "find prompt")
+
+	wantFileName := "file-from-devcontainer"
+	wantFile := filepath.Join(tempWorkspaceFolder, wantFileName)
+
+	require.NoError(t, json.NewEncoder(ac).Encode(workspacesdk.ReconnectingPTYRequest{
+		// NOTE(mafredri): We must use absolute path here for some reason.
+		Data: fmt.Sprintf("touch /workspaces/mywork/%s; exit\r", wantFileName),
+	}), "create file inside devcontainer")
+
+	// Wait for the connection to close to ensure the touch was executed.
+	require.ErrorIs(t, tr.ReadUntil(ctx, nil), io.EOF)
+
+	_, err = os.Stat(wantFile)
+	require.NoError(t, err, "file should exist outside devcontainer")
+}
+
 func TestAgent_Dial(t *testing.T) {
 	t.Parallel()
 

agent/agentcontainers/devcontainer.go

@@ -0,0 +1,98 @@
+package agentcontainers
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"cdr.dev/slog"
+
+	"github.com/coder/coder/v2/codersdk"
+)
+
+const devcontainerUpScriptTemplate = `
+if ! which devcontainer > /dev/null 2>&1; then
+  echo "ERROR: Unable to start devcontainer, @devcontainers/cli is not installed."
+  exit 1
+fi
+devcontainer up %s
+`
+
+// ExtractAndInitializeDevcontainerScripts extracts devcontainer scripts from
+// the given scripts and devcontainers. The devcontainer scripts are removed
+// from the returned scripts so that they can be run separately.
+//
+// Dev Containers have an inherent dependency on start scripts, since they
+// initialize the workspace (e.g. git clone, npm install, etc). This is
+// important if e.g. a Coder module to install @devcontainer/cli is used.
+func ExtractAndInitializeDevcontainerScripts(
+	logger slog.Logger,
+	expandPath func(string) (string, error),
+	devcontainers []codersdk.WorkspaceAgentDevcontainer,
+	scripts []codersdk.WorkspaceAgentScript,
+) (filteredScripts []codersdk.WorkspaceAgentScript, devcontainerScripts []codersdk.WorkspaceAgentScript) {
+ScriptLoop:
+	for _, script := range scripts {
+		for _, dc := range devcontainers {
+			// The devcontainer scripts match the devcontainer ID for
+			// identification.
+			if script.ID == dc.ID {
+				dc = expandDevcontainerPaths(logger, expandPath, dc)
+				devcontainerScripts = append(devcontainerScripts, devcontainerStartupScript(dc, script))
+				continue ScriptLoop
+			}
+		}
+
+		filteredScripts = append(filteredScripts, script)
+	}
+
+	return filteredScripts, devcontainerScripts
+}
+
+func devcontainerStartupScript(dc codersdk.WorkspaceAgentDevcontainer, script codersdk.WorkspaceAgentScript) codersdk.WorkspaceAgentScript {
+	var args []string
+	args = append(args, fmt.Sprintf("--workspace-folder %q", dc.WorkspaceFolder))
+	if dc.ConfigPath != "" {
+		args = append(args, fmt.Sprintf("--config %q", dc.ConfigPath))
+	}
+	cmd := fmt.Sprintf(devcontainerUpScriptTemplate, strings.Join(args, " "))
+	script.Script = cmd
+	// Disable RunOnStart, scripts have this set so that when devcontainers
+	// have not been enabled, a warning will be surfaced in the agent logs.
+	script.RunOnStart = false
+	return script
+}
+
+func expandDevcontainerPaths(logger slog.Logger, expandPath func(string) (string, error), dc codersdk.WorkspaceAgentDevcontainer) codersdk.WorkspaceAgentDevcontainer {
+	logger = logger.With(slog.F("devcontainer", dc.Name), slog.F("workspace_folder", dc.WorkspaceFolder), slog.F("config_path", dc.ConfigPath))
+
+	if wf, err := expandPath(dc.WorkspaceFolder); err != nil {
+		logger.Warn(context.Background(), "expand devcontainer workspace folder failed", slog.Error(err))
+	} else {
+		dc.WorkspaceFolder = wf
+	}
+	if dc.ConfigPath != "" {
+		// Let expandPath handle home directory, otherwise assume relative to
+		// workspace folder or absolute.
+		if dc.ConfigPath[0] == '~' {
+			if cp, err := expandPath(dc.ConfigPath); err != nil {
+				logger.Warn(context.Background(), "expand devcontainer config path failed", slog.Error(err))
+			} else {
+				dc.ConfigPath = cp
+			}
+		} else {
+			dc.ConfigPath = relativePathToAbs(dc.WorkspaceFolder, dc.ConfigPath)
+		}
+	}
+	return dc
+}
+
+func relativePathToAbs(workdir, path string) string {
+	path = os.ExpandEnv(path)
+	if !filepath.IsAbs(path) {
+		path = filepath.Join(workdir, path)
+	}
+	return path
+}

agent/agentcontainers/devcontainer_test.go

@@ -0,0 +1,276 @@
+package agentcontainers_test
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+
+	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/v2/agent/agentcontainers"
+	"github.com/coder/coder/v2/codersdk"
+)
+
+func TestExtractAndInitializeDevcontainerScripts(t *testing.T) {
+	t.Parallel()
+
+	scriptIDs := []uuid.UUID{uuid.New(), uuid.New()}
+	devcontainerIDs := []uuid.UUID{uuid.New(), uuid.New()}
+
+	type args struct {
+		expandPath    func(string) (string, error)
+		devcontainers []codersdk.WorkspaceAgentDevcontainer
+		scripts       []codersdk.WorkspaceAgentScript
+	}
+	tests := []struct {
+		name                    string
+		args                    args
+		wantFilteredScripts     []codersdk.WorkspaceAgentScript
+		wantDevcontainerScripts []codersdk.WorkspaceAgentScript
+
+		skipOnWindowsDueToPathSeparator bool
+	}{
+		{
+			name: "no scripts",
+			args: args{
+				expandPath:    nil,
+				devcontainers: nil,
+				scripts:       nil,
+			},
+			wantFilteredScripts:     nil,
+			wantDevcontainerScripts: nil,
+		},
+		{
+			name: "no devcontainers",
+			args: args{
+				expandPath:    nil,
+				devcontainers: nil,
+				scripts: []codersdk.WorkspaceAgentScript{
+					{ID: scriptIDs[0]},
+					{ID: scriptIDs[1]},
+				},
+			},
+			wantFilteredScripts: []codersdk.WorkspaceAgentScript{
+				{ID: scriptIDs[0]},
+				{ID: scriptIDs[1]},
+			},
+			wantDevcontainerScripts: nil,
+		},
+		{
+			name: "no scripts match devcontainers",
+			args: args{
+				expandPath: nil,
+				devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+					{ID: devcontainerIDs[0]},
+					{ID: devcontainerIDs[1]},
+				},
+				scripts: []codersdk.WorkspaceAgentScript{
+					{ID: scriptIDs[0]},
+					{ID: scriptIDs[1]},
+				},
+			},
+			wantFilteredScripts: []codersdk.WorkspaceAgentScript{
+				{ID: scriptIDs[0]},
+				{ID: scriptIDs[1]},
+			},
+			wantDevcontainerScripts: nil,
+		},
+		{
+			name: "scripts match devcontainers and sets RunOnStart=false",
+			args: args{
+				expandPath: nil,
+				devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+					{ID: devcontainerIDs[0], WorkspaceFolder: "workspace1"},
+					{ID: devcontainerIDs[1], WorkspaceFolder: "workspace2"},
+				},
+				scripts: []codersdk.WorkspaceAgentScript{
+					{ID: scriptIDs[0], RunOnStart: true},
+					{ID: scriptIDs[1], RunOnStart: true},
+					{ID: devcontainerIDs[0], RunOnStart: true},
+					{ID: devcontainerIDs[1], RunOnStart: true},
+				},
+			},
+			wantFilteredScripts: []codersdk.WorkspaceAgentScript{
+				{ID: scriptIDs[0], RunOnStart: true},
+				{ID: scriptIDs[1], RunOnStart: true},
+			},
+			wantDevcontainerScripts: []codersdk.WorkspaceAgentScript{
+				{
+					ID:         devcontainerIDs[0],
+					Script:     "devcontainer up --workspace-folder \"workspace1\"",
+					RunOnStart: false,
+				},
+				{
+					ID:         devcontainerIDs[1],
+					Script:     "devcontainer up --workspace-folder \"workspace2\"",
+					RunOnStart: false,
+				},
+			},
+		},
+		{
+			name: "scripts match devcontainers with config path",
+			args: args{
+				expandPath: nil,
+				devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+					{
+						ID:              devcontainerIDs[0],
+						WorkspaceFolder: "workspace1",
+						ConfigPath:      "config1",
+					},
+					{
+						ID:              devcontainerIDs[1],
+						WorkspaceFolder: "workspace2",
+						ConfigPath:      "config2",
+					},
+				},
+				scripts: []codersdk.WorkspaceAgentScript{
+					{ID: devcontainerIDs[0]},
+					{ID: devcontainerIDs[1]},
+				},
+			},
+			wantFilteredScripts: []codersdk.WorkspaceAgentScript{},
+			wantDevcontainerScripts: []codersdk.WorkspaceAgentScript{
+				{
+					ID:         devcontainerIDs[0],
+					Script:     "devcontainer up --workspace-folder \"workspace1\" --config \"workspace1/config1\"",
+					RunOnStart: false,
+				},
+				{
+					ID:         devcontainerIDs[1],
+					Script:     "devcontainer up --workspace-folder \"workspace2\" --config \"workspace2/config2\"",
+					RunOnStart: false,
+				},
+			},
+			skipOnWindowsDueToPathSeparator: true,
+		},
+		{
+			name: "scripts match devcontainers with expand path",
+			args: args{
+				expandPath: func(s string) (string, error) {
+					return "/home/" + s, nil
+				},
+				devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+					{
+						ID:              devcontainerIDs[0],
+						WorkspaceFolder: "workspace1",
+						ConfigPath:      "config1",
+					},
+					{
+						ID:              devcontainerIDs[1],
+						WorkspaceFolder: "workspace2",
+						ConfigPath:      "config2",
+					},
+				},
+				scripts: []codersdk.WorkspaceAgentScript{
+					{ID: devcontainerIDs[0], RunOnStart: true},
+					{ID: devcontainerIDs[1], RunOnStart: true},
+				},
+			},
+			wantFilteredScripts: []codersdk.WorkspaceAgentScript{},
+			wantDevcontainerScripts: []codersdk.WorkspaceAgentScript{
+				{
+					ID:         devcontainerIDs[0],
+					Script:     "devcontainer up --workspace-folder \"/home/workspace1\" --config \"/home/workspace1/config1\"",
+					RunOnStart: false,
+				},
+				{
+					ID:         devcontainerIDs[1],
+					Script:     "devcontainer up --workspace-folder \"/home/workspace2\" --config \"/home/workspace2/config2\"",
+					RunOnStart: false,
+				},
+			},
+			skipOnWindowsDueToPathSeparator: true,
+		},
+		{
+			name: "expand config path when ~",
+			args: args{
+				expandPath: func(s string) (string, error) {
+					s = strings.Replace(s, "~/", "", 1)
+					if filepath.IsAbs(s) {
+						return s, nil
+					}
+					return "/home/" + s, nil
+				},
+				devcontainers: []codersdk.WorkspaceAgentDevcontainer{
+					{
+						ID:              devcontainerIDs[0],
+						WorkspaceFolder: "workspace1",
+						ConfigPath:      "~/config1",
+					},
+					{
+						ID:              devcontainerIDs[1],
+						WorkspaceFolder: "workspace2",
+						ConfigPath:      "/config2",
+					},
+				},
+				scripts: []codersdk.WorkspaceAgentScript{
+					{ID: devcontainerIDs[0], RunOnStart: true},
+					{ID: devcontainerIDs[1], RunOnStart: true},
+				},
+			},
+			wantFilteredScripts: []codersdk.WorkspaceAgentScript{},
+			wantDevcontainerScripts: []codersdk.WorkspaceAgentScript{
+				{
+					ID:         devcontainerIDs[0],
+					Script:     "devcontainer up --workspace-folder \"/home/workspace1\" --config \"/home/config1\"",
+					RunOnStart: false,
+				},
+				{
+					ID:         devcontainerIDs[1],
+					Script:     "devcontainer up --workspace-folder \"/home/workspace2\" --config \"/config2\"",
+					RunOnStart: false,
+				},
+			},
+			skipOnWindowsDueToPathSeparator: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			if tt.skipOnWindowsDueToPathSeparator && filepath.Separator == '\\' {
+				t.Skip("Skipping test on Windows due to path separator difference.")
+			}
+
+			logger := slogtest.Make(t, nil)
+			if tt.args.expandPath == nil {
+				tt.args.expandPath = func(s string) (string, error) {
+					return s, nil
+				}
+			}
+			gotFilteredScripts, gotDevcontainerScripts := agentcontainers.ExtractAndInitializeDevcontainerScripts(
+				logger,
+				tt.args.expandPath,
+				tt.args.devcontainers,
+				tt.args.scripts,
+			)
+
+			if diff := cmp.Diff(tt.wantFilteredScripts, gotFilteredScripts, cmpopts.EquateEmpty()); diff != "" {
+				t.Errorf("ExtractAndInitializeDevcontainerScripts() gotFilteredScripts mismatch (-want +got):\n%s", diff)
+			}
+
+			// Preprocess the devcontainer scripts to remove scripting part.
+			for i := range gotDevcontainerScripts {
+				gotDevcontainerScripts[i].Script = textGrep("devcontainer up", gotDevcontainerScripts[i].Script)
+				require.NotEmpty(t, gotDevcontainerScripts[i].Script, "devcontainer up script not found")
+			}
+			if diff := cmp.Diff(tt.wantDevcontainerScripts, gotDevcontainerScripts); diff != "" {
+				t.Errorf("ExtractAndInitializeDevcontainerScripts() gotDevcontainerScripts mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
+// textGrep returns matching lines from multiline string.
+func textGrep(want, got string) (filtered string) {
+	var lines []string
+	for _, line := range strings.Split(got, "\n") {
+		if strings.Contains(line, want) {
+			lines = append(lines, line)
+		}
+	}
+	return strings.Join(lines, "\n")
+}

agent/agentscripts/agentscripts.go

@@ -80,6 +80,21 @@ func New(opts Options) *Runner {
 
 type ScriptCompletedFunc func(context.Context, *proto.WorkspaceAgentScriptCompletedRequest) (*proto.WorkspaceAgentScriptCompletedResponse, error)
 
+type runnerScript struct {
+	runOnPostStart bool
+	codersdk.WorkspaceAgentScript
+}
+
+func toRunnerScript(scripts ...codersdk.WorkspaceAgentScript) []runnerScript {
+	var rs []runnerScript
+	for _, s := range scripts {
+		rs = append(rs, runnerScript{
+			WorkspaceAgentScript: s,
+		})
+	}
+	return rs
+}
+
 type Runner struct {
 	Options
 
@@ -90,7 +105,7 @@ type Runner struct {
 	closeMutex      sync.Mutex
 	cron            *cron.Cron
 	initialized     atomic.Bool
-	scripts         []codersdk.WorkspaceAgentScript
+	scripts         []runnerScript
 	dataDir         string
 	scriptCompleted ScriptCompletedFunc
 
@@ -119,16 +134,35 @@ func (r *Runner) RegisterMetrics(reg prometheus.Registerer) {
 	reg.MustRegister(r.scriptsExecuted)
 }
 
+// InitOption describes an option for the runner initialization.
+type InitOption func(*Runner)
+
+// WithPostStartScripts adds scripts that should be run after the workspace
+// start scripts but before the workspace is marked as started.
+func WithPostStartScripts(scripts ...codersdk.WorkspaceAgentScript) InitOption {
+	return func(r *Runner) {
+		for _, s := range scripts {
+			r.scripts = append(r.scripts, runnerScript{
+				runOnPostStart:       true,
+				WorkspaceAgentScript: s,
+			})
+		}
+	}
+}
+
 // Init initializes the runner with the provided scripts.
 // It also schedules any scripts that have a schedule.
 // This function must be called before Execute.
-func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript, scriptCompleted ScriptCompletedFunc) error {
+func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript, scriptCompleted ScriptCompletedFunc, opts ...InitOption) error {
 	if r.initialized.Load() {
 		return xerrors.New("init: already initialized")
 	}
 	r.initialized.Store(true)
-	r.scripts = scripts
+	r.scripts = toRunnerScript(scripts...)
 	r.scriptCompleted = scriptCompleted
+	for _, opt := range opts {
+		opt(r)
+	}
 	r.Logger.Info(r.cronCtx, "initializing agent scripts", slog.F("script_count", len(scripts)), slog.F("log_dir", r.LogDir))
 
 	err := r.Filesystem.MkdirAll(r.ScriptBinDir(), 0o700)
@@ -136,13 +170,13 @@ func (r *Runner) Init(scripts []codersdk.WorkspaceAgentScript, scriptCompleted S
 		return xerrors.Errorf("create script bin dir: %w", err)
 	}
 
-	for _, script := range scripts {
+	for _, script := range r.scripts {
 		if script.Cron == "" {
 			continue
 		}
 		script := script
 		_, err := r.cron.AddFunc(script.Cron, func() {
-			err := r.trackRun(r.cronCtx, script, ExecuteCronScripts)
+			err := r.trackRun(r.cronCtx, script.WorkspaceAgentScript, ExecuteCronScripts)
 			if err != nil {
 				r.Logger.Warn(context.Background(), "run agent script on schedule", slog.Error(err))
 			}
@@ -186,6 +220,7 @@ type ExecuteOption int
 const (
 	ExecuteAllScripts ExecuteOption = iota
 	ExecuteStartScripts
+	ExecutePostStartScripts
 	ExecuteStopScripts
 	ExecuteCronScripts
 )
@@ -196,6 +231,7 @@ func (r *Runner) Execute(ctx context.Context, option ExecuteOption) error {
 	for _, script := range r.scripts {
 		runScript := (option == ExecuteStartScripts && script.RunOnStart) ||
 			(option == ExecuteStopScripts && script.RunOnStop) ||
+			(option == ExecutePostStartScripts && script.runOnPostStart) ||
 			(option == ExecuteCronScripts && script.Cron != "") ||
 			option == ExecuteAllScripts
 
@@ -205,7 +241,7 @@ func (r *Runner) Execute(ctx context.Context, option ExecuteOption) error {
 
 		script := script
 		eg.Go(func() error {
-			err := r.trackRun(ctx, script, option)
+			err := r.trackRun(ctx, script.WorkspaceAgentScript, option)
 			if err != nil {
 				return xerrors.Errorf("run agent script %q: %w", script.LogSourceID, err)
 			}

agent/agentscripts/agentscripts_test.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"path/filepath"
 	"runtime"
+	"slices"
+	"sync"
 	"testing"
 	"time"
 
@@ -151,11 +153,160 @@ func TestCronClose(t *testing.T) {
 	require.NoError(t, runner.Close(), "close runner")
 }
 
+func TestExecuteOptions(t *testing.T) {
+	t.Parallel()
+
+	startScript := codersdk.WorkspaceAgentScript{
+		ID:          uuid.New(),
+		LogSourceID: uuid.New(),
+		Script:      "echo start",
+		RunOnStart:  true,
+	}
+	stopScript := codersdk.WorkspaceAgentScript{
+		ID:          uuid.New(),
+		LogSourceID: uuid.New(),
+		Script:      "echo stop",
+		RunOnStop:   true,
+	}
+	postStartScript := codersdk.WorkspaceAgentScript{
+		ID:          uuid.New(),
+		LogSourceID: uuid.New(),
+		Script:      "echo poststart",
+	}
+	regularScript := codersdk.WorkspaceAgentScript{
+		ID:          uuid.New(),
+		LogSourceID: uuid.New(),
+		Script:      "echo regular",
+	}
+
+	scripts := []codersdk.WorkspaceAgentScript{
+		startScript,
+		stopScript,
+		regularScript,
+	}
+	allScripts := append(slices.Clone(scripts), postStartScript)
+
+	scriptByID := func(t *testing.T, id uuid.UUID) codersdk.WorkspaceAgentScript {
+		for _, script := range allScripts {
+			if script.ID == id {
+				return script
+			}
+		}
+		t.Fatal("script not found")
+		return codersdk.WorkspaceAgentScript{}
+	}
+
+	wantOutput := map[uuid.UUID]string{
+		startScript.ID:     "start",
+		stopScript.ID:      "stop",
+		postStartScript.ID: "poststart",
+		regularScript.ID:   "regular",
+	}
+
+	testCases := []struct {
+		name    string
+		option  agentscripts.ExecuteOption
+		wantRun []uuid.UUID
+	}{
+		{
+			name:    "ExecuteAllScripts",
+			option:  agentscripts.ExecuteAllScripts,
+			wantRun: []uuid.UUID{startScript.ID, stopScript.ID, regularScript.ID, postStartScript.ID},
+		},
+		{
+			name:    "ExecuteStartScripts",
+			option:  agentscripts.ExecuteStartScripts,
+			wantRun: []uuid.UUID{startScript.ID},
+		},
+		{
+			name:    "ExecutePostStartScripts",
+			option:  agentscripts.ExecutePostStartScripts,
+			wantRun: []uuid.UUID{postStartScript.ID},
+		},
+		{
+			name:    "ExecuteStopScripts",
+			option:  agentscripts.ExecuteStopScripts,
+			wantRun: []uuid.UUID{stopScript.ID},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx := testutil.Context(t, testutil.WaitMedium)
+			executedScripts := make(map[uuid.UUID]bool)
+			fLogger := &executeOptionTestLogger{
+				tb:              t,
+				executedScripts: executedScripts,
+				wantOutput:      wantOutput,
+			}
+
+			runner := setup(t, func(uuid.UUID) agentscripts.ScriptLogger {
+				return fLogger
+			})
+			defer runner.Close()
+
+			aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
+			err := runner.Init(
+				scripts,
+				aAPI.ScriptCompleted,
+				agentscripts.WithPostStartScripts(postStartScript),
+			)
+			require.NoError(t, err)
+
+			err = runner.Execute(ctx, tc.option)
+			require.NoError(t, err)
+
+			gotRun := map[uuid.UUID]bool{}
+			for _, id := range tc.wantRun {
+				gotRun[id] = true
+				require.True(t, executedScripts[id],
+					"script %s should have run when using filter %s", scriptByID(t, id).Script, tc.name)
+			}
+
+			for _, script := range allScripts {
+				if _, ok := gotRun[script.ID]; ok {
+					continue
+				}
+				require.False(t, executedScripts[script.ID],
+					"script %s should not have run when using filter %s", script.Script, tc.name)
+			}
+		})
+	}
+}
+
+type executeOptionTestLogger struct {
+	tb              testing.TB
+	executedScripts map[uuid.UUID]bool
+	wantOutput      map[uuid.UUID]string
+	mu              sync.Mutex
+}
+
+func (l *executeOptionTestLogger) Send(_ context.Context, logs ...agentsdk.Log) error {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	for _, log := range logs {
+		l.tb.Log(log.Output)
+		for id, output := range l.wantOutput {
+			if log.Output == output {
+				l.executedScripts[id] = true
+				break
+			}
+		}
+	}
+	return nil
+}
+
+func (*executeOptionTestLogger) Flush(context.Context) error {
+	return nil
+}
+
 func setup(t *testing.T, getScriptLogger func(logSourceID uuid.UUID) agentscripts.ScriptLogger) *agentscripts.Runner {
 	t.Helper()
 	if getScriptLogger == nil {
 		// noop
-		getScriptLogger = func(uuid uuid.UUID) agentscripts.ScriptLogger {
+		getScriptLogger = func(uuid.UUID) agentscripts.ScriptLogger {
 			return noopScriptLogger{}
 		}
 	}

coderd/provisionerdserver/provisionerdserver.go

@@ -2081,6 +2081,55 @@ func InsertWorkspaceResource(ctx context.Context, db database.Store, jobID uuid.
 			scriptRunOnStop = append(scriptRunOnStop, script.RunOnStop)
 		}
 
+		// Dev Containers require a script and log/source, so we do this before
+		// the logs insert below.
+		if devcontainers := prAgent.GetDevcontainers(); len(devcontainers) > 0 {
+			var (
+				devcontainerIDs              = make([]uuid.UUID, 0, len(devcontainers))
+				devcontainerNames            = make([]string, 0, len(devcontainers))
+				devcontainerWorkspaceFolders = make([]string, 0, len(devcontainers))
+				devcontainerConfigPaths      = make([]string, 0, len(devcontainers))
+			)
+			for _, dc := range devcontainers {
+				id := uuid.New()
+				devcontainerIDs = append(devcontainerIDs, id)
+				devcontainerNames = append(devcontainerNames, dc.Name)
+				devcontainerWorkspaceFolders = append(devcontainerWorkspaceFolders, dc.WorkspaceFolder)
+				devcontainerConfigPaths = append(devcontainerConfigPaths, dc.ConfigPath)
+
+				// Add a log source and script for each devcontainer so we can
+				// track logs and timings for each devcontainer.
+				displayName := fmt.Sprintf("Dev Container (%s)", dc.Name)
+				logSourceIDs = append(logSourceIDs, uuid.New())
+				logSourceDisplayNames = append(logSourceDisplayNames, displayName)
+				logSourceIcons = append(logSourceIcons, "/emojis/1f4e6.png") // Emoji package. Or perhaps /icon/container.svg?
+				scriptIDs = append(scriptIDs, id)                            // Re-use the devcontainer ID as the script ID for identification.
+				scriptDisplayName = append(scriptDisplayName, displayName)
+				scriptLogPaths = append(scriptLogPaths, "")
+				scriptSources = append(scriptSources, `echo "WARNING: Dev Containers are early access. If you're seeing this message then Dev Containers haven't been enabled for your workspace yet. To enable, the agent needs to run with the environment variable CODER_AGENT_DEVCONTAINERS_ENABLE=true set."`)
+				scriptCron = append(scriptCron, "")
+				scriptTimeout = append(scriptTimeout, 0)
+				scriptStartBlocksLogin = append(scriptStartBlocksLogin, false)
+				// Run on start to surface the warning message in case the
+				// terraform resource is used, but the experiment hasn't
+				// been enabled.
+				scriptRunOnStart = append(scriptRunOnStart, true)
+				scriptRunOnStop = append(scriptRunOnStop, false)
+			}
+
+			_, err = db.InsertWorkspaceAgentDevcontainers(ctx, database.InsertWorkspaceAgentDevcontainersParams{
+				WorkspaceAgentID: agentID,
+				CreatedAt:        dbtime.Now(),
+				ID:               devcontainerIDs,
+				Name:             devcontainerNames,
+				WorkspaceFolder:  devcontainerWorkspaceFolders,
+				ConfigPath:       devcontainerConfigPaths,
+			})
+			if err != nil {
+				return xerrors.Errorf("insert agent devcontainer: %w", err)
+			}
+		}
+
 		_, err = db.InsertWorkspaceAgentLogSources(ctx, database.InsertWorkspaceAgentLogSourcesParams{
 			WorkspaceAgentID: agentID,
 			ID:               logSourceIDs,
@@ -2110,33 +2159,6 @@ func InsertWorkspaceResource(ctx context.Context, db database.Store, jobID uuid.
 			return xerrors.Errorf("insert agent scripts: %w", err)
 		}
 
-		if devcontainers := prAgent.GetDevcontainers(); len(devcontainers) > 0 {
-			var (
-				devcontainerIDs              = make([]uuid.UUID, 0, len(devcontainers))
-				devcontainerNames            = make([]string, 0, len(devcontainers))
-				devcontainerWorkspaceFolders = make([]string, 0, len(devcontainers))
-				devcontainerConfigPaths      = make([]string, 0, len(devcontainers))
-			)
-			for _, dc := range devcontainers {
-				devcontainerIDs = append(devcontainerIDs, uuid.New())
-				devcontainerNames = append(devcontainerNames, dc.Name)
-				devcontainerWorkspaceFolders = append(devcontainerWorkspaceFolders, dc.WorkspaceFolder)
-				devcontainerConfigPaths = append(devcontainerConfigPaths, dc.ConfigPath)
-			}
-
-			_, err = db.InsertWorkspaceAgentDevcontainers(ctx, database.InsertWorkspaceAgentDevcontainersParams{
-				WorkspaceAgentID: agentID,
-				CreatedAt:        dbtime.Now(),
-				ID:               devcontainerIDs,
-				Name:             devcontainerNames,
-				WorkspaceFolder:  devcontainerWorkspaceFolders,
-				ConfigPath:       devcontainerConfigPaths,
-			})
-			if err != nil {
-				return xerrors.Errorf("insert agent devcontainer: %w", err)
-			}
-		}
-
 		for _, app := range prAgent.Apps {
 			// Similar logic is duplicated in terraform/resources.go.
 			slug := app.Slug