feat: track resource replacements when claiming a prebuilt workspace (#17571)

Closes https://github.com/coder/internal/issues/369 We can't know whether a replacement (i.e. drift of terraform state leading to a resource needing to be deleted/recreated) will take place apriori; we can only detect it at `plan` time, because the provider decides whether a resource must be replaced and it cannot be inferred through static analysis of the template. **This is likely to be the most common gotcha with using prebuilds, since it requires a slight template modification to use prebuilds effectively**, so let's head this off before it's an issue for customers. Drift details will now be logged in the workspace build logs: ![image](https://github.com/user-attachments/assets/da1988b6-2cbe-4a79-a3c5-ea29891f3d6f) Plus a notification will be sent to template admins when this situation arises: ![image](https://github.com/user-attachments/assets/39d555b1-a262-4a3e-b529-03b9f23bf66a) A new metric - `coderd_prebuilt_workspaces_resource_replacements_total` - will also increment each time a workspace encounters replacements. We only track _that_ a resource replacement occurred, not how many. Just one is enough to ruin a prebuild, but we can't know apriori which replacement would cause this. For example, say we have 2 replacements: a `docker_container` and a `null_resource`; we don't know which one might cause an issue (or indeed if either would), so we just track the replacement. --------- Signed-off-by: Danny Kopping <dannykopping@gmail.com>
2025-07-15 22:20:27 +00:00 · 2025-05-14 14:52:22 +02:00
parent e75d1c1ce5
commit 6e967780c9
33 changed files with 2048 additions and 969 deletions
--- a/coderd/coderd.go
+++ b/coderd/coderd.go
@ -40,10 +40,11 @@ import (
 	"tailscale.com/util/singleflight"

 	"cdr.dev/slog"
-	"github.com/coder/coder/v2/codersdk/drpcsdk"
 	"github.com/coder/quartz"
 	"github.com/coder/serpent"

+	"github.com/coder/coder/v2/codersdk/drpcsdk"
+
 	"github.com/coder/coder/v2/coderd/ai"
 	"github.com/coder/coder/v2/coderd/cryptokeys"
 	"github.com/coder/coder/v2/coderd/entitlements"
@ -1795,6 +1796,7 @@ func (api *API) CreateInMemoryTaggedProvisionerDaemon(dialCtx context.Context, n
 			Clock:               api.Clock,
 		},
 		api.NotificationsEnqueuer,
+		&api.PrebuildsReconciler,
 	)
 	if err != nil {
 		return nil, err
--- a/coderd/database/migrations/000324_resource_replacements_notification.down.sql
+++ b/coderd/database/migrations/000324_resource_replacements_notification.down.sql
@ -0,0 +1 @@
+DELETE FROM notification_templates WHERE id = '89d9745a-816e-4695-a17f-3d0a229e2b8d';
--- a/coderd/database/migrations/000324_resource_replacements_notification.up.sql
+++ b/coderd/database/migrations/000324_resource_replacements_notification.up.sql
@ -0,0 +1,34 @@
+INSERT INTO notification_templates
+	(id, name, title_template, body_template, "group", actions)
+VALUES ('89d9745a-816e-4695-a17f-3d0a229e2b8d',
+		'Prebuilt Workspace Resource Replaced',
+		E'There might be a problem with a recently claimed prebuilt workspace',
+		$$
+Workspace **{{.Labels.workspace}}** was claimed from a prebuilt workspace by **{{.Labels.claimant}}**.
+
+During the claim, Terraform destroyed and recreated the following resources
+because one or more immutable attributes changed:
+
+{{range $resource, $paths := .Data.replacements -}}
+- _{{ $resource }}_  was replaced due to changes to _{{ $paths }}_
+{{end}}
+
+When Terraform must change an immutable attribute, it replaces the entire resource.
+If you’re using prebuilds to speed up provisioning, unexpected replacements will slow down
+workspace startup—even when claiming a prebuilt environment.
+
+For tips on preventing replacements and improving claim performance, see [this guide](https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#preventing-resource-replacement).
+
+NOTE: this prebuilt workspace used the **{{.Labels.preset}}** preset.
+$$,
+		'Template Events',
+		'[
+		{
+			"label": "View workspace build",
+			"url": "{{base_url}}/@{{.Labels.claimant}}/{{.Labels.workspace}}/builds/{{.Labels.workspace_build_num}}"
+		},
+		{
+			"label": "View template version",
+			"url": "{{base_url}}/templates/{{.Labels.org}}/{{.Labels.template}}/versions/{{.Labels.template_version}}"
+		}
+	]'::jsonb);
--- a/coderd/notifications/events.go
+++ b/coderd/notifications/events.go
@ -39,6 +39,7 @@ var (
 	TemplateTemplateDeprecated = uuid.MustParse("f40fae84-55a2-42cd-99fa-b41c1ca64894")

 	TemplateWorkspaceBuildsFailedReport = uuid.MustParse("34a20db2-e9cc-4a93-b0e4-8569699d7a00")
+	TemplateWorkspaceResourceReplaced   = uuid.MustParse("89d9745a-816e-4695-a17f-3d0a229e2b8d")
 )

 // Notification-related events.
--- a/coderd/notifications/notifications_test.go
+++ b/coderd/notifications/notifications_test.go
@ -35,6 +35,9 @@ import (
 	"golang.org/x/xerrors"

 	"cdr.dev/slog"
+	"github.com/coder/quartz"
+	"github.com/coder/serpent"
+
 	"github.com/coder/coder/v2/coderd/coderdtest"
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbauthz"
@ -48,8 +51,6 @@ import (
 	"github.com/coder/coder/v2/coderd/util/syncmap"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/testutil"
-	"github.com/coder/quartz"
-	"github.com/coder/serpent"
 )

 // updateGoldenFiles is a flag that can be set to update golden files.
@ -1226,6 +1227,29 @@ func TestNotificationTemplates_Golden(t *testing.T) {
 				Labels:       map[string]string{},
 			},
 		},
+		{
+			name: "TemplateWorkspaceResourceReplaced",
+			id:   notifications.TemplateWorkspaceResourceReplaced,
+			payload: types.MessagePayload{
+				UserName:     "Bobby",
+				UserEmail:    "bobby@coder.com",
+				UserUsername: "bobby",
+				Labels: map[string]string{
+					"org":                 "cern",
+					"workspace":           "my-workspace",
+					"workspace_build_num": "2",
+					"template":            "docker",
+					"template_version":    "angry_torvalds",
+					"preset":              "particle-accelerator",
+					"claimant":            "prebuilds-claimer",
+				},
+				Data: map[string]any{
+					"replacements": map[string]string{
+						"docker_container[0]": "env, hostname",
+					},
+				},
+			},
+		},
 	}

 	// We must have a test case for every notification_template. This is enforced below:
--- a/coderd/notifications/notificationstest/fake_enqueuer.go
+++ b/coderd/notifications/notificationstest/fake_enqueuer.go
@ -9,6 +9,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"

 	"github.com/coder/coder/v2/coderd/database/dbauthz"
+	"github.com/coder/coder/v2/coderd/notifications"
 	"github.com/coder/coder/v2/coderd/rbac"
 	"github.com/coder/coder/v2/coderd/rbac/policy"
 )
@ -19,6 +20,12 @@ type FakeEnqueuer struct {
 	sent       []*FakeNotification
 }

+var _ notifications.Enqueuer = &FakeEnqueuer{}
+
+func NewFakeEnqueuer() *FakeEnqueuer {
+	return &FakeEnqueuer{}
+}
+
 type FakeNotification struct {
 	UserID, TemplateID uuid.UUID
 	Labels             map[string]string
--- a/coderd/notifications/testdata/rendered-templates/smtp/TemplateWorkspaceResourceReplaced.html.golden
+++ b/coderd/notifications/testdata/rendered-templates/smtp/TemplateWorkspaceResourceReplaced.html.golden
@ -0,0 +1,131 @@
+From: system@coder.com
+To: bobby@coder.com
+Subject: There might be a problem with a recently claimed prebuilt workspace
+Message-Id: 02ee4935-73be-4fa1-a290-ff9999026b13@blush-whale-48
+Date: Fri, 11 Oct 2024 09:03:06 +0000
+Content-Type: multipart/alternative;  boundary=bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
+MIME-Version: 1.0
+
+--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset=UTF-8
+
+Hi Bobby,
+
+Workspace my-workspace was claimed from a prebuilt workspace by prebuilds-c=
+laimer.
+
+During the claim, Terraform destroyed and recreated the following resources
+because one or more immutable attributes changed:
+
+docker_container[0]  was replaced due to changes to env, hostname
+
+When Terraform must change an immutable attribute, it replaces the entire r=
+esource.
+If you=E2=80=99re using prebuilds to speed up provisioning, unexpected repl=
+acements will slow down
+workspace startup=E2=80=94even when claiming a prebuilt environment.
+
+For tips on preventing replacements and improving claim performance, see th=
+is guide (https://coder.com/docs/admin/templates/extending-templates/prebui=
+lt-workspaces#preventing-resource-replacement).
+
+NOTE: this prebuilt workspace used the particle-accelerator preset.
+
+
+View workspace build: http://test.com/@prebuilds-claimer/my-workspace/build=
+s/2
+
+View template version: http://test.com/templates/cern/docker/versions/angry=
+_torvalds
+
+--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/html; charset=UTF-8
+
+<!doctype html>
+<html lang=3D"en">
+  <head>
+    <meta charset=3D"UTF-8" />
+    <meta name=3D"viewport" content=3D"width=3Ddevice-width, initial-scale=
+=3D1.0" />
+    <title>There might be a problem with a recently claimed prebuilt worksp=
+ace</title>
+  </head>
+  <body style=3D"margin: 0; padding: 0; font-family: -apple-system, system-=
+ui, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 'Ubuntu', 'Cantarel=
+l', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', sans-serif; color: #020617=
+; background: #f8fafc;">
+    <div style=3D"max-width: 600px; margin: 20px auto; padding: 60px; borde=
+r: 1px solid #e2e8f0; border-radius: 8px; background-color: #fff; text-alig=
+n: left; font-size: 14px; line-height: 1.5;">
+      <div style=3D"text-align: center;">
+        <img src=3D"https://coder.com/coder-logo-horizontal.png" alt=3D"Cod=
+er Logo" style=3D"height: 40px;" />
+      </div>
+      <h1 style=3D"text-align: center; font-size: 24px; font-weight: 400; m=
+argin: 8px 0 32px; line-height: 1.5;">
+        There might be a problem with a recently claimed prebuilt workspace
+      </h1>
+      <div style=3D"line-height: 1.5;">
+        <p>Hi Bobby,</p>
+        <p>Workspace <strong>my-workspace</strong> was claimed from a prebu=
+ilt workspace by <strong>prebuilds-claimer</strong>.</p>
+
+<p>During the claim, Terraform destroyed and recreated the following resour=
+ces<br>
+because one or more immutable attributes changed:</p>
+
+<ul>
+<li>_docker<em>container[0]</em>  was replaced due to changes to <em>env, h=
+ostname</em><br>
+</li>
+</ul>
+
+<p>When Terraform must change an immutable attribute, it replaces the entir=
+e resource.<br>
+If you=E2=80=99re using prebuilds to speed up provisioning, unexpected repl=
+acements will slow down<br>
+workspace startup=E2=80=94even when claiming a prebuilt environment.</p>
+
+<p>For tips on preventing replacements and improving claim performance, see=
+ <a href=3D"https://coder.com/docs/admin/templates/extending-templates/preb=
+uilt-workspaces#preventing-resource-replacement">this guide</a>.</p>
+
+<p>NOTE: this prebuilt workspace used the <strong>particle-accelerator</str=
+ong> preset.</p>
+      </div>
+      <div style=3D"text-align: center; margin-top: 32px;">
+       =20
+        <a href=3D"http://test.com/@prebuilds-claimer/my-workspace/builds/2=
+" style=3D"display: inline-block; padding: 13px 24px; background-color: #02=
+0617; color: #f8fafc; text-decoration: none; border-radius: 8px; margin: 0 =
+4px;">
+          View workspace build
+        </a>
+       =20
+        <a href=3D"http://test.com/templates/cern/docker/versions/angry_tor=
+valds" style=3D"display: inline-block; padding: 13px 24px; background-color=
+: #020617; color: #f8fafc; text-decoration: none; border-radius: 8px; margi=
+n: 0 4px;">
+          View template version
+        </a>
+       =20
+      </div>
+      <div style=3D"border-top: 1px solid #e2e8f0; color: #475569; font-siz=
+e: 12px; margin-top: 64px; padding-top: 24px; line-height: 1.6;">
+        <p>&copy;&nbsp;2024&nbsp;Coder. All rights reserved&nbsp;-&nbsp;<a =
+href=3D"http://test.com" style=3D"color: #2563eb; text-decoration: none;">h=
+ttp://test.com</a></p>
+        <p><a href=3D"http://test.com/settings/notifications" style=3D"colo=
+r: #2563eb; text-decoration: none;">Click here to manage your notification =
+settings</a></p>
+        <p><a href=3D"http://test.com/settings/notifications?disabled=3D89d=
+9745a-816e-4695-a17f-3d0a229e2b8d" style=3D"color: #2563eb; text-decoration=
+: none;">Stop receiving emails like this</a></p>
+      </div>
+    </div>
+  </body>
+</html>
+
+--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4--
--- a/coderd/notifications/testdata/rendered-templates/webhook/TemplateWorkspaceResourceReplaced.json.golden
+++ b/coderd/notifications/testdata/rendered-templates/webhook/TemplateWorkspaceResourceReplaced.json.golden
@ -0,0 +1,42 @@
+{
+  "_version": "1.1",
+  "msg_id": "00000000-0000-0000-0000-000000000000",
+  "payload": {
+    "_version": "1.2",
+    "notification_name": "Prebuilt Workspace Resource Replaced",
+    "notification_template_id": "00000000-0000-0000-0000-000000000000",
+    "user_id": "00000000-0000-0000-0000-000000000000",
+    "user_email": "bobby@coder.com",
+    "user_name": "Bobby",
+    "user_username": "bobby",
+    "actions": [
+      {
+        "label": "View workspace build",
+        "url": "http://test.com/@prebuilds-claimer/my-workspace/builds/2"
+      },
+      {
+        "label": "View template version",
+        "url": "http://test.com/templates/cern/docker/versions/angry_torvalds"
+      }
+    ],
+    "labels": {
+      "claimant": "prebuilds-claimer",
+      "org": "cern",
+      "preset": "particle-accelerator",
+      "template": "docker",
+      "template_version": "angry_torvalds",
+      "workspace": "my-workspace",
+      "workspace_build_num": "2"
+    },
+    "data": {
+      "replacements": {
+        "docker_container[0]": "env, hostname"
+      }
+    },
+    "targets": null
+  },
+  "title": "There might be a problem with a recently claimed prebuilt workspace",
+  "title_markdown": "There might be a problem with a recently claimed prebuilt workspace",
+  "body": "Workspace my-workspace was claimed from a prebuilt workspace by prebuilds-claimer.\n\nDuring the claim, Terraform destroyed and recreated the following resources\nbecause one or more immutable attributes changed:\n\ndocker_container[0]  was replaced due to changes to env, hostname\n\nWhen Terraform must change an immutable attribute, it replaces the entire resource.\nIf you’re using prebuilds to speed up provisioning, unexpected replacements will slow down\nworkspace startup—even when claiming a prebuilt environment.\n\nFor tips on preventing replacements and improving claim performance, see this guide (https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#preventing-resource-replacement).\n\nNOTE: this prebuilt workspace used the particle-accelerator preset.",
+  "body_markdown": "\nWorkspace **my-workspace** was claimed from a prebuilt workspace by **prebuilds-claimer**.\n\nDuring the claim, Terraform destroyed and recreated the following resources\nbecause one or more immutable attributes changed:\n\n- _docker_container[0]_  was replaced due to changes to _env, hostname_\n\n\nWhen Terraform must change an immutable attribute, it replaces the entire resource.\nIf you’re using prebuilds to speed up provisioning, unexpected replacements will slow down\nworkspace startup—even when claiming a prebuilt environment.\n\nFor tips on preventing replacements and improving claim performance, see [this guide](https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#preventing-resource-replacement).\n\nNOTE: this prebuilt workspace used the **particle-accelerator** preset.\n"
+}
--- a/coderd/prebuilds/api.go
+++ b/coderd/prebuilds/api.go
@ -7,6 +7,7 @@ import (
 	"golang.org/x/xerrors"

 	"github.com/coder/coder/v2/coderd/database"
+	sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
 )

 var (
@ -27,6 +28,11 @@ type ReconciliationOrchestrator interface {
 	// Stop gracefully shuts down the orchestrator with the given cause.
 	// The cause is used for logging and error reporting.
 	Stop(ctx context.Context, cause error)
+
+	// TrackResourceReplacement handles a pathological situation whereby a terraform resource is replaced due to drift,
+	// which can obviate the whole point of pre-provisioning a prebuilt workspace.
+	// See more detail at https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#preventing-resource-replacement.
+	TrackResourceReplacement(ctx context.Context, workspaceID, buildID uuid.UUID, replacements []*sdkproto.ResourceReplacement)
 }

 type Reconciler interface {
--- a/coderd/prebuilds/noop.go
+++ b/coderd/prebuilds/noop.go
@ -6,12 +6,15 @@ import (
 	"github.com/google/uuid"

 	"github.com/coder/coder/v2/coderd/database"
+	sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
 )

 type NoopReconciler struct{}

-func (NoopReconciler) Run(context.Context)                {}
-func (NoopReconciler) Stop(context.Context, error)        {}
+func (NoopReconciler) Run(context.Context)         {}
+func (NoopReconciler) Stop(context.Context, error) {}
+func (NoopReconciler) TrackResourceReplacement(context.Context, uuid.UUID, uuid.UUID, []*sdkproto.ResourceReplacement) {
+}
 func (NoopReconciler) ReconcileAll(context.Context) error { return nil }
 func (NoopReconciler) SnapshotState(context.Context, database.Store) (*GlobalSnapshot, error) {
 	return &GlobalSnapshot{}, nil
--- a/coderd/provisionerdserver/provisionerdserver.go
+++ b/coderd/provisionerdserver/provisionerdserver.go
@ -28,6 +28,7 @@ import (
 	protobuf "google.golang.org/protobuf/proto"

 	"cdr.dev/slog"
+
 	"github.com/coder/coder/v2/codersdk/drpcsdk"

 	"github.com/coder/quartz"
@ -116,6 +117,7 @@ type server struct {
 	UserQuietHoursScheduleStore *atomic.Pointer[schedule.UserQuietHoursScheduleStore]
 	DeploymentValues            *codersdk.DeploymentValues
 	NotificationsEnqueuer       notifications.Enqueuer
+	PrebuildsOrchestrator       *atomic.Pointer[prebuilds.ReconciliationOrchestrator]

 	OIDCConfig promoauth.OAuth2Config

@ -151,8 +153,7 @@ func (t Tags) Valid() error {
 	return nil
 }

-func NewServer(
-	lifecycleCtx context.Context,
+func NewServer(lifecycleCtx context.Context,
 	accessURL *url.URL,
 	id uuid.UUID,
 	organizationID uuid.UUID,
@ -171,6 +172,7 @@ func NewServer(
 	deploymentValues *codersdk.DeploymentValues,
 	options Options,
 	enqueuer notifications.Enqueuer,
+	prebuildsOrchestrator *atomic.Pointer[prebuilds.ReconciliationOrchestrator],
 ) (proto.DRPCProvisionerDaemonServer, error) {
 	// Fail-fast if pointers are nil
 	if lifecycleCtx == nil {
@ -235,6 +237,7 @@ func NewServer(
 		acquireJobLongPollDur:       options.AcquireJobLongPollDur,
 		heartbeatInterval:           options.HeartbeatInterval,
 		heartbeatFn:                 options.HeartbeatFn,
+		PrebuildsOrchestrator:       prebuildsOrchestrator,
 	}

 	if s.heartbeatFn == nil {
@ -1828,6 +1831,15 @@ func (s *server) CompleteJob(ctx context.Context, completed *proto.CompletedJob)
 			})
 		}

+		if s.PrebuildsOrchestrator != nil {
+			// Track resource replacements, if there are any.
+			orchestrator := s.PrebuildsOrchestrator.Load()
+			if resourceReplacements := completed.GetWorkspaceBuild().GetResourceReplacements(); orchestrator != nil && len(resourceReplacements) > 0 {
+				// Fire and forget. Bind to the lifecycle of the server so shutdowns are handled gracefully.
+				go (*orchestrator).TrackResourceReplacement(s.lifecycleCtx, workspace.ID, workspaceBuild.ID, resourceReplacements)
+			}
+		}
+
 		msg, err := json.Marshal(wspubsub.WorkspaceEvent{
 			Kind:        wspubsub.WorkspaceEventKindStateChange,
 			WorkspaceID: workspace.ID,
--- a/coderd/provisionerdserver/provisionerdserver_test.go
+++ b/coderd/provisionerdserver/provisionerdserver_test.go
@ -26,11 +26,6 @@ import (
 	"github.com/coder/quartz"
 	"github.com/coder/serpent"

-	"github.com/coder/coder/v2/coderd/prebuilds"
-	"github.com/coder/coder/v2/coderd/provisionerdserver"
-	"github.com/coder/coder/v2/coderd/rbac"
-	"github.com/coder/coder/v2/codersdk/agentsdk"
-
 	"github.com/coder/coder/v2/buildinfo"
 	"github.com/coder/coder/v2/coderd/audit"
 	"github.com/coder/coder/v2/coderd/database"
@ -42,11 +37,15 @@ import (
 	"github.com/coder/coder/v2/coderd/externalauth"
 	"github.com/coder/coder/v2/coderd/notifications"
 	"github.com/coder/coder/v2/coderd/notifications/notificationstest"
+	agplprebuilds "github.com/coder/coder/v2/coderd/prebuilds"
+	"github.com/coder/coder/v2/coderd/provisionerdserver"
+	"github.com/coder/coder/v2/coderd/rbac"
 	"github.com/coder/coder/v2/coderd/schedule"
 	"github.com/coder/coder/v2/coderd/schedule/cron"
 	"github.com/coder/coder/v2/coderd/telemetry"
 	"github.com/coder/coder/v2/coderd/wspubsub"
 	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
 	"github.com/coder/coder/v2/provisionerd/proto"
 	"github.com/coder/coder/v2/provisionersdk"
 	sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
@ -1889,7 +1888,7 @@ func TestCompleteJob(t *testing.T) {

 				// GIVEN something is listening to process workspace reinitialization:
 				reinitChan := make(chan agentsdk.ReinitializationEvent, 1) // Buffered to simplify test structure
-				cancel, err := prebuilds.NewPubsubWorkspaceClaimListener(ps, testutil.Logger(t)).ListenForWorkspaceClaims(ctx, workspace.ID, reinitChan)
+				cancel, err := agplprebuilds.NewPubsubWorkspaceClaimListener(ps, testutil.Logger(t)).ListenForWorkspaceClaims(ctx, workspace.ID, reinitChan)
 				require.NoError(t, err)
 				defer cancel()

@ -1917,6 +1916,106 @@ func TestCompleteJob(t *testing.T) {
 			})
 		}
 	})
+
+	t.Run("PrebuiltWorkspaceClaimWithResourceReplacements", func(t *testing.T) {
+		t.Parallel()
+
+		ctx := testutil.Context(t, testutil.WaitLong)
+
+		// Given: a mock prebuild orchestrator which stores calls to TrackResourceReplacement.
+		done := make(chan struct{})
+		orchestrator := &mockPrebuildsOrchestrator{
+			ReconciliationOrchestrator: agplprebuilds.DefaultReconciler,
+			done:                       done,
+		}
+		srv, db, ps, pd := setup(t, false, &overrides{
+			prebuildsOrchestrator: orchestrator,
+		})
+
+		// Given: a workspace build which simulates claiming a prebuild.
+		user := dbgen.User(t, db, database.User{})
+		template := dbgen.Template(t, db, database.Template{
+			Name:           "template",
+			Provisioner:    database.ProvisionerTypeEcho,
+			OrganizationID: pd.OrganizationID,
+		})
+		file := dbgen.File(t, db, database.File{CreatedBy: user.ID})
+		workspaceTable := dbgen.Workspace(t, db, database.WorkspaceTable{
+			TemplateID:     template.ID,
+			OwnerID:        user.ID,
+			OrganizationID: pd.OrganizationID,
+		})
+		version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
+			OrganizationID: pd.OrganizationID,
+			TemplateID: uuid.NullUUID{
+				UUID:  template.ID,
+				Valid: true,
+			},
+			JobID: uuid.New(),
+		})
+		build := dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
+			WorkspaceID:       workspaceTable.ID,
+			InitiatorID:       user.ID,
+			TemplateVersionID: version.ID,
+			Transition:        database.WorkspaceTransitionStart,
+			Reason:            database.BuildReasonInitiator,
+		})
+		job := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{
+			FileID:      file.ID,
+			InitiatorID: user.ID,
+			Type:        database.ProvisionerJobTypeWorkspaceBuild,
+			Input: must(json.Marshal(provisionerdserver.WorkspaceProvisionJob{
+				WorkspaceBuildID:            build.ID,
+				PrebuiltWorkspaceBuildStage: sdkproto.PrebuiltWorkspaceBuildStage_CLAIM,
+			})),
+			OrganizationID: pd.OrganizationID,
+		})
+		_, err := db.AcquireProvisionerJob(ctx, database.AcquireProvisionerJobParams{
+			OrganizationID: pd.OrganizationID,
+			WorkerID: uuid.NullUUID{
+				UUID:  pd.ID,
+				Valid: true,
+			},
+			Types: []database.ProvisionerType{database.ProvisionerTypeEcho},
+		})
+		require.NoError(t, err)
+
+		// When: a replacement is encountered.
+		replacements := []*sdkproto.ResourceReplacement{
+			{
+				Resource: "docker_container[0]",
+				Paths:    []string{"env"},
+			},
+		}
+
+		// Then: CompleteJob makes a call to TrackResourceReplacement.
+		_, err = srv.CompleteJob(ctx, &proto.CompletedJob{
+			JobId: job.ID.String(),
+			Type: &proto.CompletedJob_WorkspaceBuild_{
+				WorkspaceBuild: &proto.CompletedJob_WorkspaceBuild{
+					State:                []byte{},
+					ResourceReplacements: replacements,
+				},
+			},
+		})
+		require.NoError(t, err)
+
+		// Then: the replacements are as we expected.
+		testutil.RequireReceive(ctx, t, done)
+		require.Equal(t, replacements, orchestrator.replacements)
+	})
+}
+
+type mockPrebuildsOrchestrator struct {
+	agplprebuilds.ReconciliationOrchestrator
+
+	replacements []*sdkproto.ResourceReplacement
+	done         chan struct{}
+}
+
+func (m *mockPrebuildsOrchestrator) TrackResourceReplacement(_ context.Context, _, _ uuid.UUID, replacements []*sdkproto.ResourceReplacement) {
+	m.replacements = replacements
+	m.done <- struct{}{}
 }

 func TestInsertWorkspacePresetsAndParameters(t *testing.T) {
@ -2803,6 +2902,7 @@ type overrides struct {
 	heartbeatInterval           time.Duration
 	auditor                     audit.Auditor
 	notificationEnqueuer        notifications.Enqueuer
+	prebuildsOrchestrator       agplprebuilds.ReconciliationOrchestrator
 }

 func setup(t *testing.T, ignoreLogErrors bool, ov *overrides) (proto.DRPCProvisionerDaemonServer, database.Store, pubsub.Pubsub, database.ProvisionerDaemon) {
@ -2884,6 +2984,13 @@ func setup(t *testing.T, ignoreLogErrors bool, ov *overrides) (proto.DRPCProvisi
 	})
 	require.NoError(t, err)

+	prebuildsOrchestrator := ov.prebuildsOrchestrator
+	if prebuildsOrchestrator == nil {
+		prebuildsOrchestrator = agplprebuilds.DefaultReconciler
+	}
+	var op atomic.Pointer[agplprebuilds.ReconciliationOrchestrator]
+	op.Store(&prebuildsOrchestrator)
+
 	srv, err := provisionerdserver.NewServer(
 		ov.ctx,
 		&url.URL{},
@ -2911,6 +3018,7 @@ func setup(t *testing.T, ignoreLogErrors bool, ov *overrides) (proto.DRPCProvisi
 			HeartbeatFn:           ov.heartbeatFn,
 		},
 		notifEnq,
+		&op,
 	)
 	require.NoError(t, err)
 	return srv, db, ps, daemon
				`@ -0,0 +1 @@`
				`DELETE FROM notification_templates WHERE id = '89d9745a-816e-4695-a17f-3d0a229e2b8d';`