feat: implement expiration policy logic for prebuilds (#17996)

## Summary 

This PR introduces support for expiration policies in prebuilds. The TTL
(time-to-live) is retrieved from the Terraform configuration
([terraform-provider-coder
PR](https://github.com/coder/terraform-provider-coder/pull/404)):
```
prebuilds = {
	  instances = 2
	  expiration_policy {
		  ttl = 86400
	  }
  }
```
**Note**: Since there is no need for precise TTL enforcement down to the
second, in this implementation expired prebuilds are handled in a single
reconciliation cycle: they are deleted, and new instances are created
only if needed to match the desired count.

## Changes

* The outcome of a reconciliation cycle is now expressed as a slice of
reconciliation actions, instead of a single aggregated action.
* Adjusted reconciliation logic to delete expired prebuilds and
guarantee that the number of desired instances is correct.
* Updated relevant data structures and methods to support expiration
policies parameters.
* Added documentation to `Prebuilt workspaces` page
* Update `terraform-provider-coder` to version 2.5.0:
https://github.com/coder/terraform-provider-coder/releases/tag/v2.5.0

Depends on: https://github.com/coder/terraform-provider-coder/pull/404
Fixes: https://github.com/coder/coder/issues/17916

coderd/database/queries.sql.go

@@ -6511,6 +6511,7 @@ SELECT
 		tvp.id,
 		tvp.name,
 		tvp.desired_instances       AS desired_instances,
+		tvp.invalidate_after_secs 	AS ttl,
 		tvp.prebuild_status,
 		t.deleted,
 		t.deprecated != ''          AS deprecated
@@ -6534,6 +6535,7 @@ type GetTemplatePresetsWithPrebuildsRow struct {
 	ID                  uuid.UUID      `db:"id" json:"id"`
 	Name                string         `db:"name" json:"name"`
 	DesiredInstances    sql.NullInt32  `db:"desired_instances" json:"desired_instances"`
+	Ttl                 sql.NullInt32  `db:"ttl" json:"ttl"`
 	PrebuildStatus      PrebuildStatus `db:"prebuild_status" json:"prebuild_status"`
 	Deleted             bool           `db:"deleted" json:"deleted"`
 	Deprecated          bool           `db:"deprecated" json:"deprecated"`
@@ -6562,6 +6564,7 @@ func (q *sqlQuerier) GetTemplatePresetsWithPrebuilds(ctx context.Context, templa
 			&i.ID,
 			&i.Name,
 			&i.DesiredInstances,
+			&i.Ttl,
 			&i.PrebuildStatus,
 			&i.Deleted,
 			&i.Deprecated,

coderd/database/queries/prebuilds.sql

@@ -35,6 +35,7 @@ SELECT
 		tvp.id,
 		tvp.name,
 		tvp.desired_instances       AS desired_instances,
+		tvp.invalidate_after_secs 	AS ttl,
 		tvp.prebuild_status,
 		t.deleted,
 		t.deprecated != ''          AS deprecated

coderd/prebuilds/global_snapshot.go

@@ -1,6 +1,8 @@
 package prebuilds
 
 import (
+	"time"
+
 	"github.com/google/uuid"
 	"golang.org/x/xerrors"
 
@@ -41,6 +43,7 @@ func (s GlobalSnapshot) FilterByPreset(presetID uuid.UUID) (*PresetSnapshot, err
 		return nil, xerrors.Errorf("no preset found with ID %q", presetID)
 	}
 
+	// Only include workspaces that have successfully started
 	running := slice.Filter(s.RunningPrebuilds, func(prebuild database.GetRunningPrebuiltWorkspacesRow) bool {
 		if !prebuild.CurrentPresetID.Valid {
 			return false
@@ -48,6 +51,9 @@ func (s GlobalSnapshot) FilterByPreset(presetID uuid.UUID) (*PresetSnapshot, err
 		return prebuild.CurrentPresetID.UUID == preset.ID
 	})
 
+	// Separate running workspaces into non-expired and expired based on the preset's TTL
+	nonExpired, expired := filterExpiredWorkspaces(preset, running)
+
 	inProgress := slice.Filter(s.PrebuildsInProgress, func(prebuild database.CountInProgressPrebuildsRow) bool {
 		return prebuild.PresetID.UUID == preset.ID
 	})
@@ -66,9 +72,33 @@ func (s GlobalSnapshot) FilterByPreset(presetID uuid.UUID) (*PresetSnapshot, err
 
 	return &PresetSnapshot{
 		Preset:        preset,
-		Running:       running,
+		Running:       nonExpired,
+		Expired:       expired,
 		InProgress:    inProgress,
 		Backoff:       backoffPtr,
 		IsHardLimited: isHardLimited,
 	}, nil
 }
+
+// filterExpiredWorkspaces splits running workspaces into expired and non-expired
+// based on the preset's TTL.
+// If TTL is missing or zero, all workspaces are considered non-expired.
+func filterExpiredWorkspaces(preset database.GetTemplatePresetsWithPrebuildsRow, runningWorkspaces []database.GetRunningPrebuiltWorkspacesRow) (nonExpired []database.GetRunningPrebuiltWorkspacesRow, expired []database.GetRunningPrebuiltWorkspacesRow) {
+	if !preset.Ttl.Valid {
+		return runningWorkspaces, expired
+	}
+
+	ttl := time.Duration(preset.Ttl.Int32) * time.Second
+	if ttl <= 0 {
+		return runningWorkspaces, expired
+	}
+
+	for _, prebuild := range runningWorkspaces {
+		if time.Since(prebuild.CreatedAt) > ttl {
+			expired = append(expired, prebuild)
+		} else {
+			nonExpired = append(nonExpired, prebuild)
+		}
+	}
+	return nonExpired, expired
+}

coderd/prebuilds/preset_snapshot.go

@@ -31,9 +31,14 @@ const (
 // PresetSnapshot is a filtered view of GlobalSnapshot focused on a single preset.
 // It contains the raw data needed to calculate the current state of a preset's prebuilds,
 // including running prebuilds, in-progress builds, and backoff information.
+// - Running: prebuilds running and non-expired
+// - Expired: prebuilds running and expired due to the preset's TTL
+// - InProgress: prebuilds currently in progress
+// - Backoff: holds failure info to decide if prebuild creation should be backed off
 type PresetSnapshot struct {
 	Preset        database.GetTemplatePresetsWithPrebuildsRow
 	Running       []database.GetRunningPrebuiltWorkspacesRow
+	Expired       []database.GetRunningPrebuiltWorkspacesRow
 	InProgress    []database.CountInProgressPrebuildsRow
 	Backoff       *database.GetPresetsBackoffRow
 	IsHardLimited bool
@@ -43,10 +48,11 @@ type PresetSnapshot struct {
 // calculated from a PresetSnapshot. While PresetSnapshot contains raw data,
 // ReconciliationState contains derived metrics that are directly used to
 // determine what actions are needed (create, delete, or backoff).
-// For example, it calculates how many prebuilds are eligible, how many are
+// For example, it calculates how many prebuilds are expired, eligible,
-// extraneous, and how many are in various transition states.
+// how many are extraneous, and how many are in various transition states.
 type ReconciliationState struct {
-	Actual     int32 // Number of currently running prebuilds
+	Actual     int32 // Number of currently running prebuilds, i.e., non-expired, expired and extraneous prebuilds
+	Expired    int32 // Number of currently running prebuilds that exceeded their allowed time-to-live (TTL)
 	Desired    int32 // Number of prebuilds desired as defined in the preset
 	Eligible   int32 // Number of prebuilds that are ready to be claimed
 	Extraneous int32 // Number of extra running prebuilds beyond the desired count
@@ -78,7 +84,8 @@ func (ra *ReconciliationActions) IsNoop() bool {
 }
 
 // CalculateState computes the current state of prebuilds for a preset, including:
-// - Actual: Number of currently running prebuilds
+// - Actual: Number of currently running prebuilds, i.e., non-expired and expired prebuilds
+// - Expired: Number of currently running expired prebuilds
 // - Desired: Number of prebuilds desired as defined in the preset
 // - Eligible: Number of prebuilds that are ready to be claimed
 // - Extraneous: Number of extra running prebuilds beyond the desired count
@@ -92,23 +99,28 @@ func (p PresetSnapshot) CalculateState() *ReconciliationState {
 	var (
 		actual     int32
 		desired    int32
+		expired    int32
 		eligible   int32
 		extraneous int32
 	)
 
-	// #nosec G115 - Safe conversion as p.Running slice length is expected to be within int32 range
+	// #nosec G115 - Safe conversion as p.Running and p.Expired slice length is expected to be within int32 range
-	actual = int32(len(p.Running))
+	actual = int32(len(p.Running) + len(p.Expired))
+
+	// #nosec G115 - Safe conversion as p.Expired slice length is expected to be within int32 range
+	expired = int32(len(p.Expired))
 
 	if p.isActive() {
 		desired = p.Preset.DesiredInstances.Int32
 		eligible = p.countEligible()
-		extraneous = max(actual-desired, 0)
+		extraneous = max(actual-expired-desired, 0)
 	}
 
 	starting, stopping, deleting := p.countInProgress()
 
 	return &ReconciliationState{
 		Actual:     actual,
+		Expired:    expired,
 		Desired:    desired,
 		Eligible:   eligible,
 		Extraneous: extraneous,
@@ -126,6 +138,7 @@ func (p PresetSnapshot) CalculateState() *ReconciliationState {
 // 3. For active presets, it calculates the number of prebuilds to create or delete based on:
 //   - The desired number of instances
 //   - Currently running prebuilds
+//   - Currently running expired prebuilds
 //   - Prebuilds in transition states (starting/stopping/deleting)
 //   - Any extraneous prebuilds that need to be removed
 //
@@ -133,7 +146,7 @@ func (p PresetSnapshot) CalculateState() *ReconciliationState {
 // - ActionTypeBackoff: Only BackoffUntil is set, indicating when to retry
 // - ActionTypeCreate: Only Create is set, indicating how many prebuilds to create
 // - ActionTypeDelete: Only DeleteIDs is set, containing IDs of prebuilds to delete
-func (p PresetSnapshot) CalculateActions(clock quartz.Clock, backoffInterval time.Duration) (*ReconciliationActions, error) {
+func (p PresetSnapshot) CalculateActions(clock quartz.Clock, backoffInterval time.Duration) ([]*ReconciliationActions, error) {
 	// TODO: align workspace states with how we represent them on the FE and the CLI
 	//	     right now there's some slight differences which can lead to additional prebuilds being created
 
@@ -158,45 +171,77 @@ func (p PresetSnapshot) isActive() bool {
 	return p.Preset.UsingActiveVersion && !p.Preset.Deleted && !p.Preset.Deprecated
 }
 
-// handleActiveTemplateVersion deletes excess prebuilds if there are too many,
+// handleActiveTemplateVersion determines the reconciliation actions for a preset with an active template version.
-// otherwise creates new ones to reach the desired count.
+// It ensures the system moves towards the desired number of healthy prebuilds.
-func (p PresetSnapshot) handleActiveTemplateVersion() (*ReconciliationActions, error) {
+//
+// The reconciliation follows this order:
+//  1. Delete expired prebuilds: These are no longer valid and must be removed first.
+//  2. Delete extraneous prebuilds: After expired ones are removed, if the number of running non-expired prebuilds
+//     still exceeds the desired count, the oldest prebuilds are deleted to reduce excess.
+//  3. Create missing prebuilds: If the number of non-expired, non-starting prebuilds is still below the desired count,
+//     create the necessary number of prebuilds to reach the target.
+//
+// The function returns a list of actions to be executed to achieve the desired state.
+func (p PresetSnapshot) handleActiveTemplateVersion() (actions []*ReconciliationActions, err error) {
 	state := p.CalculateState()
 
-	// If we have more prebuilds than desired, delete the oldest ones
+	// If we have expired prebuilds, delete them
+	if state.Expired > 0 {
+		var deleteIDs []uuid.UUID
+		for _, expired := range p.Expired {
+			deleteIDs = append(deleteIDs, expired.ID)
+		}
+		actions = append(actions,
+			&ReconciliationActions{
+				ActionType: ActionTypeDelete,
+				DeleteIDs:  deleteIDs,
+			})
+	}
+
+	// If we still have more prebuilds than desired, delete the oldest ones
 	if state.Extraneous > 0 {
-		return &ReconciliationActions{
+		actions = append(actions,
+			&ReconciliationActions{
 				ActionType: ActionTypeDelete,
 				DeleteIDs:  p.getOldestPrebuildIDs(int(state.Extraneous)),
-		}, nil
+			})
 	}
 
+	// Number of running prebuilds excluding the recently deleted Expired
+	runningValid := state.Actual - state.Expired
+
 	// Calculate how many new prebuilds we need to create
 	// We subtract starting prebuilds since they're already being created
-	prebuildsToCreate := max(state.Desired-state.Actual-state.Starting, 0)
+	prebuildsToCreate := max(state.Desired-runningValid-state.Starting, 0)
-
+	if prebuildsToCreate > 0 {
-	return &ReconciliationActions{
+		actions = append(actions,
+			&ReconciliationActions{
 				ActionType: ActionTypeCreate,
 				Create:     prebuildsToCreate,
-	}, nil
+			})
+	}
+
+	return actions, nil
 }
 
 // handleInactiveTemplateVersion deletes all running prebuilds except those already being deleted
 // to avoid duplicate deletion attempts.
-func (p PresetSnapshot) handleInactiveTemplateVersion() (*ReconciliationActions, error) {
+func (p PresetSnapshot) handleInactiveTemplateVersion() ([]*ReconciliationActions, error) {
 	prebuildsToDelete := len(p.Running)
 	deleteIDs := p.getOldestPrebuildIDs(prebuildsToDelete)
 
-	return &ReconciliationActions{
+	return []*ReconciliationActions{
+		{
 			ActionType: ActionTypeDelete,
 			DeleteIDs:  deleteIDs,
+		},
 	}, nil
 }
 
 // needsBackoffPeriod checks if we should delay prebuild creation due to recent failures.
 // If there were failures, it calculates a backoff period based on the number of failures
 // and returns true if we're still within that period.
-func (p PresetSnapshot) needsBackoffPeriod(clock quartz.Clock, backoffInterval time.Duration) (*ReconciliationActions, bool) {
+func (p PresetSnapshot) needsBackoffPeriod(clock quartz.Clock, backoffInterval time.Duration) ([]*ReconciliationActions, bool) {
 	if p.Backoff == nil || p.Backoff.NumFailed == 0 {
 		return nil, false
 	}
@@ -205,9 +250,11 @@ func (p PresetSnapshot) needsBackoffPeriod(clock quartz.Clock, backoffInterval t
 		return nil, false
 	}
 
-	return &ReconciliationActions{
+	return []*ReconciliationActions{
+		{
 			ActionType:   ActionTypeBackoff,
 			BackoffUntil: backoffUntil,
+		},
 	}, true
 }
 

coderd/prebuilds/preset_snapshot_test.go

@@ -23,6 +23,7 @@ type options struct {
 	presetName          string
 	prebuiltWorkspaceID uuid.UUID
 	workspaceName       string
+	ttl                 int32
 }
 
 // templateID is common across all option sets.
@@ -34,6 +35,7 @@ const (
 	optionSet0 = iota
 	optionSet1
 	optionSet2
+	optionSet3
 )
 
 var opts = map[uint]options{
@@ -61,6 +63,15 @@ var opts = map[uint]options{
 		prebuiltWorkspaceID: uuid.UUID{33},
 		workspaceName:       "prebuilds2",
 	},
+	optionSet3: {
+		templateID:          templateID,
+		templateVersionID:   uuid.UUID{41},
+		presetID:            uuid.UUID{42},
+		presetName:          "my-preset",
+		prebuiltWorkspaceID: uuid.UUID{43},
+		workspaceName:       "prebuilds3",
+		ttl:                 5, // seconds
+	},
 }
 
 // A new template version with a preset without prebuilds configured should result in no prebuilds being created.
@@ -82,10 +93,7 @@ func TestNoPrebuilds(t *testing.T) {
 	require.NoError(t, err)
 
 	validateState(t, prebuilds.ReconciliationState{ /*all zero values*/ }, *state)
-	validateActions(t, prebuilds.ReconciliationActions{
+	validateActions(t, nil, actions)
-		ActionType: prebuilds.ActionTypeCreate,
-		Create:     0,
-	}, *actions)
 }
 
 // A new template version with a preset with prebuilds configured should result in a new prebuild being created.
@@ -109,10 +117,12 @@ func TestNetNew(t *testing.T) {
 	validateState(t, prebuilds.ReconciliationState{
 		Desired: 1,
 	}, *state)
-	validateActions(t, prebuilds.ReconciliationActions{
+	validateActions(t, []*prebuilds.ReconciliationActions{
+		{
 			ActionType: prebuilds.ActionTypeCreate,
 			Create:     1,
-	}, *actions)
+		},
+	}, actions)
 }
 
 // A new template version is created with a preset with prebuilds configured; this outdates the older version and
@@ -149,10 +159,12 @@ func TestOutdatedPrebuilds(t *testing.T) {
 	validateState(t, prebuilds.ReconciliationState{
 		Actual: 1,
 	}, *state)
-	validateActions(t, prebuilds.ReconciliationActions{
+	validateActions(t, []*prebuilds.ReconciliationActions{
+		{
 			ActionType: prebuilds.ActionTypeDelete,
 			DeleteIDs:  []uuid.UUID{outdated.prebuiltWorkspaceID},
-	}, *actions)
+		},
+	}, actions)
 
 	// WHEN: calculating the current preset's state.
 	ps, err = snapshot.FilterByPreset(current.presetID)
@@ -163,10 +175,12 @@ func TestOutdatedPrebuilds(t *testing.T) {
 	actions, err = ps.CalculateActions(clock, backoffInterval)
 	require.NoError(t, err)
 	validateState(t, prebuilds.ReconciliationState{Desired: 1}, *state)
-	validateActions(t, prebuilds.ReconciliationActions{
+	validateActions(t, []*prebuilds.ReconciliationActions{
+		{
 			ActionType: prebuilds.ActionTypeCreate,
 			Create:     1,
-	}, *actions)
+		},
+	}, actions)
 }
 
 // Make sure that outdated prebuild will be deleted, even if deletion of another outdated prebuild is already in progress.
@@ -214,10 +228,12 @@ func TestDeleteOutdatedPrebuilds(t *testing.T) {
 		Deleting: 1,
 	}, *state)
 
-	validateActions(t, prebuilds.ReconciliationActions{
+	validateActions(t, []*prebuilds.ReconciliationActions{
+		{
 			ActionType: prebuilds.ActionTypeDelete,
 			DeleteIDs:  []uuid.UUID{outdated.prebuiltWorkspaceID},
-	}, *actions)
+		},
+	}, actions)
 }
 
 // A new template version is created with a preset with prebuilds configured; while a prebuild is provisioning up or down,
@@ -233,7 +249,7 @@ func TestInProgressActions(t *testing.T) {
 		desired    int32
 		running    int32
 		inProgress int32
-		checkFn    func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions)
+		checkFn    func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions)
 	}{
 		// With no running prebuilds and one starting, no creations/deletions should take place.
 		{
@@ -242,11 +258,9 @@ func TestInProgressActions(t *testing.T) {
 			desired:    1,
 			running:    0,
 			inProgress: 1,
-			checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
+			checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
 				validateState(t, prebuilds.ReconciliationState{Desired: 1, Starting: 1}, state)
-				validateActions(t, prebuilds.ReconciliationActions{
+				validateActions(t, nil, actions)
-					ActionType: prebuilds.ActionTypeCreate,
-				}, actions)
 			},
 		},
 		// With one running prebuild and one starting, no creations/deletions should occur since we're approaching the correct state.
@@ -256,11 +270,9 @@ func TestInProgressActions(t *testing.T) {
 			desired:    2,
 			running:    1,
 			inProgress: 1,
-			checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
+			checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
 				validateState(t, prebuilds.ReconciliationState{Actual: 1, Desired: 2, Starting: 1}, state)
-				validateActions(t, prebuilds.ReconciliationActions{
+				validateActions(t, nil, actions)
-					ActionType: prebuilds.ActionTypeCreate,
-				}, actions)
 			},
 		},
 		// With one running prebuild and one starting, no creations/deletions should occur
@@ -271,11 +283,9 @@ func TestInProgressActions(t *testing.T) {
 			desired:    2,
 			running:    2,
 			inProgress: 1,
-			checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
+			checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
 				validateState(t, prebuilds.ReconciliationState{Actual: 2, Desired: 2, Starting: 1}, state)
-				validateActions(t, prebuilds.ReconciliationActions{
+				validateActions(t, nil, actions)
-					ActionType: prebuilds.ActionTypeCreate,
-				}, actions)
 			},
 		},
 		// With one prebuild desired and one stopping, a new prebuild will be created.
@@ -285,11 +295,13 @@ func TestInProgressActions(t *testing.T) {
 			desired:    1,
 			running:    0,
 			inProgress: 1,
-			checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
+			checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
 				validateState(t, prebuilds.ReconciliationState{Desired: 1, Stopping: 1}, state)
-				validateActions(t, prebuilds.ReconciliationActions{
+				validateActions(t, []*prebuilds.ReconciliationActions{
+					{
 						ActionType: prebuilds.ActionTypeCreate,
 						Create:     1,
+					},
 				}, actions)
 			},
 		},
@@ -300,11 +312,13 @@ func TestInProgressActions(t *testing.T) {
 			desired:    3,
 			running:    2,
 			inProgress: 1,
-			checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
+			checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
 				validateState(t, prebuilds.ReconciliationState{Actual: 2, Desired: 3, Stopping: 1}, state)
-				validateActions(t, prebuilds.ReconciliationActions{
+				validateActions(t, []*prebuilds.ReconciliationActions{
+					{
 						ActionType: prebuilds.ActionTypeCreate,
 						Create:     1,
+					},
 				}, actions)
 			},
 		},
@@ -315,11 +329,9 @@ func TestInProgressActions(t *testing.T) {
 			desired:    3,
 			running:    3,
 			inProgress: 1,
-			checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
+			checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
 				validateState(t, prebuilds.ReconciliationState{Actual: 3, Desired: 3, Stopping: 1}, state)
-				validateActions(t, prebuilds.ReconciliationActions{
+				validateActions(t, nil, actions)
-					ActionType: prebuilds.ActionTypeCreate,
-				}, actions)
 			},
 		},
 		// With one prebuild desired and one deleting, a new prebuild will be created.
@@ -329,11 +341,13 @@ func TestInProgressActions(t *testing.T) {
 			desired:    1,
 			running:    0,
 			inProgress: 1,
-			checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
+			checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
 				validateState(t, prebuilds.ReconciliationState{Desired: 1, Deleting: 1}, state)
-				validateActions(t, prebuilds.ReconciliationActions{
+				validateActions(t, []*prebuilds.ReconciliationActions{
+					{
 						ActionType: prebuilds.ActionTypeCreate,
 						Create:     1,
+					},
 				}, actions)
 			},
 		},
@@ -344,11 +358,13 @@ func TestInProgressActions(t *testing.T) {
 			desired:    2,
 			running:    1,
 			inProgress: 1,
-			checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
+			checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
 				validateState(t, prebuilds.ReconciliationState{Actual: 1, Desired: 2, Deleting: 1}, state)
-				validateActions(t, prebuilds.ReconciliationActions{
+				validateActions(t, []*prebuilds.ReconciliationActions{
+					{
 						ActionType: prebuilds.ActionTypeCreate,
 						Create:     1,
+					},
 				}, actions)
 			},
 		},
@@ -359,11 +375,9 @@ func TestInProgressActions(t *testing.T) {
 			desired:    2,
 			running:    2,
 			inProgress: 1,
-			checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
+			checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
 				validateState(t, prebuilds.ReconciliationState{Actual: 2, Desired: 2, Deleting: 1}, state)
-				validateActions(t, prebuilds.ReconciliationActions{
+				validateActions(t, nil, actions)
-					ActionType: prebuilds.ActionTypeCreate,
-				}, actions)
 			},
 		},
 		// With 3 prebuilds desired, 1 running, and 2 starting, no creations should occur since the builds are in progress.
@@ -373,9 +387,9 @@ func TestInProgressActions(t *testing.T) {
 			desired:    3,
 			running:    1,
 			inProgress: 2,
-			checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
+			checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
 				validateState(t, prebuilds.ReconciliationState{Actual: 1, Desired: 3, Starting: 2}, state)
-				validateActions(t, prebuilds.ReconciliationActions{ActionType: prebuilds.ActionTypeCreate, Create: 0}, actions)
+				validateActions(t, nil, actions)
 			},
 		},
 		// With 3 prebuilds desired, 5 running, and 2 deleting, no deletions should occur since the builds are in progress.
@@ -385,17 +399,20 @@ func TestInProgressActions(t *testing.T) {
 			desired:    3,
 			running:    5,
 			inProgress: 2,
-			checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
+			checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
 				expectedState := prebuilds.ReconciliationState{Actual: 5, Desired: 3, Deleting: 2, Extraneous: 2}
-				expectedActions := prebuilds.ReconciliationActions{
+				expectedActions := []*prebuilds.ReconciliationActions{
+					{
 						ActionType: prebuilds.ActionTypeDelete,
+					},
 				}
 
 				validateState(t, expectedState, state)
-				assert.EqualValuesf(t, expectedActions.ActionType, actions.ActionType, "'ActionType' did not match expectation")
+				require.Equal(t, len(expectedActions), len(actions))
-				assert.Len(t, actions.DeleteIDs, 2, "'deleteIDs' did not match expectation")
+				assert.EqualValuesf(t, expectedActions[0].ActionType, actions[0].ActionType, "'ActionType' did not match expectation")
-				assert.EqualValuesf(t, expectedActions.Create, actions.Create, "'create' did not match expectation")
+				assert.Len(t, actions[0].DeleteIDs, 2, "'deleteIDs' did not match expectation")
-				assert.EqualValuesf(t, expectedActions.BackoffUntil, actions.BackoffUntil, "'BackoffUntil' did not match expectation")
+				assert.EqualValuesf(t, expectedActions[0].Create, actions[0].Create, "'create' did not match expectation")
+				assert.EqualValuesf(t, expectedActions[0].BackoffUntil, actions[0].BackoffUntil, "'BackoffUntil' did not match expectation")
 			},
 		},
 	}
@@ -450,7 +467,7 @@ func TestInProgressActions(t *testing.T) {
 			state := ps.CalculateState()
 			actions, err := ps.CalculateActions(clock, backoffInterval)
 			require.NoError(t, err)
-			tc.checkFn(*state, *actions)
+			tc.checkFn(*state, actions)
 		})
 	}
 }
@@ -496,10 +513,187 @@ func TestExtraneous(t *testing.T) {
 	validateState(t, prebuilds.ReconciliationState{
 		Actual: 2, Desired: 1, Extraneous: 1, Eligible: 2,
 	}, *state)
-	validateActions(t, prebuilds.ReconciliationActions{
+	validateActions(t, []*prebuilds.ReconciliationActions{
+		{
 			ActionType: prebuilds.ActionTypeDelete,
 			DeleteIDs:  []uuid.UUID{older},
-	}, *actions)
+		},
+	}, actions)
+}
+
+// A prebuild is considered Expired when it has exceeded their time-to-live (TTL)
+// specified in the preset's cache invalidation invalidate_after_secs parameter.
+func TestExpiredPrebuilds(t *testing.T) {
+	t.Parallel()
+	current := opts[optionSet3]
+	clock := quartz.NewMock(t)
+
+	cases := []struct {
+		name    string
+		running int32
+		desired int32
+		expired int32
+		checkFn func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions)
+	}{
+		// With 2 running prebuilds, none of which are expired, and the desired count is met,
+		// no deletions or creations should occur.
+		{
+			name:    "no expired prebuilds - no actions taken",
+			running: 2,
+			desired: 2,
+			expired: 0,
+			checkFn: func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
+				validateState(t, prebuilds.ReconciliationState{Actual: 2, Desired: 2, Expired: 0}, state)
+				validateActions(t, nil, actions)
+			},
+		},
+		// With 2 running prebuilds, 1 of which is expired, the expired prebuild should be deleted,
+		// and one new prebuild should be created to maintain the desired count.
+		{
+			name:    "one expired prebuild – deleted and replaced",
+			running: 2,
+			desired: 2,
+			expired: 1,
+			checkFn: func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
+				expectedState := prebuilds.ReconciliationState{Actual: 2, Desired: 2, Expired: 1}
+				expectedActions := []*prebuilds.ReconciliationActions{
+					{
+						ActionType: prebuilds.ActionTypeDelete,
+						DeleteIDs:  []uuid.UUID{runningPrebuilds[0].ID},
+					},
+					{
+						ActionType: prebuilds.ActionTypeCreate,
+						Create:     1,
+					},
+				}
+
+				validateState(t, expectedState, state)
+				validateActions(t, expectedActions, actions)
+			},
+		},
+		// With 2 running prebuilds, both expired, both should be deleted,
+		// and 2 new prebuilds created to match the desired count.
+		{
+			name:    "all prebuilds expired – all deleted and recreated",
+			running: 2,
+			desired: 2,
+			expired: 2,
+			checkFn: func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
+				expectedState := prebuilds.ReconciliationState{Actual: 2, Desired: 2, Expired: 2}
+				expectedActions := []*prebuilds.ReconciliationActions{
+					{
+						ActionType: prebuilds.ActionTypeDelete,
+						DeleteIDs:  []uuid.UUID{runningPrebuilds[0].ID, runningPrebuilds[1].ID},
+					},
+					{
+						ActionType: prebuilds.ActionTypeCreate,
+						Create:     2,
+					},
+				}
+
+				validateState(t, expectedState, state)
+				validateActions(t, expectedActions, actions)
+			},
+		},
+		// With 4 running prebuilds, 2 of which are expired, and the desired count is 2,
+		// the expired prebuilds should be deleted. No new creations are needed
+		// since removing the expired ones brings actual = desired.
+		{
+			name:    "expired prebuilds deleted to reach desired count",
+			running: 4,
+			desired: 2,
+			expired: 2,
+			checkFn: func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
+				expectedState := prebuilds.ReconciliationState{Actual: 4, Desired: 2, Expired: 2, Extraneous: 0}
+				expectedActions := []*prebuilds.ReconciliationActions{
+					{
+						ActionType: prebuilds.ActionTypeDelete,
+						DeleteIDs:  []uuid.UUID{runningPrebuilds[0].ID, runningPrebuilds[1].ID},
+					},
+				}
+
+				validateState(t, expectedState, state)
+				validateActions(t, expectedActions, actions)
+			},
+		},
+		// With 4 running prebuilds (1 expired), and the desired count is 2,
+		// the first action should delete the expired one,
+		// and the second action should delete one additional (non-expired) prebuild
+		// to eliminate the remaining excess.
+		{
+			name:    "expired prebuild deleted first, then extraneous",
+			running: 4,
+			desired: 2,
+			expired: 1,
+			checkFn: func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
+				expectedState := prebuilds.ReconciliationState{Actual: 4, Desired: 2, Expired: 1, Extraneous: 1}
+				expectedActions := []*prebuilds.ReconciliationActions{
+					// First action correspond to deleting the expired prebuild,
+					// and the second action corresponds to deleting the extraneous prebuild
+					// corresponding to the oldest one after the expired prebuild
+					{
+						ActionType: prebuilds.ActionTypeDelete,
+						DeleteIDs:  []uuid.UUID{runningPrebuilds[0].ID},
+					},
+					{
+						ActionType: prebuilds.ActionTypeDelete,
+						DeleteIDs:  []uuid.UUID{runningPrebuilds[1].ID},
+					},
+				}
+
+				validateState(t, expectedState, state)
+				validateActions(t, expectedActions, actions)
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			// GIVEN: a preset.
+			defaultPreset := preset(true, tc.desired, current)
+			presets := []database.GetTemplatePresetsWithPrebuildsRow{
+				defaultPreset,
+			}
+
+			// GIVEN: running prebuilt workspaces for the preset.
+			running := make([]database.GetRunningPrebuiltWorkspacesRow, 0, tc.running)
+			expiredCount := 0
+			ttlDuration := time.Duration(defaultPreset.Ttl.Int32)
+			for range tc.running {
+				name, err := prebuilds.GenerateName()
+				require.NoError(t, err)
+				prebuildCreateAt := time.Now()
+				if int(tc.expired) > expiredCount {
+					// Update the prebuild workspace createdAt to exceed its TTL (5 seconds)
+					prebuildCreateAt = prebuildCreateAt.Add(-ttlDuration - 10*time.Second)
+					expiredCount++
+				}
+				running = append(running, database.GetRunningPrebuiltWorkspacesRow{
+					ID:                uuid.New(),
+					Name:              name,
+					TemplateID:        current.templateID,
+					TemplateVersionID: current.templateVersionID,
+					CurrentPresetID:   uuid.NullUUID{UUID: current.presetID, Valid: true},
+					Ready:             false,
+					CreatedAt:         prebuildCreateAt,
+				})
+			}
+
+			// WHEN: calculating the current preset's state.
+			snapshot := prebuilds.NewGlobalSnapshot(presets, running, nil, nil, nil)
+			ps, err := snapshot.FilterByPreset(current.presetID)
+			require.NoError(t, err)
+
+			// THEN: we should identify that this prebuild is expired.
+			state := ps.CalculateState()
+			actions, err := ps.CalculateActions(clock, backoffInterval)
+			require.NoError(t, err)
+			tc.checkFn(running, *state, actions)
+		})
+	}
 }
 
 // A template marked as deprecated will not have prebuilds running.
@@ -536,10 +730,12 @@ func TestDeprecated(t *testing.T) {
 	validateState(t, prebuilds.ReconciliationState{
 		Actual: 1,
 	}, *state)
-	validateActions(t, prebuilds.ReconciliationActions{
+	validateActions(t, []*prebuilds.ReconciliationActions{
+		{
 			ActionType: prebuilds.ActionTypeDelete,
 			DeleteIDs:  []uuid.UUID{current.prebuiltWorkspaceID},
-	}, *actions)
+		},
+	}, actions)
 }
 
 // If the latest build failed, backoff exponentially with the given interval.
@@ -587,10 +783,12 @@ func TestLatestBuildFailed(t *testing.T) {
 	validateState(t, prebuilds.ReconciliationState{
 		Actual: 0, Desired: 1,
 	}, *state)
-	validateActions(t, prebuilds.ReconciliationActions{
+	validateActions(t, []*prebuilds.ReconciliationActions{
+		{
 			ActionType:   prebuilds.ActionTypeBackoff,
 			BackoffUntil: lastBuildTime.Add(time.Duration(numFailed) * backoffInterval),
-	}, *actions)
+		},
+	}, actions)
 
 	// WHEN: calculating the other preset's state.
 	psOther, err := snapshot.FilterByPreset(other.presetID)
@@ -603,10 +801,7 @@ func TestLatestBuildFailed(t *testing.T) {
 	validateState(t, prebuilds.ReconciliationState{
 		Actual: 1, Desired: 1, Eligible: 1,
 	}, *state)
-	validateActions(t, prebuilds.ReconciliationActions{
+	validateActions(t, nil, actions)
-		ActionType:   prebuilds.ActionTypeCreate,
-		BackoffUntil: time.Time{},
-	}, *actions)
 
 	// WHEN: the clock is advanced a backoff interval.
 	clock.Advance(backoffInterval + time.Microsecond)
@@ -620,11 +815,12 @@ func TestLatestBuildFailed(t *testing.T) {
 	validateState(t, prebuilds.ReconciliationState{
 		Actual: 0, Desired: 1,
 	}, *state)
-	validateActions(t, prebuilds.ReconciliationActions{
+	validateActions(t, []*prebuilds.ReconciliationActions{
+		{
 			ActionType: prebuilds.ActionTypeCreate,
 			Create:     1, // <--- NOTE: we're now able to create a new prebuild because the interval has elapsed.
-
+		},
-	}, *actions)
+	}, actions)
 }
 
 func TestMultiplePresetsPerTemplateVersion(t *testing.T) {
@@ -684,10 +880,7 @@ func TestMultiplePresetsPerTemplateVersion(t *testing.T) {
 			Starting: 1,
 			Desired:  1,
 		}, *state)
-		validateActions(t, prebuilds.ReconciliationActions{
+		validateActions(t, nil, actions)
-			ActionType: prebuilds.ActionTypeCreate,
-			Create:     0,
-		}, *actions)
 	}
 
 	// One prebuild has to be created for preset 2. Make sure preset 1 doesn't block preset 2.
@@ -703,14 +896,23 @@ func TestMultiplePresetsPerTemplateVersion(t *testing.T) {
 			Starting: 0,
 			Desired:  1,
 		}, *state)
-		validateActions(t, prebuilds.ReconciliationActions{
+		validateActions(t, []*prebuilds.ReconciliationActions{
+			{
 				ActionType: prebuilds.ActionTypeCreate,
 				Create:     1,
-		}, *actions)
+			},
+		}, actions)
 	}
 }
 
 func preset(active bool, instances int32, opts options, muts ...func(row database.GetTemplatePresetsWithPrebuildsRow) database.GetTemplatePresetsWithPrebuildsRow) database.GetTemplatePresetsWithPrebuildsRow {
+	ttl := sql.NullInt32{}
+	if opts.ttl > 0 {
+		ttl = sql.NullInt32{
+			Valid: true,
+			Int32: opts.ttl,
+		}
+	}
 	entry := database.GetTemplatePresetsWithPrebuildsRow{
 		TemplateID:         opts.templateID,
 		TemplateVersionID:  opts.templateVersionID,
@@ -723,6 +925,7 @@ func preset(active bool, instances int32, opts options, muts ...func(row databas
 		},
 		Deleted:    false,
 		Deprecated: false,
+		Ttl:        ttl,
 	}
 
 	for _, mut := range muts {
@@ -758,6 +961,6 @@ func validateState(t *testing.T, expected, actual prebuilds.ReconciliationState)
 
 // validateActions is a convenience func to make tests more readable; it exploits the fact that the default states for
 // prebuilds align with zero values.
-func validateActions(t *testing.T, expected, actual prebuilds.ReconciliationActions) {
+func validateActions(t *testing.T, expected, actual []*prebuilds.ReconciliationActions) {
 	require.Equal(t, expected, actual)
 }

coderd/provisionerdserver/provisionerdserver.go

@@ -2059,12 +2059,18 @@ func InsertWorkspacePresetsAndParameters(ctx context.Context, logger slog.Logger
 
 func InsertWorkspacePresetAndParameters(ctx context.Context, db database.Store, templateVersionID uuid.UUID, protoPreset *sdkproto.Preset, t time.Time) error {
 	err := db.InTx(func(tx database.Store) error {
-		var desiredInstances sql.NullInt32
+		var desiredInstances, ttl sql.NullInt32
 		if protoPreset != nil && protoPreset.Prebuild != nil {
 			desiredInstances = sql.NullInt32{
 				Int32: protoPreset.Prebuild.Instances,
 				Valid: true,
 			}
+			if protoPreset.Prebuild.ExpirationPolicy != nil {
+				ttl = sql.NullInt32{
+					Int32: protoPreset.Prebuild.ExpirationPolicy.Ttl,
+					Valid: true,
+				}
+			}
 		}
 		dbPreset, err := tx.InsertPreset(ctx, database.InsertPresetParams{
 			ID:                  uuid.New(),
@@ -2072,10 +2078,7 @@ func InsertWorkspacePresetAndParameters(ctx context.Context, db database.Store,
 			Name:                protoPreset.Name,
 			CreatedAt:           t,
 			DesiredInstances:    desiredInstances,
-			InvalidateAfterSecs: sql.NullInt32{
+			InvalidateAfterSecs: ttl,
-				Int32: 0,
-				Valid: false,
-			}, // TODO: implement cache invalidation
 		})
 		if err != nil {
 			return xerrors.Errorf("insert preset: %w", err)

docs/admin/templates/extending-templates/prebuilt-workspaces.md

@@ -31,7 +31,8 @@ Prebuilt workspaces are tightly integrated with [workspace presets](./parameters
 ## Enable prebuilt workspaces for template presets
 
 In your template, add a `prebuilds` block within a `coder_workspace_preset` definition to identify the number of prebuilt
-instances your Coder deployment should maintain:
+instances your Coder deployment should maintain, and optionally configure a `expiration_policy` block to set a TTL
+(Time To Live) for unclaimed prebuilt workspaces to ensure stale resources are automatically cleaned up.
 
    ```hcl
    data "coder_workspace_preset" "goland" {
@@ -43,6 +44,9 @@ instances your Coder deployment should maintain:
      }
      prebuilds {
        instances = 3   # Number of prebuilt workspaces to maintain
+       expiration_policy {
+          ttl = 86400  # Time (in seconds) after which unclaimed prebuilds are expired (1 day)
+      }
      }
    }
    ```
@@ -50,6 +54,9 @@ instances your Coder deployment should maintain:
 After you publish a new template version, Coder will automatically provision and maintain prebuilt workspaces through an
 internal reconciliation loop (similar to Kubernetes) to ensure the defined `instances` count are running.
 
+The `expiration_policy` block ensures that any prebuilt workspaces left unclaimed for more than `ttl` seconds is considered
+expired and automatically cleaned up.
+
 ## Prebuilt workspace lifecycle
 
 Prebuilt workspaces follow a specific lifecycle from creation through eligibility to claiming.
@@ -95,6 +102,15 @@ Unclaimed prebuilt workspaces can be interacted with in the same way as any othe
 However, if a Prebuilt workspace is stopped, the reconciliation loop will not destroy it.
 This gives template admins the ability to park problematic prebuilt workspaces in a stopped state for further investigation.
 
+### Expiration Policy
+
+Prebuilt workspaces support expiration policies through the `ttl` setting inside the `expiration_policy` block.
+This value defines the Time To Live (TTL) of a prebuilt workspace, i.e., the duration in seconds that an unclaimed
+prebuilt workspace can remain before it is considered expired and eligible for cleanup.
+
+Expired prebuilt workspaces are removed during the reconciliation loop to avoid stale environments and resource waste.
+New prebuilt workspaces are only created to maintain the desired count if needed.
+
 ### Template updates and the prebuilt workspace lifecycle
 
 Prebuilt workspaces are not updated after they are provisioned.

enterprise/coderd/prebuilds/reconcile.go

@@ -396,97 +396,28 @@ func (c *StoreReconciler) ReconcilePreset(ctx context.Context, ps prebuilds.Pres
 	actions, err := c.CalculateActions(ctx, ps)
 	if err != nil {
 		logger.Error(ctx, "failed to calculate actions for preset", slog.Error(err))
-		return nil
+		return err
-	}
-
-	// Nothing has to be done.
-	if !ps.Preset.UsingActiveVersion && actions.IsNoop() {
-		logger.Debug(ctx, "skipping reconciliation for preset - nothing has to be done")
-		return nil
-	}
-
-	// nolint:gocritic // ReconcilePreset needs Prebuilds Orchestrator permissions.
-	prebuildsCtx := dbauthz.AsPrebuildsOrchestrator(ctx)
-
-	levelFn := logger.Debug
-	switch {
-	case actions.ActionType == prebuilds.ActionTypeBackoff:
-		levelFn = logger.Warn
-	// Log at info level when there's a change to be effected.
-	case actions.ActionType == prebuilds.ActionTypeCreate && actions.Create > 0:
-		levelFn = logger.Info
-	case actions.ActionType == prebuilds.ActionTypeDelete && len(actions.DeleteIDs) > 0:
-		levelFn = logger.Info
 	}
 
 	fields := []any{
-		slog.F("action_type", actions.ActionType),
-		slog.F("create_count", actions.Create), slog.F("delete_count", len(actions.DeleteIDs)),
-		slog.F("to_delete", actions.DeleteIDs),
 		slog.F("desired", state.Desired), slog.F("actual", state.Actual),
 		slog.F("extraneous", state.Extraneous), slog.F("starting", state.Starting),
 		slog.F("stopping", state.Stopping), slog.F("deleting", state.Deleting),
 		slog.F("eligible", state.Eligible),
 	}
 
-	levelFn(ctx, "calculated reconciliation actions for preset", fields...)
+	levelFn := logger.Debug
-
+	levelFn(ctx, "calculated reconciliation state for preset", fields...)
-	switch actions.ActionType {
-	case prebuilds.ActionTypeBackoff:
-		// If there is anything to backoff for (usually a cycle of failed prebuilds), then log and bail out.
-		levelFn(ctx, "template prebuild state retrieved, backing off",
-			append(fields,
-				slog.F("backoff_until", actions.BackoffUntil.Format(time.RFC3339)),
-				slog.F("backoff_secs", math.Round(actions.BackoffUntil.Sub(c.clock.Now()).Seconds())),
-			)...)
-
-		return nil
-
-	case prebuilds.ActionTypeCreate:
-		// Unexpected things happen (i.e. bugs or bitflips); let's defend against disastrous outcomes.
-		// See https://blog.robertelder.org/causes-of-bit-flips-in-computer-memory/.
-		// This is obviously not comprehensive protection against this sort of problem, but this is one essential check.
-		desired := ps.Preset.DesiredInstances.Int32
-		if actions.Create > desired {
-			logger.Critical(ctx, "determined excessive count of prebuilds to create; clamping to desired count",
-				slog.F("create_count", actions.Create), slog.F("desired_count", desired))
-
-			actions.Create = desired
-		}
-
-		// If preset is hard-limited, and it's a create operation, log it and exit early.
-		// Creation operation is disallowed for hard-limited preset.
-		if ps.IsHardLimited && actions.Create > 0 {
-			logger.Warn(ctx, "skipping hard limited preset for create operation")
-			return nil
-		}
 
 	var multiErr multierror.Error
-
+	for _, action := range actions {
-		for range actions.Create {
+		err = c.executeReconciliationAction(ctx, logger, ps, action)
-			if err := c.createPrebuiltWorkspace(prebuildsCtx, uuid.New(), ps.Preset.TemplateID, ps.Preset.ID); err != nil {
+		if err != nil {
-				logger.Error(ctx, "failed to create prebuild", slog.Error(err))
+			logger.Error(ctx, "failed to execute action", "type", action.ActionType, slog.Error(err))
 			multiErr.Errors = append(multiErr.Errors, err)
 		}
 	}
-
 	return multiErr.ErrorOrNil()
-
-	case prebuilds.ActionTypeDelete:
-		var multiErr multierror.Error
-
-		for _, id := range actions.DeleteIDs {
-			if err := c.deletePrebuiltWorkspace(prebuildsCtx, id, ps.Preset.TemplateID, ps.Preset.ID); err != nil {
-				logger.Error(ctx, "failed to delete prebuild", slog.Error(err))
-				multiErr.Errors = append(multiErr.Errors, err)
-			}
-		}
-
-		return multiErr.ErrorOrNil()
-
-	default:
-		return xerrors.Errorf("unknown action type: %v", actions.ActionType)
-	}
 }
 
 func (c *StoreReconciler) notifyPrebuildFailureLimitReached(ctx context.Context, ps prebuilds.PresetSnapshot) error {
@@ -532,7 +463,7 @@ func (c *StoreReconciler) notifyPrebuildFailureLimitReached(ctx context.Context,
 	return nil
 }
 
-func (c *StoreReconciler) CalculateActions(ctx context.Context, snapshot prebuilds.PresetSnapshot) (*prebuilds.ReconciliationActions, error) {
+func (c *StoreReconciler) CalculateActions(ctx context.Context, snapshot prebuilds.PresetSnapshot) ([]*prebuilds.ReconciliationActions, error) {
 	if ctx.Err() != nil {
 		return nil, ctx.Err()
 	}
@@ -581,6 +512,101 @@ func (c *StoreReconciler) WithReconciliationLock(
 	})
 }
 
+// executeReconciliationAction executes a reconciliation action on the given preset snapshot.
+//
+// The action can be of different types (create, delete, backoff), and may internally include
+// multiple items to process, for example, a delete action can contain multiple prebuild IDs to delete,
+// and a create action includes a count of prebuilds to create.
+//
+// This method handles logging at appropriate levels and performs the necessary operations
+// according to the action type. It returns an error if any part of the action fails.
+func (c *StoreReconciler) executeReconciliationAction(ctx context.Context, logger slog.Logger, ps prebuilds.PresetSnapshot, action *prebuilds.ReconciliationActions) error {
+	levelFn := logger.Debug
+
+	// Nothing has to be done.
+	if !ps.Preset.UsingActiveVersion && action.IsNoop() {
+		logger.Debug(ctx, "skipping reconciliation for preset - nothing has to be done",
+			slog.F("template_id", ps.Preset.TemplateID.String()), slog.F("template_name", ps.Preset.TemplateName),
+			slog.F("template_version_id", ps.Preset.TemplateVersionID.String()), slog.F("template_version_name", ps.Preset.TemplateVersionName),
+			slog.F("preset_id", ps.Preset.ID.String()), slog.F("preset_name", ps.Preset.Name))
+		return nil
+	}
+
+	// nolint:gocritic // ReconcilePreset needs Prebuilds Orchestrator permissions.
+	prebuildsCtx := dbauthz.AsPrebuildsOrchestrator(ctx)
+
+	fields := []any{
+		slog.F("action_type", action.ActionType), slog.F("create_count", action.Create),
+		slog.F("delete_count", len(action.DeleteIDs)), slog.F("to_delete", action.DeleteIDs),
+	}
+	levelFn(ctx, "calculated reconciliation action for preset", fields...)
+
+	switch {
+	case action.ActionType == prebuilds.ActionTypeBackoff:
+		levelFn = logger.Warn
+	// Log at info level when there's a change to be effected.
+	case action.ActionType == prebuilds.ActionTypeCreate && action.Create > 0:
+		levelFn = logger.Info
+	case action.ActionType == prebuilds.ActionTypeDelete && len(action.DeleteIDs) > 0:
+		levelFn = logger.Info
+	}
+
+	switch action.ActionType {
+	case prebuilds.ActionTypeBackoff:
+		// If there is anything to backoff for (usually a cycle of failed prebuilds), then log and bail out.
+		levelFn(ctx, "template prebuild state retrieved, backing off",
+			append(fields,
+				slog.F("backoff_until", action.BackoffUntil.Format(time.RFC3339)),
+				slog.F("backoff_secs", math.Round(action.BackoffUntil.Sub(c.clock.Now()).Seconds())),
+			)...)
+
+		return nil
+
+	case prebuilds.ActionTypeCreate:
+		// Unexpected things happen (i.e. bugs or bitflips); let's defend against disastrous outcomes.
+		// See https://blog.robertelder.org/causes-of-bit-flips-in-computer-memory/.
+		// This is obviously not comprehensive protection against this sort of problem, but this is one essential check.
+		desired := ps.Preset.DesiredInstances.Int32
+		if action.Create > desired {
+			logger.Critical(ctx, "determined excessive count of prebuilds to create; clamping to desired count",
+				slog.F("create_count", action.Create), slog.F("desired_count", desired))
+
+			action.Create = desired
+		}
+
+		// If preset is hard-limited, and it's a create operation, log it and exit early.
+		// Creation operation is disallowed for hard-limited preset.
+		if ps.IsHardLimited && action.Create > 0 {
+			logger.Warn(ctx, "skipping hard limited preset for create operation")
+			return nil
+		}
+
+		var multiErr multierror.Error
+		for range action.Create {
+			if err := c.createPrebuiltWorkspace(prebuildsCtx, uuid.New(), ps.Preset.TemplateID, ps.Preset.ID); err != nil {
+				logger.Error(ctx, "failed to create prebuild", slog.Error(err))
+				multiErr.Errors = append(multiErr.Errors, err)
+			}
+		}
+
+		return multiErr.ErrorOrNil()
+
+	case prebuilds.ActionTypeDelete:
+		var multiErr multierror.Error
+		for _, id := range action.DeleteIDs {
+			if err := c.deletePrebuiltWorkspace(prebuildsCtx, id, ps.Preset.TemplateID, ps.Preset.ID); err != nil {
+				logger.Error(ctx, "failed to delete prebuild", slog.Error(err))
+				multiErr.Errors = append(multiErr.Errors, err)
+			}
+		}
+
+		return multiErr.ErrorOrNil()
+
+	default:
+		return xerrors.Errorf("unknown action type: %v", action.ActionType)
+	}
+}
+
 func (c *StoreReconciler) createPrebuiltWorkspace(ctx context.Context, prebuiltWorkspaceID uuid.UUID, templateID uuid.UUID, presetID uuid.UUID) error {
 	name, err := prebuilds.GenerateName()
 	if err != nil {

enterprise/coderd/prebuilds/reconcile_test.go

@@ -1226,17 +1226,18 @@ func TestFailedBuildBackoff(t *testing.T) {
 	state := presetState.CalculateState()
 	actions, err := reconciler.CalculateActions(ctx, *presetState)
 	require.NoError(t, err)
+	require.Equal(t, 1, len(actions))
 
 	// Then: the backoff time is in the future, no prebuilds are running, and we won't create any new prebuilds.
 	require.EqualValues(t, 0, state.Actual)
-	require.EqualValues(t, 0, actions.Create)
+	require.EqualValues(t, 0, actions[0].Create)
 	require.EqualValues(t, desiredInstances, state.Desired)
-	require.True(t, clock.Now().Before(actions.BackoffUntil))
+	require.True(t, clock.Now().Before(actions[0].BackoffUntil))
 
 	// Then: the backoff time is as expected based on the number of failed builds.
 	require.NotNil(t, presetState.Backoff)
 	require.EqualValues(t, desiredInstances, presetState.Backoff.NumFailed)
-	require.EqualValues(t, backoffInterval*time.Duration(presetState.Backoff.NumFailed), clock.Until(actions.BackoffUntil).Truncate(backoffInterval))
+	require.EqualValues(t, backoffInterval*time.Duration(presetState.Backoff.NumFailed), clock.Until(actions[0].BackoffUntil).Truncate(backoffInterval))
 
 	// When: advancing to the next tick which is still within the backoff time.
 	clock.Advance(cfg.ReconciliationInterval.Value())
@@ -1249,13 +1250,15 @@ func TestFailedBuildBackoff(t *testing.T) {
 	newState := presetState.CalculateState()
 	newActions, err := reconciler.CalculateActions(ctx, *presetState)
 	require.NoError(t, err)
+	require.Equal(t, 1, len(newActions))
+
 	require.EqualValues(t, 0, newState.Actual)
-	require.EqualValues(t, 0, newActions.Create)
+	require.EqualValues(t, 0, newActions[0].Create)
 	require.EqualValues(t, desiredInstances, newState.Desired)
-	require.EqualValues(t, actions.BackoffUntil, newActions.BackoffUntil)
+	require.EqualValues(t, actions[0].BackoffUntil, newActions[0].BackoffUntil)
 
 	// When: advancing beyond the backoff time.
-	clock.Advance(clock.Until(actions.BackoffUntil.Add(time.Second)))
+	clock.Advance(clock.Until(actions[0].BackoffUntil.Add(time.Second)))
 
 	// Then: we will attempt to create a new prebuild.
 	snapshot, err = reconciler.SnapshotState(ctx, db)
@@ -1265,9 +1268,11 @@ func TestFailedBuildBackoff(t *testing.T) {
 	state = presetState.CalculateState()
 	actions, err = reconciler.CalculateActions(ctx, *presetState)
 	require.NoError(t, err)
+	require.Equal(t, 1, len(actions))
+
 	require.EqualValues(t, 0, state.Actual)
 	require.EqualValues(t, desiredInstances, state.Desired)
-	require.EqualValues(t, desiredInstances, actions.Create)
+	require.EqualValues(t, desiredInstances, actions[0].Create)
 
 	// When: the desired number of new prebuild are provisioned, but one fails again.
 	for i := 0; i < desiredInstances; i++ {
@@ -1286,11 +1291,13 @@ func TestFailedBuildBackoff(t *testing.T) {
 	state = presetState.CalculateState()
 	actions, err = reconciler.CalculateActions(ctx, *presetState)
 	require.NoError(t, err)
+	require.Equal(t, 1, len(actions))
+
 	require.EqualValues(t, 1, state.Actual)
 	require.EqualValues(t, desiredInstances, state.Desired)
-	require.EqualValues(t, 0, actions.Create)
+	require.EqualValues(t, 0, actions[0].Create)
 	require.EqualValues(t, 3, presetState.Backoff.NumFailed)
-	require.EqualValues(t, backoffInterval*time.Duration(presetState.Backoff.NumFailed), clock.Until(actions.BackoffUntil).Truncate(backoffInterval))
+	require.EqualValues(t, backoffInterval*time.Duration(presetState.Backoff.NumFailed), clock.Until(actions[0].BackoffUntil).Truncate(backoffInterval))
 }
 
 func TestReconciliationLock(t *testing.T) {

provisioner/terraform/resources.go

@@ -897,14 +897,21 @@ func ConvertState(ctx context.Context, modules []*tfjson.StateModule, rawGraph s
 			)
 		}
 		var prebuildInstances int32
+		var expirationPolicy *proto.ExpirationPolicy
 		if len(preset.Prebuilds) > 0 {
 			prebuildInstances = int32(math.Min(math.MaxInt32, float64(preset.Prebuilds[0].Instances)))
+			if len(preset.Prebuilds[0].ExpirationPolicy) > 0 {
+				expirationPolicy = &proto.ExpirationPolicy{
+					Ttl: int32(math.Min(math.MaxInt32, float64(preset.Prebuilds[0].ExpirationPolicy[0].TTL))),
+				}
+			}
 		}
 		protoPreset := &proto.Preset{
 			Name:       preset.Name,
 			Parameters: presetParameters,
 			Prebuild: &proto.Prebuild{
 				Instances:        prebuildInstances,
+				ExpirationPolicy: expirationPolicy,
 			},
 		}
 

provisioner/terraform/resources_test.go

@@ -786,6 +786,7 @@ func TestConvertResources(t *testing.T) {
 					Name:                     "dev",
 					OperatingSystem:          "windows",
 					Architecture:             "arm64",
+					ApiKeyScope:              "all",
 					Auth:                     &proto.Agent_Token{},
 					ConnectionTimeoutSeconds: 120,
 					DisplayApps:              &displayApps,
@@ -830,6 +831,9 @@ func TestConvertResources(t *testing.T) {
 				}},
 				Prebuild: &proto.Prebuild{
 					Instances: 4,
+					ExpirationPolicy: &proto.ExpirationPolicy{
+						Ttl: 86400,
+					},
 				},
 			}},
 		},

provisioner/terraform/testdata/resources/presets/presets.tf

@@ -25,6 +25,9 @@ data "coder_workspace_preset" "MyFirstProject" {
   }
   prebuilds {
     instances = 4
+    expiration_policy {
+      ttl = 86400
+    }
   }
 }
 

provisioner/terraform/testdata/resources/presets/presets.tfplan.json

@@ -12,6 +12,7 @@
           "provider_name": "registry.terraform.io/coder/coder",
           "schema_version": 1,
           "values": {
+            "api_key_scope": "all",
             "arch": "arm64",
             "auth": "token",
             "connection_timeout": 120,
@@ -62,6 +63,7 @@
         ],
         "before": null,
         "after": {
+          "api_key_scope": "all",
           "arch": "arm64",
           "auth": "token",
           "connection_timeout": 120,
@@ -134,6 +136,7 @@
               "description": "blah blah",
               "display_name": null,
               "ephemeral": false,
+              "form_type": "input",
               "icon": null,
               "id": "57ccea62-8edf-41d1-a2c1-33f365e27567",
               "mutable": false,
@@ -141,6 +144,7 @@
               "option": null,
               "optional": true,
               "order": null,
+              "styling": "{}",
               "type": "string",
               "validation": [],
               "value": "ok"
@@ -164,6 +168,11 @@
               },
               "prebuilds": [
                 {
+                  "expiration_policy": [
+                    {
+                      "ttl": 86400
+                    }
+                  ],
                   "instances": 4
                 }
               ]
@@ -171,9 +180,13 @@
             "sensitive_values": {
               "parameters": {},
               "prebuilds": [
+                {
+                  "expiration_policy": [
                     {}
                   ]
                 }
+              ]
+            }
           }
         ],
         "child_modules": [
@@ -191,6 +204,7 @@
                   "description": "First parameter from module",
                   "display_name": null,
                   "ephemeral": false,
+                  "form_type": "input",
                   "icon": null,
                   "id": "1774175f-0efd-4a79-8d40-dbbc559bf7c1",
                   "mutable": true,
@@ -198,6 +212,7 @@
                   "option": null,
                   "optional": true,
                   "order": null,
+                  "styling": "{}",
                   "type": "string",
                   "validation": [],
                   "value": "abcdef"
@@ -218,6 +233,7 @@
                   "description": "Second parameter from module",
                   "display_name": null,
                   "ephemeral": false,
+                  "form_type": "input",
                   "icon": null,
                   "id": "23d6841f-bb95-42bb-b7ea-5b254ce6c37d",
                   "mutable": true,
@@ -225,6 +241,7 @@
                   "option": null,
                   "optional": true,
                   "order": null,
+                  "styling": "{}",
                   "type": "string",
                   "validation": [],
                   "value": "ghijkl"
@@ -250,6 +267,7 @@
                       "description": "First parameter from child module",
                       "display_name": null,
                       "ephemeral": false,
+                      "form_type": "input",
                       "icon": null,
                       "id": "9d629df2-9846-47b2-ab1f-e7c882f35117",
                       "mutable": true,
@@ -257,6 +275,7 @@
                       "option": null,
                       "optional": true,
                       "order": null,
+                      "styling": "{}",
                       "type": "string",
                       "validation": [],
                       "value": "abcdef"
@@ -277,6 +296,7 @@
                       "description": "Second parameter from child module",
                       "display_name": null,
                       "ephemeral": false,
+                      "form_type": "input",
                       "icon": null,
                       "id": "52ca7b77-42a1-4887-a2f5-7a728feebdd5",
                       "mutable": true,
@@ -284,6 +304,7 @@
                       "option": null,
                       "optional": true,
                       "order": null,
+                      "styling": "{}",
                       "type": "string",
                       "validation": [],
                       "value": "ghijkl"
@@ -388,6 +409,13 @@
             },
             "prebuilds": [
               {
+                "expiration_policy": [
+                  {
+                    "ttl": {
+                      "constant_value": 86400
+                    }
+                  }
+                ],
                 "instances": {
                   "constant_value": 4
                 }

provisioner/terraform/testdata/resources/presets/presets.tfstate.json

@@ -16,6 +16,7 @@
             "description": "blah blah",
             "display_name": null,
             "ephemeral": false,
+            "form_type": "input",
             "icon": null,
             "id": "491d202d-5658-40d9-9adc-fd3a67f6042b",
             "mutable": false,
@@ -23,6 +24,7 @@
             "option": null,
             "optional": true,
             "order": null,
+            "styling": "{}",
             "type": "string",
             "validation": [],
             "value": "ok"
@@ -46,6 +48,11 @@
             },
             "prebuilds": [
               {
+                "expiration_policy": [
+                  {
+                    "ttl": 86400
+                  }
+                ],
                 "instances": 4
               }
             ]
@@ -53,9 +60,13 @@
           "sensitive_values": {
             "parameters": {},
             "prebuilds": [
+              {
+                "expiration_policy": [
                   {}
                 ]
               }
+            ]
+          }
         },
         {
           "address": "coder_agent.dev",
@@ -65,6 +76,7 @@
           "provider_name": "registry.terraform.io/coder/coder",
           "schema_version": 1,
           "values": {
+            "api_key_scope": "all",
             "arch": "arm64",
             "auth": "token",
             "connection_timeout": 120,
@@ -133,6 +145,7 @@
                 "description": "First parameter from module",
                 "display_name": null,
                 "ephemeral": false,
+                "form_type": "input",
                 "icon": null,
                 "id": "0a4d1299-b174-43b0-91ad-50c1ca9a4c25",
                 "mutable": true,
@@ -140,6 +153,7 @@
                 "option": null,
                 "optional": true,
                 "order": null,
+                "styling": "{}",
                 "type": "string",
                 "validation": [],
                 "value": "abcdef"
@@ -160,6 +174,7 @@
                 "description": "Second parameter from module",
                 "display_name": null,
                 "ephemeral": false,
+                "form_type": "input",
                 "icon": null,
                 "id": "f0812474-29fd-4c3c-ab40-9e66e36d4017",
                 "mutable": true,
@@ -167,6 +182,7 @@
                 "option": null,
                 "optional": true,
                 "order": null,
+                "styling": "{}",
                 "type": "string",
                 "validation": [],
                 "value": "ghijkl"
@@ -192,6 +208,7 @@
                     "description": "First parameter from child module",
                     "display_name": null,
                     "ephemeral": false,
+                    "form_type": "input",
                     "icon": null,
                     "id": "27b5fae3-7671-4e61-bdfe-c940627a21b8",
                     "mutable": true,
@@ -199,6 +216,7 @@
                     "option": null,
                     "optional": true,
                     "order": null,
+                    "styling": "{}",
                     "type": "string",
                     "validation": [],
                     "value": "abcdef"
@@ -219,6 +237,7 @@
                     "description": "Second parameter from child module",
                     "display_name": null,
                     "ephemeral": false,
+                    "form_type": "input",
                     "icon": null,
                     "id": "d285bb17-27ff-4a49-a12b-28582264b4d9",
                     "mutable": true,
@@ -226,6 +245,7 @@
                     "option": null,
                     "optional": true,
                     "order": null,
+                    "styling": "{}",
                     "type": "string",
                     "validation": [],
                     "value": "ghijkl"

provisionerd/proto/version.go

@@ -25,6 +25,8 @@ import "github.com/coder/coder/v2/apiversion"
 //   - Add previous parameter values to 'WorkspaceBuild' jobs. Provisioner passes
 //     the previous values for the `terraform apply` to enforce monotonicity
 //     in the terraform provider.
+//   - Add new field named `expiration_policy` to `Prebuild`, with a field named
+//     `ttl` to define TTL-based expiration for unclaimed prebuilds.
 const (
 	CurrentMajor = 1
 	CurrentMinor = 6

provisionersdk/proto/provisioner.proto

@@ -57,8 +57,16 @@ message RichParameterValue {
     string value = 2;
 }
 
+// ExpirationPolicy defines the policy for expiring unclaimed prebuilds.
+// If a prebuild remains unclaimed for longer than ttl seconds, it is deleted and
+// recreated to prevent staleness.
+message ExpirationPolicy {
+	int32 ttl = 1;
+}
+
 message Prebuild {
     int32 instances = 1;
+	ExpirationPolicy expiration_policy = 2;
 }
 
 // Preset represents a set of preset parameters for a template version.

site/e2e/provisionerGenerated.ts

@@ -104,8 +104,18 @@ export interface RichParameterValue {
   value: string;
 }
 
+/**
+ * ExpirationPolicy defines the policy for expiring unclaimed prebuilds.
+ * If a prebuild remains unclaimed for longer than ttl seconds, it is deleted and
+ * recreated to prevent staleness.
+ */
+export interface ExpirationPolicy {
+  ttl: number;
+}
+
 export interface Prebuild {
   instances: number;
+  expirationPolicy: ExpirationPolicy | undefined;
 }
 
 /** Preset represents a set of preset parameters for a template version. */
@@ -544,11 +554,23 @@ export const RichParameterValue = {
   },
 };
 
+export const ExpirationPolicy = {
+  encode(message: ExpirationPolicy, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.ttl !== 0) {
+      writer.uint32(8).int32(message.ttl);
+    }
+    return writer;
+  },
+};
+
 export const Prebuild = {
   encode(message: Prebuild, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
     if (message.instances !== 0) {
       writer.uint32(8).int32(message.instances);
     }
+    if (message.expirationPolicy !== undefined) {
+      ExpirationPolicy.encode(message.expirationPolicy, writer.uint32(18).fork()).ldelim();
+    }
     return writer;
   },
 };