feat: implement expiration policy logic for prebuilds (#17996)

## Summary 

This PR introduces support for expiration policies in prebuilds. The TTL
(time-to-live) is retrieved from the Terraform configuration
([terraform-provider-coder
PR](https://github.com/coder/terraform-provider-coder/pull/404)):
```
prebuilds = {
	  instances = 2
	  expiration_policy {
		  ttl = 86400
	  }
  }
```
**Note**: Since there is no need for precise TTL enforcement down to the
second, in this implementation expired prebuilds are handled in a single
reconciliation cycle: they are deleted, and new instances are created
only if needed to match the desired count.

## Changes

* The outcome of a reconciliation cycle is now expressed as a slice of
reconciliation actions, instead of a single aggregated action.
* Adjusted reconciliation logic to delete expired prebuilds and
guarantee that the number of desired instances is correct.
* Updated relevant data structures and methods to support expiration
policies parameters.
* Added documentation to `Prebuilt workspaces` page
* Update `terraform-provider-coder` to version 2.5.0:
https://github.com/coder/terraform-provider-coder/releases/tag/v2.5.0

Depends on: https://github.com/coder/terraform-provider-coder/pull/404
Fixes: https://github.com/coder/coder/issues/17916
This commit is contained in:
Susana Ferreira
2025-05-26 20:31:24 +01:00
committed by GitHub
parent 589f18627e
commit 6f6e73af03
18 changed files with 1503 additions and 994 deletions

View File

@ -6511,6 +6511,7 @@ SELECT
tvp.id,
tvp.name,
tvp.desired_instances AS desired_instances,
tvp.invalidate_after_secs AS ttl,
tvp.prebuild_status,
t.deleted,
t.deprecated != '' AS deprecated
@ -6534,6 +6535,7 @@ type GetTemplatePresetsWithPrebuildsRow struct {
ID uuid.UUID `db:"id" json:"id"`
Name string `db:"name" json:"name"`
DesiredInstances sql.NullInt32 `db:"desired_instances" json:"desired_instances"`
Ttl sql.NullInt32 `db:"ttl" json:"ttl"`
PrebuildStatus PrebuildStatus `db:"prebuild_status" json:"prebuild_status"`
Deleted bool `db:"deleted" json:"deleted"`
Deprecated bool `db:"deprecated" json:"deprecated"`
@ -6562,6 +6564,7 @@ func (q *sqlQuerier) GetTemplatePresetsWithPrebuilds(ctx context.Context, templa
&i.ID,
&i.Name,
&i.DesiredInstances,
&i.Ttl,
&i.PrebuildStatus,
&i.Deleted,
&i.Deprecated,

View File

@ -35,6 +35,7 @@ SELECT
tvp.id,
tvp.name,
tvp.desired_instances AS desired_instances,
tvp.invalidate_after_secs AS ttl,
tvp.prebuild_status,
t.deleted,
t.deprecated != '' AS deprecated

View File

@ -1,6 +1,8 @@
package prebuilds
import (
"time"
"github.com/google/uuid"
"golang.org/x/xerrors"
@ -41,6 +43,7 @@ func (s GlobalSnapshot) FilterByPreset(presetID uuid.UUID) (*PresetSnapshot, err
return nil, xerrors.Errorf("no preset found with ID %q", presetID)
}
// Only include workspaces that have successfully started
running := slice.Filter(s.RunningPrebuilds, func(prebuild database.GetRunningPrebuiltWorkspacesRow) bool {
if !prebuild.CurrentPresetID.Valid {
return false
@ -48,6 +51,9 @@ func (s GlobalSnapshot) FilterByPreset(presetID uuid.UUID) (*PresetSnapshot, err
return prebuild.CurrentPresetID.UUID == preset.ID
})
// Separate running workspaces into non-expired and expired based on the preset's TTL
nonExpired, expired := filterExpiredWorkspaces(preset, running)
inProgress := slice.Filter(s.PrebuildsInProgress, func(prebuild database.CountInProgressPrebuildsRow) bool {
return prebuild.PresetID.UUID == preset.ID
})
@ -66,9 +72,33 @@ func (s GlobalSnapshot) FilterByPreset(presetID uuid.UUID) (*PresetSnapshot, err
return &PresetSnapshot{
Preset: preset,
Running: running,
Running: nonExpired,
Expired: expired,
InProgress: inProgress,
Backoff: backoffPtr,
IsHardLimited: isHardLimited,
}, nil
}
// filterExpiredWorkspaces splits running workspaces into expired and non-expired
// based on the preset's TTL.
// If TTL is missing or zero, all workspaces are considered non-expired.
func filterExpiredWorkspaces(preset database.GetTemplatePresetsWithPrebuildsRow, runningWorkspaces []database.GetRunningPrebuiltWorkspacesRow) (nonExpired []database.GetRunningPrebuiltWorkspacesRow, expired []database.GetRunningPrebuiltWorkspacesRow) {
if !preset.Ttl.Valid {
return runningWorkspaces, expired
}
ttl := time.Duration(preset.Ttl.Int32) * time.Second
if ttl <= 0 {
return runningWorkspaces, expired
}
for _, prebuild := range runningWorkspaces {
if time.Since(prebuild.CreatedAt) > ttl {
expired = append(expired, prebuild)
} else {
nonExpired = append(nonExpired, prebuild)
}
}
return nonExpired, expired
}

View File

@ -31,9 +31,14 @@ const (
// PresetSnapshot is a filtered view of GlobalSnapshot focused on a single preset.
// It contains the raw data needed to calculate the current state of a preset's prebuilds,
// including running prebuilds, in-progress builds, and backoff information.
// - Running: prebuilds running and non-expired
// - Expired: prebuilds running and expired due to the preset's TTL
// - InProgress: prebuilds currently in progress
// - Backoff: holds failure info to decide if prebuild creation should be backed off
type PresetSnapshot struct {
Preset database.GetTemplatePresetsWithPrebuildsRow
Running []database.GetRunningPrebuiltWorkspacesRow
Expired []database.GetRunningPrebuiltWorkspacesRow
InProgress []database.CountInProgressPrebuildsRow
Backoff *database.GetPresetsBackoffRow
IsHardLimited bool
@ -43,10 +48,11 @@ type PresetSnapshot struct {
// calculated from a PresetSnapshot. While PresetSnapshot contains raw data,
// ReconciliationState contains derived metrics that are directly used to
// determine what actions are needed (create, delete, or backoff).
// For example, it calculates how many prebuilds are eligible, how many are
// extraneous, and how many are in various transition states.
// For example, it calculates how many prebuilds are expired, eligible,
// how many are extraneous, and how many are in various transition states.
type ReconciliationState struct {
Actual int32 // Number of currently running prebuilds
Actual int32 // Number of currently running prebuilds, i.e., non-expired, expired and extraneous prebuilds
Expired int32 // Number of currently running prebuilds that exceeded their allowed time-to-live (TTL)
Desired int32 // Number of prebuilds desired as defined in the preset
Eligible int32 // Number of prebuilds that are ready to be claimed
Extraneous int32 // Number of extra running prebuilds beyond the desired count
@ -78,7 +84,8 @@ func (ra *ReconciliationActions) IsNoop() bool {
}
// CalculateState computes the current state of prebuilds for a preset, including:
// - Actual: Number of currently running prebuilds
// - Actual: Number of currently running prebuilds, i.e., non-expired and expired prebuilds
// - Expired: Number of currently running expired prebuilds
// - Desired: Number of prebuilds desired as defined in the preset
// - Eligible: Number of prebuilds that are ready to be claimed
// - Extraneous: Number of extra running prebuilds beyond the desired count
@ -92,23 +99,28 @@ func (p PresetSnapshot) CalculateState() *ReconciliationState {
var (
actual int32
desired int32
expired int32
eligible int32
extraneous int32
)
// #nosec G115 - Safe conversion as p.Running slice length is expected to be within int32 range
actual = int32(len(p.Running))
// #nosec G115 - Safe conversion as p.Running and p.Expired slice length is expected to be within int32 range
actual = int32(len(p.Running) + len(p.Expired))
// #nosec G115 - Safe conversion as p.Expired slice length is expected to be within int32 range
expired = int32(len(p.Expired))
if p.isActive() {
desired = p.Preset.DesiredInstances.Int32
eligible = p.countEligible()
extraneous = max(actual-desired, 0)
extraneous = max(actual-expired-desired, 0)
}
starting, stopping, deleting := p.countInProgress()
return &ReconciliationState{
Actual: actual,
Expired: expired,
Desired: desired,
Eligible: eligible,
Extraneous: extraneous,
@ -126,6 +138,7 @@ func (p PresetSnapshot) CalculateState() *ReconciliationState {
// 3. For active presets, it calculates the number of prebuilds to create or delete based on:
// - The desired number of instances
// - Currently running prebuilds
// - Currently running expired prebuilds
// - Prebuilds in transition states (starting/stopping/deleting)
// - Any extraneous prebuilds that need to be removed
//
@ -133,7 +146,7 @@ func (p PresetSnapshot) CalculateState() *ReconciliationState {
// - ActionTypeBackoff: Only BackoffUntil is set, indicating when to retry
// - ActionTypeCreate: Only Create is set, indicating how many prebuilds to create
// - ActionTypeDelete: Only DeleteIDs is set, containing IDs of prebuilds to delete
func (p PresetSnapshot) CalculateActions(clock quartz.Clock, backoffInterval time.Duration) (*ReconciliationActions, error) {
func (p PresetSnapshot) CalculateActions(clock quartz.Clock, backoffInterval time.Duration) ([]*ReconciliationActions, error) {
// TODO: align workspace states with how we represent them on the FE and the CLI
// right now there's some slight differences which can lead to additional prebuilds being created
@ -158,45 +171,77 @@ func (p PresetSnapshot) isActive() bool {
return p.Preset.UsingActiveVersion && !p.Preset.Deleted && !p.Preset.Deprecated
}
// handleActiveTemplateVersion deletes excess prebuilds if there are too many,
// otherwise creates new ones to reach the desired count.
func (p PresetSnapshot) handleActiveTemplateVersion() (*ReconciliationActions, error) {
// handleActiveTemplateVersion determines the reconciliation actions for a preset with an active template version.
// It ensures the system moves towards the desired number of healthy prebuilds.
//
// The reconciliation follows this order:
// 1. Delete expired prebuilds: These are no longer valid and must be removed first.
// 2. Delete extraneous prebuilds: After expired ones are removed, if the number of running non-expired prebuilds
// still exceeds the desired count, the oldest prebuilds are deleted to reduce excess.
// 3. Create missing prebuilds: If the number of non-expired, non-starting prebuilds is still below the desired count,
// create the necessary number of prebuilds to reach the target.
//
// The function returns a list of actions to be executed to achieve the desired state.
func (p PresetSnapshot) handleActiveTemplateVersion() (actions []*ReconciliationActions, err error) {
state := p.CalculateState()
// If we have more prebuilds than desired, delete the oldest ones
if state.Extraneous > 0 {
return &ReconciliationActions{
ActionType: ActionTypeDelete,
DeleteIDs: p.getOldestPrebuildIDs(int(state.Extraneous)),
}, nil
// If we have expired prebuilds, delete them
if state.Expired > 0 {
var deleteIDs []uuid.UUID
for _, expired := range p.Expired {
deleteIDs = append(deleteIDs, expired.ID)
}
actions = append(actions,
&ReconciliationActions{
ActionType: ActionTypeDelete,
DeleteIDs: deleteIDs,
})
}
// If we still have more prebuilds than desired, delete the oldest ones
if state.Extraneous > 0 {
actions = append(actions,
&ReconciliationActions{
ActionType: ActionTypeDelete,
DeleteIDs: p.getOldestPrebuildIDs(int(state.Extraneous)),
})
}
// Number of running prebuilds excluding the recently deleted Expired
runningValid := state.Actual - state.Expired
// Calculate how many new prebuilds we need to create
// We subtract starting prebuilds since they're already being created
prebuildsToCreate := max(state.Desired-state.Actual-state.Starting, 0)
prebuildsToCreate := max(state.Desired-runningValid-state.Starting, 0)
if prebuildsToCreate > 0 {
actions = append(actions,
&ReconciliationActions{
ActionType: ActionTypeCreate,
Create: prebuildsToCreate,
})
}
return &ReconciliationActions{
ActionType: ActionTypeCreate,
Create: prebuildsToCreate,
}, nil
return actions, nil
}
// handleInactiveTemplateVersion deletes all running prebuilds except those already being deleted
// to avoid duplicate deletion attempts.
func (p PresetSnapshot) handleInactiveTemplateVersion() (*ReconciliationActions, error) {
func (p PresetSnapshot) handleInactiveTemplateVersion() ([]*ReconciliationActions, error) {
prebuildsToDelete := len(p.Running)
deleteIDs := p.getOldestPrebuildIDs(prebuildsToDelete)
return &ReconciliationActions{
ActionType: ActionTypeDelete,
DeleteIDs: deleteIDs,
return []*ReconciliationActions{
{
ActionType: ActionTypeDelete,
DeleteIDs: deleteIDs,
},
}, nil
}
// needsBackoffPeriod checks if we should delay prebuild creation due to recent failures.
// If there were failures, it calculates a backoff period based on the number of failures
// and returns true if we're still within that period.
func (p PresetSnapshot) needsBackoffPeriod(clock quartz.Clock, backoffInterval time.Duration) (*ReconciliationActions, bool) {
func (p PresetSnapshot) needsBackoffPeriod(clock quartz.Clock, backoffInterval time.Duration) ([]*ReconciliationActions, bool) {
if p.Backoff == nil || p.Backoff.NumFailed == 0 {
return nil, false
}
@ -205,9 +250,11 @@ func (p PresetSnapshot) needsBackoffPeriod(clock quartz.Clock, backoffInterval t
return nil, false
}
return &ReconciliationActions{
ActionType: ActionTypeBackoff,
BackoffUntil: backoffUntil,
return []*ReconciliationActions{
{
ActionType: ActionTypeBackoff,
BackoffUntil: backoffUntil,
},
}, true
}

View File

@ -23,6 +23,7 @@ type options struct {
presetName string
prebuiltWorkspaceID uuid.UUID
workspaceName string
ttl int32
}
// templateID is common across all option sets.
@ -34,6 +35,7 @@ const (
optionSet0 = iota
optionSet1
optionSet2
optionSet3
)
var opts = map[uint]options{
@ -61,6 +63,15 @@ var opts = map[uint]options{
prebuiltWorkspaceID: uuid.UUID{33},
workspaceName: "prebuilds2",
},
optionSet3: {
templateID: templateID,
templateVersionID: uuid.UUID{41},
presetID: uuid.UUID{42},
presetName: "my-preset",
prebuiltWorkspaceID: uuid.UUID{43},
workspaceName: "prebuilds3",
ttl: 5, // seconds
},
}
// A new template version with a preset without prebuilds configured should result in no prebuilds being created.
@ -82,10 +93,7 @@ func TestNoPrebuilds(t *testing.T) {
require.NoError(t, err)
validateState(t, prebuilds.ReconciliationState{ /*all zero values*/ }, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 0,
}, *actions)
validateActions(t, nil, actions)
}
// A new template version with a preset with prebuilds configured should result in a new prebuild being created.
@ -109,10 +117,12 @@ func TestNetNew(t *testing.T) {
validateState(t, prebuilds.ReconciliationState{
Desired: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}, actions)
}
// A new template version is created with a preset with prebuilds configured; this outdates the older version and
@ -149,10 +159,12 @@ func TestOutdatedPrebuilds(t *testing.T) {
validateState(t, prebuilds.ReconciliationState{
Actual: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{outdated.prebuiltWorkspaceID},
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{outdated.prebuiltWorkspaceID},
},
}, actions)
// WHEN: calculating the current preset's state.
ps, err = snapshot.FilterByPreset(current.presetID)
@ -163,10 +175,12 @@ func TestOutdatedPrebuilds(t *testing.T) {
actions, err = ps.CalculateActions(clock, backoffInterval)
require.NoError(t, err)
validateState(t, prebuilds.ReconciliationState{Desired: 1}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}, actions)
}
// Make sure that outdated prebuild will be deleted, even if deletion of another outdated prebuild is already in progress.
@ -214,10 +228,12 @@ func TestDeleteOutdatedPrebuilds(t *testing.T) {
Deleting: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{outdated.prebuiltWorkspaceID},
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{outdated.prebuiltWorkspaceID},
},
}, actions)
}
// A new template version is created with a preset with prebuilds configured; while a prebuild is provisioning up or down,
@ -233,7 +249,7 @@ func TestInProgressActions(t *testing.T) {
desired int32
running int32
inProgress int32
checkFn func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions)
checkFn func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions)
}{
// With no running prebuilds and one starting, no creations/deletions should take place.
{
@ -242,11 +258,9 @@ func TestInProgressActions(t *testing.T) {
desired: 1,
running: 0,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Desired: 1, Starting: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
}, actions)
validateActions(t, nil, actions)
},
},
// With one running prebuild and one starting, no creations/deletions should occur since we're approaching the correct state.
@ -256,11 +270,9 @@ func TestInProgressActions(t *testing.T) {
desired: 2,
running: 1,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 1, Desired: 2, Starting: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
}, actions)
validateActions(t, nil, actions)
},
},
// With one running prebuild and one starting, no creations/deletions should occur
@ -271,11 +283,9 @@ func TestInProgressActions(t *testing.T) {
desired: 2,
running: 2,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 2, Desired: 2, Starting: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
}, actions)
validateActions(t, nil, actions)
},
},
// With one prebuild desired and one stopping, a new prebuild will be created.
@ -285,11 +295,13 @@ func TestInProgressActions(t *testing.T) {
desired: 1,
running: 0,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Desired: 1, Stopping: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}, actions)
},
},
@ -300,11 +312,13 @@ func TestInProgressActions(t *testing.T) {
desired: 3,
running: 2,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 2, Desired: 3, Stopping: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}, actions)
},
},
@ -315,11 +329,9 @@ func TestInProgressActions(t *testing.T) {
desired: 3,
running: 3,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 3, Desired: 3, Stopping: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
}, actions)
validateActions(t, nil, actions)
},
},
// With one prebuild desired and one deleting, a new prebuild will be created.
@ -329,11 +341,13 @@ func TestInProgressActions(t *testing.T) {
desired: 1,
running: 0,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Desired: 1, Deleting: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}, actions)
},
},
@ -344,11 +358,13 @@ func TestInProgressActions(t *testing.T) {
desired: 2,
running: 1,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 1, Desired: 2, Deleting: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}, actions)
},
},
@ -359,11 +375,9 @@ func TestInProgressActions(t *testing.T) {
desired: 2,
running: 2,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 2, Desired: 2, Deleting: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
}, actions)
validateActions(t, nil, actions)
},
},
// With 3 prebuilds desired, 1 running, and 2 starting, no creations should occur since the builds are in progress.
@ -373,9 +387,9 @@ func TestInProgressActions(t *testing.T) {
desired: 3,
running: 1,
inProgress: 2,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 1, Desired: 3, Starting: 2}, state)
validateActions(t, prebuilds.ReconciliationActions{ActionType: prebuilds.ActionTypeCreate, Create: 0}, actions)
validateActions(t, nil, actions)
},
},
// With 3 prebuilds desired, 5 running, and 2 deleting, no deletions should occur since the builds are in progress.
@ -385,17 +399,20 @@ func TestInProgressActions(t *testing.T) {
desired: 3,
running: 5,
inProgress: 2,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
expectedState := prebuilds.ReconciliationState{Actual: 5, Desired: 3, Deleting: 2, Extraneous: 2}
expectedActions := prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeDelete,
expectedActions := []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
},
}
validateState(t, expectedState, state)
assert.EqualValuesf(t, expectedActions.ActionType, actions.ActionType, "'ActionType' did not match expectation")
assert.Len(t, actions.DeleteIDs, 2, "'deleteIDs' did not match expectation")
assert.EqualValuesf(t, expectedActions.Create, actions.Create, "'create' did not match expectation")
assert.EqualValuesf(t, expectedActions.BackoffUntil, actions.BackoffUntil, "'BackoffUntil' did not match expectation")
require.Equal(t, len(expectedActions), len(actions))
assert.EqualValuesf(t, expectedActions[0].ActionType, actions[0].ActionType, "'ActionType' did not match expectation")
assert.Len(t, actions[0].DeleteIDs, 2, "'deleteIDs' did not match expectation")
assert.EqualValuesf(t, expectedActions[0].Create, actions[0].Create, "'create' did not match expectation")
assert.EqualValuesf(t, expectedActions[0].BackoffUntil, actions[0].BackoffUntil, "'BackoffUntil' did not match expectation")
},
},
}
@ -450,7 +467,7 @@ func TestInProgressActions(t *testing.T) {
state := ps.CalculateState()
actions, err := ps.CalculateActions(clock, backoffInterval)
require.NoError(t, err)
tc.checkFn(*state, *actions)
tc.checkFn(*state, actions)
})
}
}
@ -496,10 +513,187 @@ func TestExtraneous(t *testing.T) {
validateState(t, prebuilds.ReconciliationState{
Actual: 2, Desired: 1, Extraneous: 1, Eligible: 2,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{older},
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{older},
},
}, actions)
}
// A prebuild is considered Expired when it has exceeded their time-to-live (TTL)
// specified in the preset's cache invalidation invalidate_after_secs parameter.
func TestExpiredPrebuilds(t *testing.T) {
t.Parallel()
current := opts[optionSet3]
clock := quartz.NewMock(t)
cases := []struct {
name string
running int32
desired int32
expired int32
checkFn func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions)
}{
// With 2 running prebuilds, none of which are expired, and the desired count is met,
// no deletions or creations should occur.
{
name: "no expired prebuilds - no actions taken",
running: 2,
desired: 2,
expired: 0,
checkFn: func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 2, Desired: 2, Expired: 0}, state)
validateActions(t, nil, actions)
},
},
// With 2 running prebuilds, 1 of which is expired, the expired prebuild should be deleted,
// and one new prebuild should be created to maintain the desired count.
{
name: "one expired prebuild deleted and replaced",
running: 2,
desired: 2,
expired: 1,
checkFn: func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
expectedState := prebuilds.ReconciliationState{Actual: 2, Desired: 2, Expired: 1}
expectedActions := []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{runningPrebuilds[0].ID},
},
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}
validateState(t, expectedState, state)
validateActions(t, expectedActions, actions)
},
},
// With 2 running prebuilds, both expired, both should be deleted,
// and 2 new prebuilds created to match the desired count.
{
name: "all prebuilds expired all deleted and recreated",
running: 2,
desired: 2,
expired: 2,
checkFn: func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
expectedState := prebuilds.ReconciliationState{Actual: 2, Desired: 2, Expired: 2}
expectedActions := []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{runningPrebuilds[0].ID, runningPrebuilds[1].ID},
},
{
ActionType: prebuilds.ActionTypeCreate,
Create: 2,
},
}
validateState(t, expectedState, state)
validateActions(t, expectedActions, actions)
},
},
// With 4 running prebuilds, 2 of which are expired, and the desired count is 2,
// the expired prebuilds should be deleted. No new creations are needed
// since removing the expired ones brings actual = desired.
{
name: "expired prebuilds deleted to reach desired count",
running: 4,
desired: 2,
expired: 2,
checkFn: func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
expectedState := prebuilds.ReconciliationState{Actual: 4, Desired: 2, Expired: 2, Extraneous: 0}
expectedActions := []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{runningPrebuilds[0].ID, runningPrebuilds[1].ID},
},
}
validateState(t, expectedState, state)
validateActions(t, expectedActions, actions)
},
},
// With 4 running prebuilds (1 expired), and the desired count is 2,
// the first action should delete the expired one,
// and the second action should delete one additional (non-expired) prebuild
// to eliminate the remaining excess.
{
name: "expired prebuild deleted first, then extraneous",
running: 4,
desired: 2,
expired: 1,
checkFn: func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
expectedState := prebuilds.ReconciliationState{Actual: 4, Desired: 2, Expired: 1, Extraneous: 1}
expectedActions := []*prebuilds.ReconciliationActions{
// First action correspond to deleting the expired prebuild,
// and the second action corresponds to deleting the extraneous prebuild
// corresponding to the oldest one after the expired prebuild
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{runningPrebuilds[0].ID},
},
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{runningPrebuilds[1].ID},
},
}
validateState(t, expectedState, state)
validateActions(t, expectedActions, actions)
},
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
// GIVEN: a preset.
defaultPreset := preset(true, tc.desired, current)
presets := []database.GetTemplatePresetsWithPrebuildsRow{
defaultPreset,
}
// GIVEN: running prebuilt workspaces for the preset.
running := make([]database.GetRunningPrebuiltWorkspacesRow, 0, tc.running)
expiredCount := 0
ttlDuration := time.Duration(defaultPreset.Ttl.Int32)
for range tc.running {
name, err := prebuilds.GenerateName()
require.NoError(t, err)
prebuildCreateAt := time.Now()
if int(tc.expired) > expiredCount {
// Update the prebuild workspace createdAt to exceed its TTL (5 seconds)
prebuildCreateAt = prebuildCreateAt.Add(-ttlDuration - 10*time.Second)
expiredCount++
}
running = append(running, database.GetRunningPrebuiltWorkspacesRow{
ID: uuid.New(),
Name: name,
TemplateID: current.templateID,
TemplateVersionID: current.templateVersionID,
CurrentPresetID: uuid.NullUUID{UUID: current.presetID, Valid: true},
Ready: false,
CreatedAt: prebuildCreateAt,
})
}
// WHEN: calculating the current preset's state.
snapshot := prebuilds.NewGlobalSnapshot(presets, running, nil, nil, nil)
ps, err := snapshot.FilterByPreset(current.presetID)
require.NoError(t, err)
// THEN: we should identify that this prebuild is expired.
state := ps.CalculateState()
actions, err := ps.CalculateActions(clock, backoffInterval)
require.NoError(t, err)
tc.checkFn(running, *state, actions)
})
}
}
// A template marked as deprecated will not have prebuilds running.
@ -536,10 +730,12 @@ func TestDeprecated(t *testing.T) {
validateState(t, prebuilds.ReconciliationState{
Actual: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{current.prebuiltWorkspaceID},
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{current.prebuiltWorkspaceID},
},
}, actions)
}
// If the latest build failed, backoff exponentially with the given interval.
@ -587,10 +783,12 @@ func TestLatestBuildFailed(t *testing.T) {
validateState(t, prebuilds.ReconciliationState{
Actual: 0, Desired: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeBackoff,
BackoffUntil: lastBuildTime.Add(time.Duration(numFailed) * backoffInterval),
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeBackoff,
BackoffUntil: lastBuildTime.Add(time.Duration(numFailed) * backoffInterval),
},
}, actions)
// WHEN: calculating the other preset's state.
psOther, err := snapshot.FilterByPreset(other.presetID)
@ -603,10 +801,7 @@ func TestLatestBuildFailed(t *testing.T) {
validateState(t, prebuilds.ReconciliationState{
Actual: 1, Desired: 1, Eligible: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
BackoffUntil: time.Time{},
}, *actions)
validateActions(t, nil, actions)
// WHEN: the clock is advanced a backoff interval.
clock.Advance(backoffInterval + time.Microsecond)
@ -620,11 +815,12 @@ func TestLatestBuildFailed(t *testing.T) {
validateState(t, prebuilds.ReconciliationState{
Actual: 0, Desired: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1, // <--- NOTE: we're now able to create a new prebuild because the interval has elapsed.
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1, // <--- NOTE: we're now able to create a new prebuild because the interval has elapsed.
},
}, actions)
}
func TestMultiplePresetsPerTemplateVersion(t *testing.T) {
@ -684,10 +880,7 @@ func TestMultiplePresetsPerTemplateVersion(t *testing.T) {
Starting: 1,
Desired: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 0,
}, *actions)
validateActions(t, nil, actions)
}
// One prebuild has to be created for preset 2. Make sure preset 1 doesn't block preset 2.
@ -703,14 +896,23 @@ func TestMultiplePresetsPerTemplateVersion(t *testing.T) {
Starting: 0,
Desired: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}, actions)
}
}
func preset(active bool, instances int32, opts options, muts ...func(row database.GetTemplatePresetsWithPrebuildsRow) database.GetTemplatePresetsWithPrebuildsRow) database.GetTemplatePresetsWithPrebuildsRow {
ttl := sql.NullInt32{}
if opts.ttl > 0 {
ttl = sql.NullInt32{
Valid: true,
Int32: opts.ttl,
}
}
entry := database.GetTemplatePresetsWithPrebuildsRow{
TemplateID: opts.templateID,
TemplateVersionID: opts.templateVersionID,
@ -723,6 +925,7 @@ func preset(active bool, instances int32, opts options, muts ...func(row databas
},
Deleted: false,
Deprecated: false,
Ttl: ttl,
}
for _, mut := range muts {
@ -758,6 +961,6 @@ func validateState(t *testing.T, expected, actual prebuilds.ReconciliationState)
// validateActions is a convenience func to make tests more readable; it exploits the fact that the default states for
// prebuilds align with zero values.
func validateActions(t *testing.T, expected, actual prebuilds.ReconciliationActions) {
func validateActions(t *testing.T, expected, actual []*prebuilds.ReconciliationActions) {
require.Equal(t, expected, actual)
}

View File

@ -2059,23 +2059,26 @@ func InsertWorkspacePresetsAndParameters(ctx context.Context, logger slog.Logger
func InsertWorkspacePresetAndParameters(ctx context.Context, db database.Store, templateVersionID uuid.UUID, protoPreset *sdkproto.Preset, t time.Time) error {
err := db.InTx(func(tx database.Store) error {
var desiredInstances sql.NullInt32
var desiredInstances, ttl sql.NullInt32
if protoPreset != nil && protoPreset.Prebuild != nil {
desiredInstances = sql.NullInt32{
Int32: protoPreset.Prebuild.Instances,
Valid: true,
}
if protoPreset.Prebuild.ExpirationPolicy != nil {
ttl = sql.NullInt32{
Int32: protoPreset.Prebuild.ExpirationPolicy.Ttl,
Valid: true,
}
}
}
dbPreset, err := tx.InsertPreset(ctx, database.InsertPresetParams{
ID: uuid.New(),
TemplateVersionID: templateVersionID,
Name: protoPreset.Name,
CreatedAt: t,
DesiredInstances: desiredInstances,
InvalidateAfterSecs: sql.NullInt32{
Int32: 0,
Valid: false,
}, // TODO: implement cache invalidation
ID: uuid.New(),
TemplateVersionID: templateVersionID,
Name: protoPreset.Name,
CreatedAt: t,
DesiredInstances: desiredInstances,
InvalidateAfterSecs: ttl,
})
if err != nil {
return xerrors.Errorf("insert preset: %w", err)