mirror of
https://github.com/coder/coder.git
synced 2025-07-12 00:14:10 +00:00
feat: implement scheduling mechanism for prebuilds (#18126)
Closes https://github.com/coder/internal/issues/312 Depends on https://github.com/coder/terraform-provider-coder/pull/408 This PR adds support for defining an **autoscaling block** for prebuilds, allowing number of desired instances to scale dynamically based on a schedule. Example usage: ``` data "coder_workspace_preset" "us-nix" { ... prebuilds = { instances = 0 # default to 0 instances scheduling = { timezone = "UTC" # a single timezone is used for simplicity # Scale to 3 instances during the work week schedule { cron = "* 8-18 * * 1-5" # from 8AM–6:59PM, Mon–Fri, UTC instances = 3 # scale to 3 instances } # Scale to 1 instance on Saturdays for urgent support queries schedule { cron = "* 8-14 * * 6" # from 8AM–2:59PM, Sat, UTC instances = 1 # scale to 1 instance } } } } ``` ### Behavior - Multiple `schedule` blocks per `prebuilds` block are supported. - If the current time matches any defined autoscaling schedule, the corresponding number of instances is used. - If no schedule matches, the **default instance count** (`prebuilds.instances`) is used as a fallback. ### Why This feature allows prebuild instance capacity to adapt to predictable usage patterns, such as: - Scaling up during business hours or high-demand periods - Reducing capacity during off-hours to save resources ### Cron specification The cron specification is interpreted as a **continuous time range.** For example, the expression: ``` * 9-18 * * 1-5 ``` is intended to represent a continuous range from **09:00 to 18:59**, Monday through Friday. However, due to minor implementation imprecision, it is currently interpreted as a range from **08:59:00 to 18:58:59**, Monday through Friday. This slight discrepancy arises because the evaluation is based on whether a specific **point in time** falls within the range, using the `github.com/coder/coder/v2/coderd/schedule/cron` library, which performs per-minute matching rather than strict range evaluation. --------- Co-authored-by: Danny Kopping <danny@coder.com>
This commit is contained in:
Yevhenii Shcherbina
committed by
GitHub
GitHub
parent
511fd09582
commit
0f6ca55238
@ -366,6 +366,11 @@ func (c *StoreReconciler) SnapshotState(ctx context.Context, store database.Stor
|
||||
return nil
|
||||
}
|
||||
|
||||
presetPrebuildSchedules, err := db.GetActivePresetPrebuildSchedules(ctx)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to get preset prebuild schedules: %w", err)
|
||||
}
|
||||
|
||||
allRunningPrebuilds, err := db.GetRunningPrebuiltWorkspaces(ctx)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to get running prebuilds: %w", err)
|
||||
@ -388,10 +393,13 @@ func (c *StoreReconciler) SnapshotState(ctx context.Context, store database.Stor
|
||||
|
||||
state = prebuilds.NewGlobalSnapshot(
|
||||
presetsWithPrebuilds,
|
||||
presetPrebuildSchedules,
|
||||
allRunningPrebuilds,
|
||||
allPrebuildsInProgress,
|
||||
presetsBackoff,
|
||||
hardLimitedPresets,
|
||||
c.clock,
|
||||
c.logger,
|
||||
)
|
||||
return nil
|
||||
}, &database.TxOptions{
|
||||
@ -608,7 +616,8 @@ func (c *StoreReconciler) executeReconciliationAction(ctx context.Context, logge
|
||||
// Unexpected things happen (i.e. bugs or bitflips); let's defend against disastrous outcomes.
|
||||
// See https://blog.robertelder.org/causes-of-bit-flips-in-computer-memory/.
|
||||
// This is obviously not comprehensive protection against this sort of problem, but this is one essential check.
|
||||
desired := ps.Preset.DesiredInstances.Int32
|
||||
desired := ps.CalculateDesiredInstances(c.clock.Now())
|
||||
|
||||
if action.Create > desired {
|
||||
logger.Critical(ctx, "determined excessive count of prebuilds to create; clamping to desired count",
|
||||
slog.F("create_count", action.Create), slog.F("desired_count", desired))
|
||||
|
||||
@ -522,6 +522,151 @@ func TestMultiplePresetsPerTemplateVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrebuildScheduling(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if !dbtestutil.WillUsePostgres() {
|
||||
t.Skip("This test requires postgres")
|
||||
}
|
||||
|
||||
templateDeleted := false
|
||||
|
||||
// The test includes 2 presets, each with 2 schedules.
|
||||
// It checks that the number of created prebuilds match expectations for various provided times,
|
||||
// based on the corresponding schedules.
|
||||
testCases := []struct {
|
||||
name string
|
||||
// now specifies the current time.
|
||||
now time.Time
|
||||
// expected prebuild counts for preset1 and preset2, respectively.
|
||||
expectedPrebuildCounts []int
|
||||
}{
|
||||
{
|
||||
name: "Before the 1st schedule",
|
||||
now: mustParseTime(t, time.RFC1123, "Mon, 02 Jun 2025 01:00:00 UTC"),
|
||||
expectedPrebuildCounts: []int{1, 1},
|
||||
},
|
||||
{
|
||||
name: "1st schedule",
|
||||
now: mustParseTime(t, time.RFC1123, "Mon, 02 Jun 2025 03:00:00 UTC"),
|
||||
expectedPrebuildCounts: []int{2, 1},
|
||||
},
|
||||
{
|
||||
name: "2nd schedule",
|
||||
now: mustParseTime(t, time.RFC1123, "Mon, 02 Jun 2025 07:00:00 UTC"),
|
||||
expectedPrebuildCounts: []int{3, 1},
|
||||
},
|
||||
{
|
||||
name: "3rd schedule",
|
||||
now: mustParseTime(t, time.RFC1123, "Mon, 02 Jun 2025 11:00:00 UTC"),
|
||||
expectedPrebuildCounts: []int{1, 4},
|
||||
},
|
||||
{
|
||||
name: "4th schedule",
|
||||
now: mustParseTime(t, time.RFC1123, "Mon, 02 Jun 2025 15:00:00 UTC"),
|
||||
expectedPrebuildCounts: []int{1, 5},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
clock := quartz.NewMock(t)
|
||||
clock.Set(tc.now)
|
||||
ctx := testutil.Context(t, testutil.WaitShort)
|
||||
cfg := codersdk.PrebuildsConfig{}
|
||||
logger := slogtest.Make(
|
||||
t, &slogtest.Options{IgnoreErrors: true},
|
||||
).Leveled(slog.LevelDebug)
|
||||
db, pubSub := dbtestutil.NewDB(t)
|
||||
controller := prebuilds.NewStoreReconciler(db, pubSub, cfg, logger, clock, prometheus.NewRegistry(), newNoopEnqueuer())
|
||||
|
||||
ownerID := uuid.New()
|
||||
dbgen.User(t, db, database.User{
|
||||
ID: ownerID,
|
||||
})
|
||||
org, template := setupTestDBTemplate(t, db, ownerID, templateDeleted)
|
||||
templateVersionID := setupTestDBTemplateVersion(
|
||||
ctx,
|
||||
t,
|
||||
clock,
|
||||
db,
|
||||
pubSub,
|
||||
org.ID,
|
||||
ownerID,
|
||||
template.ID,
|
||||
)
|
||||
preset1 := setupTestDBPresetWithScheduling(
|
||||
t,
|
||||
db,
|
||||
templateVersionID,
|
||||
1,
|
||||
uuid.New().String(),
|
||||
"UTC",
|
||||
)
|
||||
preset2 := setupTestDBPresetWithScheduling(
|
||||
t,
|
||||
db,
|
||||
templateVersionID,
|
||||
1,
|
||||
uuid.New().String(),
|
||||
"UTC",
|
||||
)
|
||||
|
||||
dbgen.PresetPrebuildSchedule(t, db, database.InsertPresetPrebuildScheduleParams{
|
||||
PresetID: preset1.ID,
|
||||
CronExpression: "* 2-4 * * 1-5",
|
||||
DesiredInstances: 2,
|
||||
})
|
||||
dbgen.PresetPrebuildSchedule(t, db, database.InsertPresetPrebuildScheduleParams{
|
||||
PresetID: preset1.ID,
|
||||
CronExpression: "* 6-8 * * 1-5",
|
||||
DesiredInstances: 3,
|
||||
})
|
||||
dbgen.PresetPrebuildSchedule(t, db, database.InsertPresetPrebuildScheduleParams{
|
||||
PresetID: preset2.ID,
|
||||
CronExpression: "* 10-12 * * 1-5",
|
||||
DesiredInstances: 4,
|
||||
})
|
||||
dbgen.PresetPrebuildSchedule(t, db, database.InsertPresetPrebuildScheduleParams{
|
||||
PresetID: preset2.ID,
|
||||
CronExpression: "* 14-16 * * 1-5",
|
||||
DesiredInstances: 5,
|
||||
})
|
||||
|
||||
err := controller.ReconcileAll(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
// get workspace builds
|
||||
workspaces, err := db.GetWorkspacesByTemplateID(ctx, template.ID)
|
||||
require.NoError(t, err)
|
||||
workspaceIDs := make([]uuid.UUID, 0, len(workspaces))
|
||||
for _, workspace := range workspaces {
|
||||
workspaceIDs = append(workspaceIDs, workspace.ID)
|
||||
}
|
||||
workspaceBuilds, err := db.GetLatestWorkspaceBuildsByWorkspaceIDs(ctx, workspaceIDs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// calculate number of workspace builds per preset
|
||||
var (
|
||||
preset1PrebuildCount int
|
||||
preset2PrebuildCount int
|
||||
)
|
||||
for _, workspaceBuild := range workspaceBuilds {
|
||||
if preset1.ID == workspaceBuild.TemplateVersionPresetID.UUID {
|
||||
preset1PrebuildCount++
|
||||
}
|
||||
if preset2.ID == workspaceBuild.TemplateVersionPresetID.UUID {
|
||||
preset2PrebuildCount++
|
||||
}
|
||||
}
|
||||
|
||||
require.Equal(t, tc.expectedPrebuildCounts[0], preset1PrebuildCount)
|
||||
require.Equal(t, tc.expectedPrebuildCounts[1], preset2PrebuildCount)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidPreset(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@ -1821,6 +1966,32 @@ func setupTestDBPreset(
|
||||
return preset
|
||||
}
|
||||
|
||||
func setupTestDBPresetWithScheduling(
|
||||
t *testing.T,
|
||||
db database.Store,
|
||||
templateVersionID uuid.UUID,
|
||||
desiredInstances int32,
|
||||
presetName string,
|
||||
schedulingTimezone string,
|
||||
) database.TemplateVersionPreset {
|
||||
t.Helper()
|
||||
preset := dbgen.Preset(t, db, database.InsertPresetParams{
|
||||
TemplateVersionID: templateVersionID,
|
||||
Name: presetName,
|
||||
DesiredInstances: sql.NullInt32{
|
||||
Valid: true,
|
||||
Int32: desiredInstances,
|
||||
},
|
||||
SchedulingTimezone: schedulingTimezone,
|
||||
})
|
||||
dbgen.PresetParameter(t, db, database.InsertPresetParametersParams{
|
||||
TemplateVersionPresetID: preset.ID,
|
||||
Names: []string{"test"},
|
||||
Values: []string{"test"},
|
||||
})
|
||||
return preset
|
||||
}
|
||||
|
||||
// prebuildOptions holds optional parameters for creating a prebuild workspace.
|
||||
type prebuildOptions struct {
|
||||
createdAt *time.Time
|
||||
@ -1988,3 +2159,10 @@ func allJobStatusesExcept(except ...database.ProvisionerJobStatus) []database.Pr
|
||||
return !slice.Contains(allJobStatuses, status)
|
||||
})
|
||||
}
|
||||
|
||||
func mustParseTime(t *testing.T, layout, value string) time.Time {
|
||||
t.Helper()
|
||||
parsedTime, err := time.Parse(layout, value)
|
||||
require.NoError(t, err)
|
||||
return parsedTime
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user