synced/coder
1
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2025-07-12 00:14:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add prebuilds configuration & bootstrapping (#17527)

Browse Source 
Closes https://github.com/coder/internal/issues/508

---------

Signed-off-by: Danny Kopping <dannykopping@gmail.com>
Co-authored-by: Cian Johnston <cian@coder.com>
This commit is contained in:
Danny Kopping
2025-04-25 11:07:15 +02:00
committed by GitHub
parent e562e3c882
commit 08ad910171
14 changed files with 328 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
enterprise/coderd/coderd.go
View File

@ -12,12 +12,15 @@ import (
"sync"
"time"
"github.com/coder/quartz"
"github.com/coder/coder/v2/buildinfo"
"github.com/coder/coder/v2/coderd/appearance"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/entitlements"
"github.com/coder/coder/v2/coderd/idpsync"
agplportsharing "github.com/coder/coder/v2/coderd/portsharing"
agplprebuilds "github.com/coder/coder/v2/coderd/prebuilds"
"github.com/coder/coder/v2/coderd/rbac/policy"
"github.com/coder/coder/v2/enterprise/coderd/enidpsync"
"github.com/coder/coder/v2/enterprise/coderd/portsharing"
@ -43,6 +46,7 @@ import (
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/enterprise/coderd/dbauthz"
"github.com/coder/coder/v2/enterprise/coderd/license"
"github.com/coder/coder/v2/enterprise/coderd/prebuilds"
"github.com/coder/coder/v2/enterprise/coderd/proxyhealth"
"github.com/coder/coder/v2/enterprise/coderd/schedule"
"github.com/coder/coder/v2/enterprise/dbcrypt"
@ -658,6 +662,7 @@ func (api *API) Close() error {
if api.Options.CheckInactiveUsersCancelFunc != nil {
api.Options.CheckInactiveUsersCancelFunc()
}
return api.AGPL.Close()
}
@ -860,6 +865,20 @@ func (api *API) updateEntitlements(ctx context.Context) error {
api.AGPL.PortSharer.Store(&ps)
}
if initial, changed, enabled := featureChanged(codersdk.FeatureWorkspacePrebuilds); shouldUpdate(initial, changed, enabled) {
reconciler, claimer := api.setupPrebuilds(enabled)
if current := api.AGPL.PrebuildsReconciler.Load(); current != nil {
stopCtx, giveUp := context.WithTimeoutCause(context.Background(), time.Second*30, xerrors.New("gave up waiting for reconciler to stop"))
defer giveUp()
(*current).Stop(stopCtx, xerrors.New("entitlements change"))
}
api.AGPL.PrebuildsReconciler.Store(&reconciler)
go reconciler.Run(context.Background())
api.AGPL.PrebuildsClaimer.Store(&claimer)
}
// External token encryption is soft-enforced
featureExternalTokenEncryption := reloadedEntitlements.Features[codersdk.FeatureExternalTokenEncryption]
featureExternalTokenEncryption.Enabled = len(api.ExternalTokenEncryption) > 0
@ -1128,3 +1147,24 @@ func (api *API) runEntitlementsLoop(ctx context.Context) {
func (api *API) Authorize(r *http.Request, action policy.Action, object rbac.Objecter) bool {
return api.AGPL.HTTPAuth.Authorize(r, action, object)
}
// nolint:revive // featureEnabled is a legit control flag.
func (api *API) setupPrebuilds(featureEnabled bool) (agplprebuilds.ReconciliationOrchestrator, agplprebuilds.Claimer) {
experimentEnabled := api.AGPL.Experiments.Enabled(codersdk.ExperimentWorkspacePrebuilds)
if !experimentEnabled || !featureEnabled {
levelFn := api.Logger.Debug
// If the experiment is enabled but the license does not entitle the feature, operators should be warned.
if !featureEnabled {
levelFn = api.Logger.Warn
}
levelFn(context.Background(), "prebuilds not enabled; ensure you have a premium license and the 'workspace-prebuilds' experiment set",
slog.F("experiment_enabled", experimentEnabled), slog.F("feature_enabled", featureEnabled))
return agplprebuilds.DefaultReconciler, agplprebuilds.DefaultClaimer
}
reconciler := prebuilds.NewStoreReconciler(api.Database, api.Pubsub, api.DeploymentValues.Prebuilds,
api.Logger.Named("prebuilds"), quartz.NewReal())
return reconciler, prebuilds.EnterpriseClaimer{}
}

91
enterprise/coderd/coderd_test.go
View File

@ -28,10 +28,15 @@ import (
"github.com/coder/coder/v2/agent"
"github.com/coder/coder/v2/agent/agenttest"
"github.com/coder/coder/v2/coderd/httpapi"
agplprebuilds "github.com/coder/coder/v2/coderd/prebuilds"
"github.com/coder/coder/v2/coderd/rbac/policy"
"github.com/coder/coder/v2/coderd/util/ptr"
"github.com/coder/coder/v2/enterprise/coderd/prebuilds"
"github.com/coder/coder/v2/tailnet/tailnettest"
"github.com/coder/retry"
"github.com/coder/serpent"
agplaudit "github.com/coder/coder/v2/coderd/audit"
"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/coderd/database"
@ -50,8 +55,6 @@ import (
"github.com/coder/coder/v2/enterprise/dbcrypt"
"github.com/coder/coder/v2/enterprise/replicasync"
"github.com/coder/coder/v2/testutil"
"github.com/coder/retry"
"github.com/coder/serpent"
)
func TestMain(m *testing.M) {
@ -253,6 +256,90 @@ func TestEntitlements_HeaderWarnings(t *testing.T) {
})
}
func TestEntitlements_Prebuilds(t *testing.T) {
t.Parallel()
cases := []struct {
name string
experimentEnabled bool
featureEnabled bool
expectedEnabled bool
}{
{
name: "Fully enabled",
featureEnabled: true,
experimentEnabled: true,
expectedEnabled: true,
},
{
name: "Feature disabled",
featureEnabled: false,
experimentEnabled: true,
expectedEnabled: false,
},
{
name: "Experiment disabled",
featureEnabled: true,
experimentEnabled: false,
expectedEnabled: false,
},
{
name: "Fully disabled",
featureEnabled: false,
experimentEnabled: false,
expectedEnabled: false,
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
var prebuildsEntitled int64
if tc.featureEnabled {
prebuildsEntitled = 1
}
_, _, api, _ := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
Options: &coderdtest.Options{
DeploymentValues: coderdtest.DeploymentValues(t, func(values *codersdk.DeploymentValues) {
if tc.experimentEnabled {
values.Experiments = serpent.StringArray{string(codersdk.ExperimentWorkspacePrebuilds)}
}
}),
},
EntitlementsUpdateInterval: time.Second,
LicenseOptions: &coderdenttest.LicenseOptions{
Features: license.Features{
codersdk.FeatureWorkspacePrebuilds: prebuildsEntitled,
},
},
})
// The entitlements will need to refresh before the reconciler is set.
require.Eventually(t, func() bool {
return api.AGPL.PrebuildsReconciler.Load() != nil
}, testutil.WaitSuperLong, testutil.IntervalFast)
reconciler := api.AGPL.PrebuildsReconciler.Load()
claimer := api.AGPL.PrebuildsClaimer.Load()
require.NotNil(t, reconciler)
require.NotNil(t, claimer)
if tc.expectedEnabled {
require.IsType(t, &prebuilds.StoreReconciler{}, *reconciler)
require.IsType(t, prebuilds.EnterpriseClaimer{}, *claimer)
} else {
require.Equal(t, &agplprebuilds.DefaultReconciler, reconciler)
require.Equal(t, &agplprebuilds.DefaultClaimer, claimer)
}
})
}
}
func TestAuditLogging(t *testing.T) {
t.Parallel()
t.Run("Enabled", func(t *testing.T) {

26
enterprise/coderd/prebuilds/reconcile.go
View File

@ -38,6 +38,7 @@ type StoreReconciler struct {
clock quartz.Clock
cancelFn context.CancelCauseFunc
running atomic.Bool
stopped atomic.Bool
done chan struct{}
}
@ -61,7 +62,7 @@ func NewStoreReconciler(
}
}
func (c *StoreReconciler) RunLoop(ctx context.Context) {
func (c *StoreReconciler) Run(ctx context.Context) {
reconciliationInterval := c.cfg.ReconciliationInterval.Value()
if reconciliationInterval <= 0 { // avoids a panic
reconciliationInterval = 5 * time.Minute
@ -82,6 +83,11 @@ func (c *StoreReconciler) RunLoop(ctx context.Context) {
ctx, cancel := context.WithCancelCause(dbauthz.AsPrebuildsOrchestrator(ctx))
c.cancelFn = cancel
// Everything is in place, reconciler can now be considered as running.
//
// NOTE: without this atomic bool, Stop might race with Run for the c.cancelFn above.
c.running.Store(true)
for {
select {
// TODO: implement pubsub listener to allow reconciling a specific template imperatively once it has been changed,
@ -107,16 +113,26 @@ func (c *StoreReconciler) RunLoop(ctx context.Context) {
}
func (c *StoreReconciler) Stop(ctx context.Context, cause error) {
defer c.running.Store(false)
if cause != nil {
c.logger.Error(context.Background(), "stopping reconciler due to an error", slog.Error(cause))
} else {
c.logger.Info(context.Background(), "gracefully stopping reconciler")
}
if c.isStopped() {
// If previously stopped (Swap returns previous value), then short-circuit.
//
// NOTE: we need to *prospectively* mark this as stopped to prevent Stop being called multiple times and causing problems.
if c.stopped.Swap(true) {
return
}
c.stopped.Store(true)
// If the reconciler is not running, there's nothing else to do.
if !c.running.Load() {
return
}
if c.cancelFn != nil {
c.cancelFn(cause)
}
@ -138,10 +154,6 @@ func (c *StoreReconciler) Stop(ctx context.Context, cause error) {
}
}
func (c *StoreReconciler) isStopped() bool {
return c.stopped.Load()
}
// ReconcileAll will attempt to resolve the desired vs actual state of all templates which have presets with prebuilds configured.
//
// NOTE:

6
enterprise/coderd/prebuilds/reconcile_test.go
View File

@ -575,7 +575,7 @@ func TestRunLoop(t *testing.T) {
t, &slogtest.Options{IgnoreErrors: true},
).Leveled(slog.LevelDebug)
db, pubSub := dbtestutil.NewDB(t)
controller := prebuilds.NewStoreReconciler(db, pubSub, cfg, logger, clock)
reconciler := prebuilds.NewStoreReconciler(db, pubSub, cfg, logger, clock)
ownerID := uuid.New()
dbgen.User(t, db, database.User{
@ -639,7 +639,7 @@ func TestRunLoop(t *testing.T) {
// we need to wait until ticker is initialized, and only then use clock.Advance()
// otherwise clock.Advance() will be ignored
trap := clock.Trap().NewTicker()
go controller.RunLoop(ctx)
go reconciler.Run(ctx)
// wait until ticker is initialized
trap.MustWait(ctx).Release()
// start 1st iteration of ReconciliationLoop
@ -681,7 +681,7 @@ func TestRunLoop(t *testing.T) {
}, testutil.WaitShort, testutil.IntervalFast)
// gracefully stop the reconciliation loop
controller.Stop(ctx, nil)
reconciler.Stop(ctx, nil)
}
func TestFailedBuildBackoff(t *testing.T) {