Files
coder/coderd/cryptokeys/rotate_internal_test.go
Spike Curtis 5861e516b9 chore: add standard test logger ignoring db canceled (#15556)
Refactors our use of `slogtest` to instantiate a "standard logger" across most of our tests.  This standard logger incorporates https://github.com/coder/slog/pull/217 to also ignore database query canceled errors by default, which are a source of low-severity flakes.

Any test that has set non-default `slogtest.Options` is left alone. In particular, `coderdtest` defaults to ignoring all errors. We might consider revisiting that decision now that we have better tools to target the really common flaky Error logs on shutdown.
2024-11-18 14:09:22 +04:00

607 lines
18 KiB
Go

package cryptokeys
import (
"database/sql"
"encoding/hex"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbgen"
"github.com/coder/coder/v2/coderd/database/dbtestutil"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/testutil"
"github.com/coder/quartz"
)
func Test_rotateKeys(t *testing.T) {
t.Parallel()
t.Run("RotatesKeysNearExpiration", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 7
logger = testutil.Logger(t)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{
database.CryptoKeyFeatureWorkspaceAppsAPIKey,
},
}
now := dbnow(clock)
// Seed the database with an existing key.
oldKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey,
StartsAt: now,
Sequence: 15,
})
// Advance the window to just inside rotation time.
_ = clock.Advance(keyDuration - time.Minute*59)
err := kr.rotateKeys(ctx)
require.NoError(t, err)
now = dbnow(clock)
expectedDeletesAt := oldKey.ExpiresAt(keyDuration).Add(WorkspaceAppsTokenDuration + time.Hour)
// Fetch the old key, it should have an deletes_at now.
oldKey, err = db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
Feature: oldKey.Feature,
Sequence: oldKey.Sequence,
})
require.NoError(t, err)
require.Equal(t, oldKey.DeletesAt.Time.UTC(), expectedDeletesAt)
// The new key should be created and have a starts_at of the old key's expires_at.
newKey, err := db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey,
Sequence: oldKey.Sequence + 1,
})
require.NoError(t, err)
requireKey(t, newKey, database.CryptoKeyFeatureWorkspaceAppsAPIKey, oldKey.ExpiresAt(keyDuration), nullTime, oldKey.Sequence+1)
// Advance the clock just before the keys delete time.
clock.Advance(oldKey.DeletesAt.Time.UTC().Sub(now) - time.Second)
// No action should be taken.
err = kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 2)
// Advance the clock just past the keys delete time.
clock.Advance(oldKey.DeletesAt.Time.UTC().Sub(now) + time.Second)
// We should have deleted the old key.
err = kr.rotateKeys(ctx)
require.NoError(t, err)
// The old key should be "deleted".
_, err = db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
Feature: oldKey.Feature,
Sequence: oldKey.Sequence,
})
require.ErrorIs(t, err, sql.ErrNoRows)
keys, err = db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 1)
require.Equal(t, newKey, keys[0])
})
t.Run("DoesNotRotateValidKeys", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 7
logger = testutil.Logger(t)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{
database.CryptoKeyFeatureWorkspaceAppsAPIKey,
},
}
now := dbnow(clock)
// Seed the database with an existing key
existingKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey,
StartsAt: now,
Sequence: 123,
})
// Advance the clock by 6 days, 22 hours. Once we
// breach the last hour we will insert a new key.
clock.Advance(keyDuration - 2*time.Hour)
err := kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 1)
require.Equal(t, existingKey, keys[0])
// Advance it again to just before the key is scheduled to be rotated for sanity purposes.
clock.Advance(time.Hour - time.Second)
err = kr.rotateKeys(ctx)
require.NoError(t, err)
// Verify that the existing key is still the only key in the database
keys, err = db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 1)
requireKey(t, keys[0], existingKey.Feature, existingKey.StartsAt.UTC(), nullTime, existingKey.Sequence)
})
// Simulate a situation where the database was manually altered such that we only have a key that is scheduled to be deleted and assert we insert a new key.
t.Run("DeletesExpiredKeys", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 7
logger = testutil.Logger(t)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{
database.CryptoKeyFeatureWorkspaceAppsAPIKey,
},
}
now := dbnow(clock)
// Seed the database with an existing key
deletingKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey,
StartsAt: now.Add(-keyDuration),
Sequence: 789,
DeletesAt: sql.NullTime{
Time: now,
Valid: true,
},
})
err := kr.rotateKeys(ctx)
require.NoError(t, err)
// We should only get one key since the old key
// should be deleted.
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 1)
requireKey(t, keys[0], deletingKey.Feature, deletingKey.DeletesAt.Time.UTC(), nullTime, deletingKey.Sequence+1)
// The old key should be "deleted".
_, err = db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
Feature: deletingKey.Feature,
Sequence: deletingKey.Sequence,
})
require.ErrorIs(t, err, sql.ErrNoRows)
})
// This tests a situation where we have a key scheduled for deletion but it's still valid for use.
// If no other key is detected we should insert a new key.
t.Run("AddsKeyForDeletingKey", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 7
logger = testutil.Logger(t)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{
database.CryptoKeyFeatureWorkspaceAppsAPIKey,
},
}
now := dbnow(clock)
// Seed the database with an existing key
deletingKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey,
StartsAt: now,
Sequence: 456,
DeletesAt: sql.NullTime{
Time: now.Add(time.Hour),
Valid: true,
},
})
// We should only have inserted a key.
err := kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 2)
oldKey, newKey := keys[0], keys[1]
if oldKey.Sequence != deletingKey.Sequence {
oldKey, newKey = newKey, oldKey
}
requireKey(t, oldKey, deletingKey.Feature, deletingKey.StartsAt.UTC(), deletingKey.DeletesAt, deletingKey.Sequence)
requireKey(t, newKey, deletingKey.Feature, now, nullTime, deletingKey.Sequence+1)
})
t.Run("NoKeys", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 7
logger = testutil.Logger(t)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{
database.CryptoKeyFeatureWorkspaceAppsAPIKey,
},
}
err := kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 1)
requireKey(t, keys[0], database.CryptoKeyFeatureWorkspaceAppsAPIKey, clock.Now().UTC(), nullTime, 1)
})
// Assert we insert a new key when the only key was manually deleted.
t.Run("OnlyDeletedKeys", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 7
logger = testutil.Logger(t)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{
database.CryptoKeyFeatureWorkspaceAppsAPIKey,
},
}
now := dbnow(clock)
deletedkey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey,
StartsAt: now,
Sequence: 19,
DeletesAt: sql.NullTime{
Time: now.Add(time.Hour),
Valid: true,
},
Secret: sql.NullString{
String: "deleted",
Valid: false,
},
})
err := kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 1)
requireKey(t, keys[0], database.CryptoKeyFeatureWorkspaceAppsAPIKey, now, nullTime, deletedkey.Sequence+1)
})
// This tests ensures that rotation works with multiple
// features. It's mainly a sanity test since some bugs
// are not unveiled in the simple n=1 case.
t.Run("AllFeatures", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 30
logger = testutil.Logger(t)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: database.AllCryptoKeyFeatureValues(),
}
now := dbnow(clock)
// We'll test a scenario where:
// - One feature has no valid keys.
// - One has a key that should be rotated.
// - One has a valid key that shouldn't trigger an action.
// - One has no keys at all.
_ = dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureTailnetResume,
StartsAt: now.Add(-keyDuration),
Sequence: 5,
Secret: sql.NullString{
String: "older key",
Valid: false,
},
})
// Generate another deleted key to ensure we insert after the latest sequence.
deletedKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureTailnetResume,
StartsAt: now.Add(-keyDuration),
Sequence: 19,
Secret: sql.NullString{
String: "old key",
Valid: false,
},
})
// Insert a key that should be rotated.
rotatedKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey,
StartsAt: now.Add(-keyDuration + time.Hour),
Sequence: 42,
})
// Insert a key that should not trigger an action.
validKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureOIDCConvert,
StartsAt: now,
Sequence: 17,
})
err := kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 5)
kbf, err := keysByFeature(keys, database.AllCryptoKeyFeatureValues())
require.NoError(t, err)
// No actions on OIDC convert.
require.Len(t, kbf[database.CryptoKeyFeatureOIDCConvert], 1)
// Workspace apps should have been rotated.
require.Len(t, kbf[database.CryptoKeyFeatureWorkspaceAppsAPIKey], 2)
// No existing key for tailnet resume should've
// caused a key to be inserted.
require.Len(t, kbf[database.CryptoKeyFeatureTailnetResume], 1)
require.Len(t, kbf[database.CryptoKeyFeatureWorkspaceAppsToken], 1)
oidcKey := kbf[database.CryptoKeyFeatureOIDCConvert][0]
tailnetKey := kbf[database.CryptoKeyFeatureTailnetResume][0]
appTokenKey := kbf[database.CryptoKeyFeatureWorkspaceAppsToken][0]
requireKey(t, oidcKey, database.CryptoKeyFeatureOIDCConvert, now, nullTime, validKey.Sequence)
requireKey(t, tailnetKey, database.CryptoKeyFeatureTailnetResume, now, nullTime, deletedKey.Sequence+1)
requireKey(t, appTokenKey, database.CryptoKeyFeatureWorkspaceAppsToken, now, nullTime, 1)
newKey := kbf[database.CryptoKeyFeatureWorkspaceAppsAPIKey][0]
oldKey := kbf[database.CryptoKeyFeatureWorkspaceAppsAPIKey][1]
if newKey.Sequence == rotatedKey.Sequence {
oldKey, newKey = newKey, oldKey
}
deletesAt := sql.NullTime{
Time: rotatedKey.ExpiresAt(keyDuration).Add(WorkspaceAppsTokenDuration + time.Hour),
Valid: true,
}
requireKey(t, oldKey, database.CryptoKeyFeatureWorkspaceAppsAPIKey, rotatedKey.StartsAt.UTC(), deletesAt, rotatedKey.Sequence)
requireKey(t, newKey, database.CryptoKeyFeatureWorkspaceAppsAPIKey, rotatedKey.ExpiresAt(keyDuration), nullTime, rotatedKey.Sequence+1)
})
t.Run("UnknownFeature", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 7
logger = testutil.Logger(t)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{database.CryptoKeyFeature("unknown")},
}
err := kr.rotateKeys(ctx)
require.Error(t, err)
})
t.Run("MinStartsAt", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 5
logger = testutil.Logger(t)
ctx = testutil.Context(t, testutil.WaitShort)
)
now := dbnow(clock)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{database.CryptoKeyFeatureWorkspaceAppsAPIKey},
}
expiringKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey,
StartsAt: now.Add(-keyDuration),
Sequence: 345,
})
err := kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 2)
rotatedKey, err := db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
Feature: expiringKey.Feature,
Sequence: expiringKey.Sequence + 1,
})
require.NoError(t, err)
require.Equal(t, now.Add(defaultRotationInterval*3), rotatedKey.StartsAt.UTC())
})
// Test that the the deletes_at of a key that is well past its expiration
// Has its deletes_at field set to value that is relative
// to the current time to afford propagation time for the
// new key.
t.Run("ExtensivelyExpiredKey", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 3
logger = testutil.Logger(t)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{database.CryptoKeyFeatureWorkspaceAppsAPIKey},
}
now := dbnow(clock)
expiredKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey,
StartsAt: now.Add(-keyDuration - 2*time.Hour),
Sequence: 19,
})
deletedKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey,
StartsAt: now,
Sequence: 20,
Secret: sql.NullString{
String: "deleted",
Valid: false,
},
})
err := kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 2)
deletesAtKey, err := db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
Feature: expiredKey.Feature,
Sequence: expiredKey.Sequence,
})
deletesAt := sql.NullTime{
Time: now.Add(defaultRotationInterval * 3).Add(WorkspaceAppsTokenDuration + time.Hour),
Valid: true,
}
require.NoError(t, err)
requireKey(t, deletesAtKey, expiredKey.Feature, expiredKey.StartsAt.UTC(), deletesAt, expiredKey.Sequence)
newKey, err := db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
Feature: expiredKey.Feature,
Sequence: deletedKey.Sequence + 1,
})
require.NoError(t, err)
requireKey(t, newKey, expiredKey.Feature, now.Add(defaultRotationInterval*3), nullTime, deletedKey.Sequence+1)
})
}
func dbnow(c quartz.Clock) time.Time {
return dbtime.Time(c.Now().UTC())
}
func requireKey(t *testing.T, key database.CryptoKey, feature database.CryptoKeyFeature, startsAt time.Time, deletesAt sql.NullTime, sequence int32) {
t.Helper()
require.Equal(t, feature, key.Feature)
require.Equal(t, startsAt, key.StartsAt.UTC())
require.Equal(t, deletesAt.Valid, key.DeletesAt.Valid)
require.Equal(t, deletesAt.Time.UTC(), key.DeletesAt.Time.UTC())
require.Equal(t, sequence, key.Sequence)
secret, err := hex.DecodeString(key.Secret.String)
require.NoError(t, err)
switch key.Feature {
case database.CryptoKeyFeatureOIDCConvert:
require.Len(t, secret, 64)
case database.CryptoKeyFeatureWorkspaceAppsToken:
require.Len(t, secret, 64)
case database.CryptoKeyFeatureWorkspaceAppsAPIKey:
require.Len(t, secret, 32)
case database.CryptoKeyFeatureTailnetResume:
require.Len(t, secret, 64)
default:
t.Fatalf("unknown key feature: %s", key.Feature)
}
}
var nullTime = sql.NullTime{Time: time.Time{}, Valid: false}