Files
coder/coderd/cryptokeys/rotate_internal_test.go
Jon Ayers 68ec532ca7 feat: add jwt pkg (#14928)
- Adds a `jwtutils` package to be shared amongst the various
packages in the codebase that make use of JWTs. It's intended to help us
standardize on one library instead of some implementations using
`go-jose` and others using `golang-jwt`.

The main reason we're converging on `go-jose` is due to its support for
JWEs, `golang-jwt` also has a repo to handle it but it doesn't look
maintained: https://github.com/golang-jwt/jwe
2024-10-03 21:09:52 -05:00

602 lines
17 KiB
Go

package cryptokeys
import (
"database/sql"
"encoding/hex"
"testing"
"time"
"github.com/stretchr/testify/require"
"cdr.dev/slog"
"cdr.dev/slog/sloggers/slogtest"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbgen"
"github.com/coder/coder/v2/coderd/database/dbtestutil"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/testutil"
"github.com/coder/quartz"
)
func Test_rotateKeys(t *testing.T) {
t.Parallel()
t.Run("RotatesKeysNearExpiration", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 7
logger = slogtest.Make(t, nil).Leveled(slog.LevelDebug)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{
database.CryptoKeyFeatureWorkspaceApps,
},
}
now := dbnow(clock)
// Seed the database with an existing key.
oldKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceApps,
StartsAt: now,
Sequence: 15,
})
// Advance the window to just inside rotation time.
_ = clock.Advance(keyDuration - time.Minute*59)
err := kr.rotateKeys(ctx)
require.NoError(t, err)
now = dbnow(clock)
expectedDeletesAt := oldKey.ExpiresAt(keyDuration).Add(WorkspaceAppsTokenDuration + time.Hour)
// Fetch the old key, it should have an deletes_at now.
oldKey, err = db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
Feature: oldKey.Feature,
Sequence: oldKey.Sequence,
})
require.NoError(t, err)
require.Equal(t, oldKey.DeletesAt.Time.UTC(), expectedDeletesAt)
// The new key should be created and have a starts_at of the old key's expires_at.
newKey, err := db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
Feature: database.CryptoKeyFeatureWorkspaceApps,
Sequence: oldKey.Sequence + 1,
})
require.NoError(t, err)
requireKey(t, newKey, database.CryptoKeyFeatureWorkspaceApps, oldKey.ExpiresAt(keyDuration), nullTime, oldKey.Sequence+1)
// Advance the clock just before the keys delete time.
clock.Advance(oldKey.DeletesAt.Time.UTC().Sub(now) - time.Second)
// No action should be taken.
err = kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 2)
// Advance the clock just past the keys delete time.
clock.Advance(oldKey.DeletesAt.Time.UTC().Sub(now) + time.Second)
// We should have deleted the old key.
err = kr.rotateKeys(ctx)
require.NoError(t, err)
// The old key should be "deleted".
_, err = db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
Feature: oldKey.Feature,
Sequence: oldKey.Sequence,
})
require.ErrorIs(t, err, sql.ErrNoRows)
keys, err = db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 1)
require.Equal(t, newKey, keys[0])
})
t.Run("DoesNotRotateValidKeys", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 7
logger = slogtest.Make(t, nil).Leveled(slog.LevelDebug)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{
database.CryptoKeyFeatureWorkspaceApps,
},
}
now := dbnow(clock)
// Seed the database with an existing key
existingKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceApps,
StartsAt: now,
Sequence: 123,
})
// Advance the clock by 6 days, 22 hours. Once we
// breach the last hour we will insert a new key.
clock.Advance(keyDuration - 2*time.Hour)
err := kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 1)
require.Equal(t, existingKey, keys[0])
// Advance it again to just before the key is scheduled to be rotated for sanity purposes.
clock.Advance(time.Hour - time.Second)
err = kr.rotateKeys(ctx)
require.NoError(t, err)
// Verify that the existing key is still the only key in the database
keys, err = db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 1)
requireKey(t, keys[0], existingKey.Feature, existingKey.StartsAt.UTC(), nullTime, existingKey.Sequence)
})
// Simulate a situation where the database was manually altered such that we only have a key that is scheduled to be deleted and assert we insert a new key.
t.Run("DeletesExpiredKeys", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 7
logger = slogtest.Make(t, nil).Leveled(slog.LevelDebug)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{
database.CryptoKeyFeatureWorkspaceApps,
},
}
now := dbnow(clock)
// Seed the database with an existing key
deletingKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceApps,
StartsAt: now.Add(-keyDuration),
Sequence: 789,
DeletesAt: sql.NullTime{
Time: now,
Valid: true,
},
})
err := kr.rotateKeys(ctx)
require.NoError(t, err)
// We should only get one key since the old key
// should be deleted.
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 1)
requireKey(t, keys[0], deletingKey.Feature, deletingKey.DeletesAt.Time.UTC(), nullTime, deletingKey.Sequence+1)
// The old key should be "deleted".
_, err = db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
Feature: deletingKey.Feature,
Sequence: deletingKey.Sequence,
})
require.ErrorIs(t, err, sql.ErrNoRows)
})
// This tests a situation where we have a key scheduled for deletion but it's still valid for use.
// If no other key is detected we should insert a new key.
t.Run("AddsKeyForDeletingKey", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 7
logger = slogtest.Make(t, nil).Leveled(slog.LevelDebug)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{
database.CryptoKeyFeatureWorkspaceApps,
},
}
now := dbnow(clock)
// Seed the database with an existing key
deletingKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceApps,
StartsAt: now,
Sequence: 456,
DeletesAt: sql.NullTime{
Time: now.Add(time.Hour),
Valid: true,
},
})
// We should only have inserted a key.
err := kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 2)
oldKey, newKey := keys[0], keys[1]
if oldKey.Sequence != deletingKey.Sequence {
oldKey, newKey = newKey, oldKey
}
requireKey(t, oldKey, deletingKey.Feature, deletingKey.StartsAt.UTC(), deletingKey.DeletesAt, deletingKey.Sequence)
requireKey(t, newKey, deletingKey.Feature, now, nullTime, deletingKey.Sequence+1)
})
t.Run("NoKeys", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 7
logger = slogtest.Make(t, nil).Leveled(slog.LevelDebug)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{
database.CryptoKeyFeatureWorkspaceApps,
},
}
err := kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 1)
requireKey(t, keys[0], database.CryptoKeyFeatureWorkspaceApps, clock.Now().UTC(), nullTime, 1)
})
// Assert we insert a new key when the only key was manually deleted.
t.Run("OnlyDeletedKeys", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 7
logger = slogtest.Make(t, nil).Leveled(slog.LevelDebug)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{
database.CryptoKeyFeatureWorkspaceApps,
},
}
now := dbnow(clock)
deletedkey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceApps,
StartsAt: now,
Sequence: 19,
DeletesAt: sql.NullTime{
Time: now.Add(time.Hour),
Valid: true,
},
Secret: sql.NullString{
String: "deleted",
Valid: false,
},
})
err := kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 1)
requireKey(t, keys[0], database.CryptoKeyFeatureWorkspaceApps, now, nullTime, deletedkey.Sequence+1)
})
// This tests ensures that rotation works with multiple
// features. It's mainly a sanity test since some bugs
// are not unveiled in the simple n=1 case.
t.Run("AllFeatures", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 30
logger = slogtest.Make(t, nil).Leveled(slog.LevelDebug)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: database.AllCryptoKeyFeatureValues(),
}
now := dbnow(clock)
// We'll test a scenario where one feature has no valid keys.
// Another has a key that should be rotate. And one that
// has a valid key that shouldn't trigger an action.
_ = dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureTailnetResume,
StartsAt: now.Add(-keyDuration),
Sequence: 5,
Secret: sql.NullString{
String: "older key",
Valid: false,
},
})
deletedKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureTailnetResume,
StartsAt: now.Add(-keyDuration),
Sequence: 19,
Secret: sql.NullString{
String: "old key",
Valid: false,
},
})
// Insert a key that should be rotated.
rotatedKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceApps,
StartsAt: now.Add(-keyDuration + time.Hour),
Sequence: 42,
})
// Insert a key that should not trigger an action.
validKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureOidcConvert,
StartsAt: now,
Sequence: 17,
})
err := kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 4)
kbf, err := keysByFeature(keys, database.AllCryptoKeyFeatureValues())
require.NoError(t, err)
// No actions on OIDC convert.
require.Len(t, kbf[database.CryptoKeyFeatureOidcConvert], 1)
// Workspace apps should have been rotated.
require.Len(t, kbf[database.CryptoKeyFeatureWorkspaceApps], 2)
// No existing key for tailnet resume should've
// caused a key to be inserted.
require.Len(t, kbf[database.CryptoKeyFeatureTailnetResume], 1)
oidcKey := kbf[database.CryptoKeyFeatureOidcConvert][0]
tailnetKey := kbf[database.CryptoKeyFeatureTailnetResume][0]
requireKey(t, oidcKey, database.CryptoKeyFeatureOidcConvert, now, nullTime, validKey.Sequence)
requireKey(t, tailnetKey, database.CryptoKeyFeatureTailnetResume, now, nullTime, deletedKey.Sequence+1)
newKey := kbf[database.CryptoKeyFeatureWorkspaceApps][0]
oldKey := kbf[database.CryptoKeyFeatureWorkspaceApps][1]
if newKey.Sequence == rotatedKey.Sequence {
oldKey, newKey = newKey, oldKey
}
deletesAt := sql.NullTime{
Time: rotatedKey.ExpiresAt(keyDuration).Add(WorkspaceAppsTokenDuration + time.Hour),
Valid: true,
}
requireKey(t, oldKey, database.CryptoKeyFeatureWorkspaceApps, rotatedKey.StartsAt.UTC(), deletesAt, rotatedKey.Sequence)
requireKey(t, newKey, database.CryptoKeyFeatureWorkspaceApps, rotatedKey.ExpiresAt(keyDuration), nullTime, rotatedKey.Sequence+1)
})
t.Run("UnknownFeature", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 7
logger = slogtest.Make(t, nil).Leveled(slog.LevelDebug)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{database.CryptoKeyFeature("unknown")},
}
err := kr.rotateKeys(ctx)
require.Error(t, err)
})
t.Run("MinStartsAt", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 5
logger = slogtest.Make(t, nil).Leveled(slog.LevelDebug)
ctx = testutil.Context(t, testutil.WaitShort)
)
now := dbnow(clock)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{database.CryptoKeyFeatureWorkspaceApps},
}
expiringKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceApps,
StartsAt: now.Add(-keyDuration),
Sequence: 345,
})
err := kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 2)
rotatedKey, err := db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
Feature: expiringKey.Feature,
Sequence: expiringKey.Sequence + 1,
})
require.NoError(t, err)
require.Equal(t, now.Add(defaultRotationInterval*3), rotatedKey.StartsAt.UTC())
})
// Test that the the deletes_at of a key that is well past its expiration
// Has its deletes_at field set to value that is relative
// to the current time to afford propagation time for the
// new key.
t.Run("ExtensivelyExpiredKey", func(t *testing.T) {
t.Parallel()
var (
db, _ = dbtestutil.NewDB(t)
clock = quartz.NewMock(t)
keyDuration = time.Hour * 24 * 3
logger = slogtest.Make(t, nil).Leveled(slog.LevelDebug)
ctx = testutil.Context(t, testutil.WaitShort)
)
kr := &rotator{
db: db,
keyDuration: keyDuration,
clock: clock,
logger: logger,
features: []database.CryptoKeyFeature{database.CryptoKeyFeatureWorkspaceApps},
}
now := dbnow(clock)
expiredKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceApps,
StartsAt: now.Add(-keyDuration - 2*time.Hour),
Sequence: 19,
})
deletedKey := dbgen.CryptoKey(t, db, database.CryptoKey{
Feature: database.CryptoKeyFeatureWorkspaceApps,
StartsAt: now,
Sequence: 20,
Secret: sql.NullString{
String: "deleted",
Valid: false,
},
})
err := kr.rotateKeys(ctx)
require.NoError(t, err)
keys, err := db.GetCryptoKeys(ctx)
require.NoError(t, err)
require.Len(t, keys, 2)
deletesAtKey, err := db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
Feature: expiredKey.Feature,
Sequence: expiredKey.Sequence,
})
deletesAt := sql.NullTime{
Time: now.Add(defaultRotationInterval * 3).Add(WorkspaceAppsTokenDuration + time.Hour),
Valid: true,
}
require.NoError(t, err)
requireKey(t, deletesAtKey, expiredKey.Feature, expiredKey.StartsAt.UTC(), deletesAt, expiredKey.Sequence)
newKey, err := db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
Feature: expiredKey.Feature,
Sequence: deletedKey.Sequence + 1,
})
require.NoError(t, err)
requireKey(t, newKey, expiredKey.Feature, now.Add(defaultRotationInterval*3), nullTime, deletedKey.Sequence+1)
})
}
func dbnow(c quartz.Clock) time.Time {
return dbtime.Time(c.Now().UTC())
}
func requireKey(t *testing.T, key database.CryptoKey, feature database.CryptoKeyFeature, startsAt time.Time, deletesAt sql.NullTime, sequence int32) {
t.Helper()
require.Equal(t, feature, key.Feature)
require.Equal(t, startsAt, key.StartsAt.UTC())
require.Equal(t, deletesAt.Valid, key.DeletesAt.Valid)
require.Equal(t, deletesAt.Time.UTC(), key.DeletesAt.Time.UTC())
require.Equal(t, sequence, key.Sequence)
secret, err := hex.DecodeString(key.Secret.String)
require.NoError(t, err)
switch key.Feature {
case database.CryptoKeyFeatureOidcConvert:
require.Len(t, secret, 64)
case database.CryptoKeyFeatureWorkspaceApps:
require.Len(t, secret, 32)
case database.CryptoKeyFeatureTailnetResume:
require.Len(t, secret, 64)
default:
t.Fatalf("unknown key feature: %s", key.Feature)
}
}
var nullTime = sql.NullTime{Time: time.Time{}, Valid: false}