feat: Add database fixtures for testing migrations (#4858)

coderd/database/migrations/create_fixture.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# Naming the fixture is optional, if missing, the name of the latest
+# migration will be used.
+#
+# Usage:
+# ./create_fixture
+# ./create_fixture name of fixture
+# ./create_fixture "name of fixture"
+# ./create_fixture name_of_fixture
+
+set -euo pipefail
+
+SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
+(
+	cd "$SCRIPT_DIR"
+
+	latest_migration=$(basename "$(find . -maxdepth 1 -name "*.up.sql" | sort -n | tail -n 1)")
+	if [[ -n "${*}" ]]; then
+		name=$*
+		name=${name// /_}
+		num=${latest_migration%%_*}
+		latest_migration="${num}_${name}.up.sql"
+	fi
+
+	filename="$(pwd)/testdata/fixtures/$latest_migration"
+	touch "$filename"
+	echo "$filename"
+	echo "Edit fixture and commit it."
+)

coderd/database/migrations/migrate.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"embed"
 	"errors"
+	"io/fs"
 	"os"
 
 	"github.com/golang-migrate/migrate/v4"
@@ -160,3 +161,52 @@ func CheckLatestVersion(sourceDriver source.Driver, currentVersion uint) error {
 	}
 	return nil
 }
+
+// Stepper returns a function that runs SQL migrations one step at a time.
+//
+// Stepper cannot be closed pre-emptively, it must be run to completion
+// (or until an error is encountered).
+func Stepper(db *sql.DB) (next func() (version uint, more bool, err error), err error) {
+	_, m, err := setup(db)
+	if err != nil {
+		return nil, xerrors.Errorf("migrate setup: %w", err)
+	}
+
+	return func() (version uint, more bool, err error) {
+		defer func() {
+			if !more {
+				srcErr, dbErr := m.Close()
+				if err != nil {
+					return
+				}
+				if dbErr != nil {
+					err = dbErr
+					return
+				}
+				err = srcErr
+			}
+		}()
+
+		err = m.Steps(1)
+		if err != nil {
+			switch {
+			case errors.Is(err, migrate.ErrNoChange):
+				// It's OK if no changes happened!
+				return 0, false, nil
+			case errors.Is(err, fs.ErrNotExist):
+				// This error is encountered at the of Steps when
+				// reading from embed.FS.
+				return 0, false, nil
+			}
+
+			return 0, false, xerrors.Errorf("Step: %w", err)
+		}
+
+		v, _, err := m.Version()
+		if err != nil {
+			return 0, false, err
+		}
+
+		return v, true, nil
+	}, nil
+}

coderd/database/migrations/migrate_test.go

@@ -3,17 +3,27 @@
 package migrations_test
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
 	"testing"
 
+	"github.com/golang-migrate/migrate/v4"
+	migratepostgres "github.com/golang-migrate/migrate/v4/database/postgres"
 	"github.com/golang-migrate/migrate/v4/source"
+	"github.com/golang-migrate/migrate/v4/source/iofs"
 	"github.com/golang-migrate/migrate/v4/source/stub"
+	"github.com/lib/pq"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
+	"golang.org/x/exp/slices"
 
 	"github.com/coder/coder/coderd/database/migrations"
 	"github.com/coder/coder/coderd/database/postgres"
+	"github.com/coder/coder/testutil"
 )
 
 func TestMain(m *testing.M) {
@@ -129,3 +139,196 @@ func TestCheckLatestVersion(t *testing.T) {
 		})
 	}
 }
+
+func setupMigrate(t *testing.T, db *sql.DB, name, path string) (source.Driver, *migrate.Migrate) {
+	t.Helper()
+
+	ctx := context.Background()
+
+	conn, err := db.Conn(ctx)
+	require.NoError(t, err)
+
+	dbDriver, err := migratepostgres.WithConnection(ctx, conn, &migratepostgres.Config{
+		MigrationsTable: "test_migrate_" + name,
+	})
+	require.NoError(t, err)
+
+	dirFS := os.DirFS(path)
+	d, err := iofs.New(dirFS, ".")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		d.Close()
+	})
+
+	m, err := migrate.NewWithInstance(name, d, "", dbDriver)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		m.Close()
+	})
+
+	return d, m
+}
+
+type tableStats struct {
+	mu sync.Mutex
+	s  map[string]int
+}
+
+func (s *tableStats) Add(table string, n int) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.s[table] = s.s[table] + n
+}
+
+func (s *tableStats) Empty() []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	var m []string
+	for table, n := range s.s {
+		if n == 0 {
+			m = append(m, table)
+		}
+	}
+	return m
+}
+
+func TestMigrateUpWithFixtures(t *testing.T) {
+	t.Parallel()
+
+	if testing.Short() {
+		t.Skip()
+		return
+	}
+
+	type testCase struct {
+		name string
+		path string
+
+		// For determining if test case table stats
+		// are used to determine test coverage.
+		useStats bool
+	}
+	tests := []testCase{
+		{
+			name:     "fixtures",
+			path:     filepath.Join("testdata", "fixtures"),
+			useStats: true,
+		},
+		// More test cases added via glob below.
+	}
+
+	// Folders in testdata/full_dumps represent fixtures for a full
+	// deployment of Coder.
+	matches, err := filepath.Glob(filepath.Join("testdata", "full_dumps", "*"))
+	require.NoError(t, err)
+	for _, match := range matches {
+		tests = append(tests, testCase{
+			name:     filepath.Base(match),
+			path:     match,
+			useStats: true,
+		})
+	}
+
+	// These tables are allowed to have zero rows for now,
+	// but we should eventually add fixtures for them.
+	ignoredTablesForStats := []string{
+		"audit_logs",
+		"git_auth_links",
+		"group_members",
+		"licenses",
+		"replicas",
+	}
+	s := &tableStats{s: make(map[string]int)}
+
+	// This will run after all subtests have run and fail the test if
+	// new tables have been added without covering them with fixtures.
+	t.Cleanup(func() {
+		emptyTables := s.Empty()
+		slices.Sort(emptyTables)
+		for _, table := range ignoredTablesForStats {
+			i := slices.Index(emptyTables, table)
+			if i >= 0 {
+				emptyTables = slices.Delete(emptyTables, i, i+1)
+			}
+		}
+		if len(emptyTables) > 0 {
+			t.Logf("The following tables have zero rows, consider adding fixtures for them or create a full database dump:")
+			t.Errorf("tables have zero rows: %v", emptyTables)
+			t.Logf("See https://github.com/coder/coder/blob/main/docs/CONTRIBUTING.md#database-fixtures-for-testing-migrations for more information")
+		}
+	})
+
+	for _, tt := range tests {
+		tt := tt
+
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			db := testSQLDB(t)
+
+			ctx, _ := testutil.Context(t)
+
+			// Prepare database for stepping up.
+			err := migrations.Down(db)
+			require.NoError(t, err)
+
+			// Initialize migrations for fixtures.
+			fDriver, fMigrate := setupMigrate(t, db, tt.name, tt.path)
+
+			nextStep, err := migrations.Stepper(db)
+			require.NoError(t, err)
+
+			var fixtureVer uint
+			nextFixtureVer, err := fDriver.First()
+			require.NoError(t, err)
+
+			for {
+				version, more, err := nextStep()
+				require.NoError(t, err)
+
+				if !more {
+					// We reached the end of the migrations.
+					break
+				}
+
+				if nextFixtureVer == version {
+					err = fMigrate.Steps(1)
+					require.NoError(t, err)
+					fixtureVer = version
+
+					nv, _ := fDriver.Next(nextFixtureVer)
+					if nv > 0 {
+						nextFixtureVer = nv
+					}
+				}
+
+				t.Logf("migrated to version %d, fixture version %d", version, fixtureVer)
+			}
+
+			// Gather number of rows for all existing tables
+			// at the end of the migrations and fixtures.
+			var tables pq.StringArray
+			err = db.QueryRowContext(ctx, `
+				SELECT array_agg(tablename)
+				FROM pg_catalog.pg_tables
+				WHERE
+					schemaname != 'information_schema'
+					AND schemaname != 'pg_catalog'
+					AND tablename NOT LIKE 'test_migrate_%'
+			`).Scan(&tables)
+			require.NoError(t, err)
+
+			for _, table := range tables {
+				var count int
+				err = db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table).Scan(&count)
+				require.NoError(t, err)
+
+				if tt.useStats {
+					s.Add(table, count)
+				}
+			}
+		})
+	}
+}

coderd/database/migrations/testdata/fixtures/000024_example.up.sql

@@ -0,0 +1,6 @@
+-- Example:
+-- This fixture is applied after migrations/000024_site_config.up.sql
+-- and inserts a value into site_configs that must not cause issues in
+-- future migrations.
+
+INSERT INTO site_configs (key, value) VALUES ('mytest', 'example');

docs/CONTRIBUTING.md

@@ -73,6 +73,77 @@ Use the following `make` commands and scripts in development:
 - `make install` installs binaries to `$GOPATH/bin`
 - `make test`
 
+### Adding database migrations and fixtures
+
+#### Database migrations
+
+Database migrations are managed with [`migrate`](https://github.com/golang-migrate/migrate).
+
+To add new migrations, use the following command:
+
+```
+$ ./coderd/database/migrations/create_fixture.sh my name
+/home/coder/src/coder/coderd/database/migrations/000070_my_name.up.sql
+/home/coder/src/coder/coderd/database/migrations/000070_my_name.down.sql
+Run "make gen" to generate models.
+```
+
+Then write queries into the generated `.up.sql` and `.down.sql` files and commit
+them into the repository. The down script should make a best-effort to retain as
+much data as possible.
+
+#### Database fixtures (for testing migrations)
+
+There are two types of fixtures that are used to test that migrations don't
+break existing Coder deployments:
+
+- Partial fixtures [`migrations/testdata/fixtures`](../coderd/database/migrations/testdata/fixtures)
+- Full database dumps [`migrations/testdata/full_dumps`](../coderd/database/migrations/testdata/full_dumps)
+
+Both types behave like database migrations (they also [`migrate`](https://github.com/golang-migrate/migrate)). Their behavior mirrors Coder migrations such that when migration
+number `000022` is applied, fixture `000022` is applied afterwards.
+
+Partial fixtures are used to conveniently add data to newly created tables so
+that we can ensure that this data is migrated without issue.
+
+Full database dumps are for testing the migration of fully-fledged Coder
+deployments. These are usually done for a specific version of Coder and are
+often fixed in time. A full database dump may be necessary when testing the
+migration of multiple features or complex configurations.
+
+To add a new partial fixture, run the following command:
+
+```
+$ ./coderd/database/migrations/create_fixture.sh my fixture
+/home/coder/src/coder/coderd/database/migrations/testdata/fixtures/000070_my_fixture.up.sql
+```
+
+Then add some queries to insert data and commit the file to the repo. See
+[`000024_example.up.sql`](../coderd/database/migrations/testdata/fixtures/000024_example.up.sql)
+for an example.
+
+To create a full dump, run a fully fledged Coder deployment and use it to
+generate data in the database. Then shut down the deployment and take a snapshot
+of the database.
+
+```
+$ mkdir -p coderd/database/migrations/testdata/full_dumps/v0.12.2 && cd $_
+$ pg_dump "postgres://coder@localhost:..." -a --inserts >000069_dump_v0.12.2.up.sql
+```
+
+Make sure sensitive data in the dump is desensitized, for instance names,
+emails, OAuth tokens and other secrets. Then commit the dump to the project.
+
+To find out what the latest migration for a version of Coder is, use the
+following command:
+
+```
+$ git ls-files v0.12.2 -- coderd/database/migrations/*.up.sql
+```
+
+This helps in naming the dump (e.g. `000069` above).
+
+
 ## Styling
 
 ### Documentation