fix: avoid race between replicas on start (#12344)

DERP mesh key setup would do a SELECT and then an INSERT on failure, without a lock. During some testing with multiple replicas, I managed to cause a replica to crash due to them initializing simultaneously.

Fixes:

Encountered an error running "coder server"
create coder API: insert mesh key: pq: duplicate key value violates unique constraint "site_configs_key_key"

Co-authored-by: Cian Johnston <cian@coder.com>

coderd/database/dbauthz/dbauthz_test.go

@@ -1874,6 +1874,7 @@ func (s *MethodTestSuite) TestSystemFunctions() {
 		check.Args(u.ID).Asserts(rbac.ResourceSystem, rbac.ActionRead)
 	}))
 	s.Run("GetDERPMeshKey", s.Subtest(func(db database.Store, check *expects) {
+		db.InsertDERPMeshKey(context.Background(), "testing")
 		check.Args().Asserts(rbac.ResourceSystem, rbac.ActionRead)
 	}))
 	s.Run("InsertDERPMeshKey", s.Subtest(func(db database.Store, check *expects) {

coderd/database/dbmem/dbmem.go

@@ -1761,6 +1761,9 @@ func (q *FakeQuerier) GetDERPMeshKey(_ context.Context) (string, error) {
 	q.mutex.RLock()
 	defer q.mutex.RUnlock()
 
+	if q.derpMeshKey == "" {
+		return "", sql.ErrNoRows
+	}
 	return q.derpMeshKey, nil
 }
 

coderd/database/lock.go

@@ -9,6 +9,7 @@ const (
 	// Keep the unused iota here so we don't need + 1 every time
 	lockIDUnused = iota
 	LockIDDeploymentSetup
+	LockIDEnterpriseDeploymentSetup
 )
 
 // GenLockID generates a unique and consistent lock ID from a given string.