fix(cli)!: protect client Logger and refactor cli scaletest tests (#8317)

- (breaking) Protects Logger and LogBodies fields of codersdk.Client with its mutex. This addresses a data race in cli/scaletest.
- Fillets the existing cli/createworkspaces unit test and moves the testing logic there into the tests under scaletest/createworkspaces.
- Adds testutil.RaceEnabled bool const and conditionaly skips previously-skipped tests under scaletest/ if the race detector is enabled. This is unfortunate and sad, but I would prefer to have these tests at least running without the race detector than not running at all.
- Adds IgnoreErrors option to fake in-memory agent loggers; having the agents fail the test immediately when they encounter any sort of error isn't really helpful.

coderd/deployment_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/coder/coder/coderd/coderdtest"
@@ -47,10 +48,13 @@ func TestDeploymentValues(t *testing.T) {
 
 func TestDeploymentStats(t *testing.T) {
 	t.Parallel()
+	t.Log("This test is time-sensitive. It may fail if the deployment is not ready in time.")
 	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
 	defer cancel()
 	client := coderdtest.New(t, &coderdtest.Options{})
 	_ = coderdtest.CreateFirstUser(t, client)
-	_, err := client.DeploymentStats(ctx)
-	require.NoError(t, err)
+	assert.True(t, testutil.Eventually(ctx, t, func(tctx context.Context) bool {
+		_, err := client.DeploymentStats(tctx)
+		return err == nil
+	}, testutil.IntervalMedium), "failed to get deployment stats in time")
 }