chore(cli): refactor TestServer/Prometheus to use testutil.Eventually (#17295)

Updates https://github.com/coder/internal/issues/282

Refactors existing tests to use `testutil.Eventually` which plays nicer
with `testutil.Context`.

cli/server_test.go

@@ -1208,7 +1208,7 @@ func TestServer(t *testing.T) {
 				}
 			}
 			return htmlFirstServedFound
-		}, testutil.WaitMedium, testutil.IntervalFast, "no html_first_served telemetry item")
+		}, testutil.WaitLong, testutil.IntervalSlow, "no html_first_served telemetry item")
 	})
 	t.Run("Prometheus", func(t *testing.T) {
 		t.Parallel()
@@ -1216,9 +1216,7 @@ func TestServer(t *testing.T) {
 		t.Run("DBMetricsDisabled", func(t *testing.T) {
 			t.Parallel()
 
-			ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
+			ctx := testutil.Context(t, testutil.WaitLong)
-			defer cancel()
-
 			randPort := testutil.RandomPort(t)
 			inv, cfg := clitest.New(t,
 				"server",
@@ -1235,46 +1233,45 @@ func TestServer(t *testing.T) {
 			clitest.Start(t, inv)
 			_ = waitAccessURL(t, cfg)
 
-			var res *http.Response
+			testutil.Eventually(ctx, t, func(ctx context.Context) bool {
-			require.Eventually(t, func() bool {
+				req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://127.0.0.1:%d/metrics", randPort), nil)
-				req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://127.0.0.1:%d", randPort), nil)
-				assert.NoError(t, err)
-				// nolint:bodyclose
-				res, err = http.DefaultClient.Do(req)
 				if err != nil {
+					t.Logf("error creating request: %s", err.Error())
+					return false
+				}
+				// nolint:bodyclose
+				res, err := http.DefaultClient.Do(req)
+				if err != nil {
+					t.Logf("error hitting prometheus endpoint: %s", err.Error())
 					return false
 				}
 				defer res.Body.Close()
-
 				scanner := bufio.NewScanner(res.Body)
-				hasActiveUsers := false
+				var activeUsersFound bool
+				var scannedOnce bool
 				for scanner.Scan() {
+					line := scanner.Text()
+					if !scannedOnce {
+						t.Logf("scanned: %s", line) // avoid spamming logs
+						scannedOnce = true
+					}
+					if strings.HasPrefix(line, "coderd_db_query_latencies_seconds") {
+						t.Errorf("db metrics should not be tracked when --prometheus-collect-db-metrics is not enabled")
+					}
 					// This metric is manually registered to be tracked in the server. That's
 					// why we test it's tracked here.
-					if strings.HasPrefix(scanner.Text(), "coderd_api_active_users_duration_hour") {
+					if strings.HasPrefix(line, "coderd_api_active_users_duration_hour") {
-						hasActiveUsers = true
+						activeUsersFound = true
-						continue
 					}
-					if strings.HasPrefix(scanner.Text(), "coderd_db_query_latencies_seconds") {
-						t.Fatal("db metrics should not be tracked when --prometheus-collect-db-metrics is not enabled")
-					}
-					t.Logf("scanned %s", scanner.Text())
 				}
-				if scanner.Err() != nil {
+				return activeUsersFound
-					t.Logf("scanner err: %s", scanner.Err().Error())
+			}, testutil.IntervalSlow, "didn't find coderd_api_active_users_duration_hour in time")
-					return false
-				}
-
-				return hasActiveUsers
-			}, testutil.WaitShort, testutil.IntervalFast, "didn't find coderd_api_active_users_duration_hour in time")
 		})
 
 		t.Run("DBMetricsEnabled", func(t *testing.T) {
 			t.Parallel()
 
-			ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
+			ctx := testutil.Context(t, testutil.WaitLong)
-			defer cancel()
-
 			randPort := testutil.RandomPort(t)
 			inv, cfg := clitest.New(t,
 				"server",
@@ -1291,31 +1288,34 @@ func TestServer(t *testing.T) {
 			clitest.Start(t, inv)
 			_ = waitAccessURL(t, cfg)
 
-			var res *http.Response
+			testutil.Eventually(ctx, t, func(ctx context.Context) bool {
-			require.Eventually(t, func() bool {
+				req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://127.0.0.1:%d/metrics", randPort), nil)
-				req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://127.0.0.1:%d", randPort), nil)
-				assert.NoError(t, err)
-				// nolint:bodyclose
-				res, err = http.DefaultClient.Do(req)
 				if err != nil {
+					t.Logf("error creating request: %s", err.Error())
+					return false
+				}
+				// nolint:bodyclose
+				res, err := http.DefaultClient.Do(req)
+				if err != nil {
+					t.Logf("error hitting prometheus endpoint: %s", err.Error())
 					return false
 				}
 				defer res.Body.Close()
-
 				scanner := bufio.NewScanner(res.Body)
-				hasDBMetrics := false
+				var dbMetricsFound bool
+				var scannedOnce bool
 				for scanner.Scan() {
-					if strings.HasPrefix(scanner.Text(), "coderd_db_query_latencies_seconds") {
+					line := scanner.Text()
-						hasDBMetrics = true
+					if !scannedOnce {
+						t.Logf("scanned: %s", line) // avoid spamming logs
+						scannedOnce = true
+					}
+					if strings.HasPrefix(line, "coderd_db_query_latencies_seconds") {
+						dbMetricsFound = true
 					}
-					t.Logf("scanned %s", scanner.Text())
 				}
-				if scanner.Err() != nil {
+				return dbMetricsFound
-					t.Logf("scanner err: %s", scanner.Err().Error())
+			}, testutil.IntervalSlow, "didn't find coderd_db_query_latencies_seconds in time")
-					return false
-				}
-				return hasDBMetrics
-			}, testutil.WaitShort, testutil.IntervalFast, "didn't find coderd_db_query_latencies_seconds in time")
 		})
 	})
 	t.Run("GitHubOAuth", func(t *testing.T) {