fix: add timeouts to test telemetry snapshot (#17879)

This PR ensures that waits on channels will time out according to the
test context, rather than waiting indefinitely. This should alleviate
the panic seen in https://github.com/coder/internal/issues/645 and, if
the deadlock recurs, allow the test to be retried automatically in CI.

testutil/chan.go

@@ -55,3 +55,61 @@ func RequireSend[A any](ctx context.Context, t testing.TB, c chan<- A, a A) {
 		// OK!
 	}
 }
+
+// SoftTryReceive will attempt to receive a value from the chan and return it. If
+// the context expires before a value can be received, it will mark the test as
+// failed but continue execution. If the channel is closed, the zero value of the
+// channel type will be returned.
+// The second return value indicates whether the receive was successful. In
+// particular, if the channel is closed, the second return value will be true.
+//
+// Safety: can be called from any goroutine.
+func SoftTryReceive[A any](ctx context.Context, t testing.TB, c <-chan A) (A, bool) {
+	t.Helper()
+	select {
+	case <-ctx.Done():
+		t.Error("timeout")
+		var a A
+		return a, false
+	case a := <-c:
+		return a, true
+	}
+}
+
+// AssertReceive will receive a value from the chan and return it. If the
+// context expires or the channel is closed before a value can be received,
+// it will mark the test as failed but continue execution.
+// The second return value indicates whether the receive was successful.
+//
+// Safety: can be called from any goroutine.
+func AssertReceive[A any](ctx context.Context, t testing.TB, c <-chan A) (A, bool) {
+	t.Helper()
+	select {
+	case <-ctx.Done():
+		t.Error("timeout")
+		var a A
+		return a, false
+	case a, ok := <-c:
+		if !ok {
+			t.Error("channel closed")
+		}
+		return a, ok
+	}
+}
+
+// AssertSend will send the given value over the chan and then return. If
+// the context expires before the send succeeds, it will mark the test as failed
+// but continue execution.
+// The second return value indicates whether the send was successful.
+//
+// Safety: can be called from any goroutine.
+func AssertSend[A any](ctx context.Context, t testing.TB, c chan<- A, a A) bool {
+	t.Helper()
+	select {
+	case <-ctx.Done():
+		t.Error("timeout")
+		return false
+	case c <- a:
+		return true
+	}
+}