feat(cli): add DataDog Go tracer (#9411)

coderd/tracing/exporter.go

@@ -13,6 +13,10 @@ import (
 	"go.opentelemetry.io/otel/sdk/resource"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	semconv "go.opentelemetry.io/otel/semconv/v1.14.0"
+	ddotel "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentelemetry"
+	ddtracer "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
+	ddprofiler "gopkg.in/DataDog/dd-trace-go.v1/profiler"
+
 	"golang.org/x/xerrors"
 	"google.golang.org/grpc/credentials"
 )
@@ -25,6 +29,8 @@ type TracerOpts struct {
 	// Coder exports traces to Coder's public tracing ingest service and is used
 	// to improve the product. It is disabled when opting out of telemetry.
 	Coder bool
+	// DataDog exports traces and profiles to the local DataDog daemon.
+	DataDog bool
 	// Exports traces to Honeycomb.io with the provided API key.
 	Honeycomb string
 }
@@ -45,6 +51,35 @@ func TracerProvider(ctx context.Context, service string, opts TracerOpts) (*sdkt
 		closers = []func(context.Context) error{}
 	)
 
+	if opts.DataDog {
+		// See more:
+		// https://docs.datadoghq.com/tracing/metrics/runtime_metrics/go/
+		dd := ddotel.NewTracerProvider(ddtracer.WithRuntimeMetrics())
+		closers = append(closers, func(_ context.Context) error {
+			// For some reason, this doesn't appear to actually wind down
+			// the goroutines.
+			return dd.Shutdown()
+		})
+
+		// See https://docs.datadoghq.com/profiler/enabling/go/
+		_ = ddprofiler.Start(
+			ddprofiler.WithService("coderd"),
+			ddprofiler.WithProfileTypes(
+				ddprofiler.CPUProfile,
+				ddprofiler.HeapProfile,
+				ddprofiler.GoroutineProfile,
+
+				// In the future, we may want to enable:
+				// ddprofiler.BlockProfile,
+				// ddprofiler.MutexProfile,
+			),
+		)
+		closers = append(closers, func(_ context.Context) error {
+			ddprofiler.Stop()
+			return nil
+		})
+	}
+
 	if opts.Default {
 		exporter, err := DefaultExporter(ctx)
 		if err != nil {