feat: loadtest output formats (#4928)

cli/loadtest.go

@@ -1,14 +1,13 @@
 package cli
 
 import (
-	"bufio"
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -21,44 +20,46 @@ import (
 
 func loadtest() *cobra.Command {
 	var (
-		configPath string
+		configPath  string
+		outputSpecs []string
 	)
 	cmd := &cobra.Command{
-		Use:   "loadtest --config <path>",
+		Use:   "loadtest --config <path> [--output json[:path]] [--output text[:path]]]",
 		Short: "Load test the Coder API",
-		// TODO: documentation and a JSON scheme file
+		// TODO: documentation and a JSON schema file
-		Long: "Perform load tests against the Coder server. The load tests " +
+		Long: "Perform load tests against the Coder server. The load tests are configurable via a JSON file.",
-			"configurable via a JSON file.",
+		Example: formatExamples(
+			example{
+				Description: "Run a loadtest with the given configuration file",
+				Command:     "coder loadtest --config path/to/config.json",
+			},
+			example{
+				Description: "Run a loadtest, reading the configuration from stdin",
+				Command:     "cat path/to/config.json | coder loadtest --config -",
+			},
+			example{
+				Description: "Run a loadtest outputting JSON results instead",
+				Command:     "coder loadtest --config path/to/config.json --output json",
+			},
+			example{
+				Description: "Run a loadtest outputting JSON results to a file",
+				Command:     "coder loadtest --config path/to/config.json --output json:path/to/results.json",
+			},
+			example{
+				Description: "Run a loadtest outputting text results to stdout and JSON results to a file",
+				Command:     "coder loadtest --config path/to/config.json --output text --output json:path/to/results.json",
+			},
+		),
 		Hidden: true,
 		Args:   cobra.ExactArgs(0),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			if configPath == "" {
+			config, err := loadLoadTestConfigFile(configPath, cmd.InOrStdin())
-				return xerrors.New("config is required")
-			}
-
-			var (
-				configReader io.ReadCloser
-			)
-			if configPath == "-" {
-				configReader = io.NopCloser(cmd.InOrStdin())
-			} else {
-				f, err := os.Open(configPath)
-				if err != nil {
-					return xerrors.Errorf("open config file %q: %w", configPath, err)
-				}
-				configReader = f
-			}
-
-			var config LoadTestConfig
-			err := json.NewDecoder(configReader).Decode(&config)
-			_ = configReader.Close()
 			if err != nil {
-				return xerrors.Errorf("read config file %q: %w", configPath, err)
+				return err
 			}
-
+			outputs, err := parseLoadTestOutputs(outputSpecs)
-			err = config.Validate()
 			if err != nil {
-				return xerrors.Errorf("validate config: %w", err)
+				return err
 			}
 
 			client, err := CreateClient(cmd)
@@ -117,52 +118,44 @@ func loadtest() *cobra.Command {
 			}
 
 			// TODO: live progress output
-			start := time.Now()
 			err = th.Run(testCtx)
 			if err != nil {
 				return xerrors.Errorf("run test harness (harness failure, not a test failure): %w", err)
 			}
-			elapsed := time.Since(start)
 
 			// Print the results.
-			// TODO: better result printing
-			// TODO: move result printing to the loadtest package, add multiple
-			//       output formats (like HTML, JSON)
 			res := th.Results()
-			var totalDuration time.Duration
+			for _, output := range outputs {
-			for _, run := range res.Runs {
+				var (
-				totalDuration += run.Duration
+					w = cmd.OutOrStdout()
-				if run.Error == nil {
+					c io.Closer
-					continue
+				)
+				if output.path != "-" {
+					f, err := os.Create(output.path)
+					if err != nil {
+						return xerrors.Errorf("create output file: %w", err)
+					}
+					w, c = f, f
 				}
 
-				_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\n== FAIL: %s\n\n", run.FullID)
+				switch output.format {
-				_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tError: %s\n\n", run.Error)
+				case loadTestOutputFormatText:
-
+					res.PrintText(w)
-				// Print log lines indented.
+				case loadTestOutputFormatJSON:
-				_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tLog:\n")
+					err = json.NewEncoder(w).Encode(res)
-				rd := bufio.NewReader(bytes.NewBuffer(run.Logs))
-				for {
-					line, err := rd.ReadBytes('\n')
-					if err == io.EOF {
-						break
-					}
 					if err != nil {
-						_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\n\tLOG PRINT ERROR: %+v\n", err)
+						return xerrors.Errorf("encode JSON: %w", err)
 					}
+				}
 
-					_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\t\t%s", line)
+				if c != nil {
+					err = c.Close()
+					if err != nil {
+						return xerrors.Errorf("close output file: %w", err)
+					}
 				}
 			}
 
-			_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\n\nTest results:")
-			_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tPass:  %d\n", res.TotalPass)
-			_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tFail:  %d\n", res.TotalFail)
-			_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tTotal: %d\n", res.TotalRuns)
-			_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "")
-			_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tTotal duration: %s\n", elapsed)
-			_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tAvg. duration:  %s\n", totalDuration/time.Duration(res.TotalRuns))
-
 			// Cleanup.
 			_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\nCleaning up...")
 			err = th.Cleanup(cmd.Context())
@@ -170,10 +163,111 @@ func loadtest() *cobra.Command {
 				return xerrors.Errorf("cleanup tests: %w", err)
 			}
 
+			if res.TotalFail > 0 {
+				return xerrors.New("load test failed, see above for more details")
+			}
+
 			return nil
 		},
 	}
 
 	cliflag.StringVarP(cmd.Flags(), &configPath, "config", "", "CODER_LOADTEST_CONFIG_PATH", "", "Path to the load test configuration file, or - to read from stdin.")
+	cliflag.StringArrayVarP(cmd.Flags(), &outputSpecs, "output", "", "CODER_LOADTEST_OUTPUTS", []string{"text"}, "Output formats, see usage for more information.")
 	return cmd
 }
+
+func loadLoadTestConfigFile(configPath string, stdin io.Reader) (LoadTestConfig, error) {
+	if configPath == "" {
+		return LoadTestConfig{}, xerrors.New("config is required")
+	}
+
+	var (
+		configReader io.ReadCloser
+	)
+	if configPath == "-" {
+		configReader = io.NopCloser(stdin)
+	} else {
+		f, err := os.Open(configPath)
+		if err != nil {
+			return LoadTestConfig{}, xerrors.Errorf("open config file %q: %w", configPath, err)
+		}
+		configReader = f
+	}
+
+	var config LoadTestConfig
+	err := json.NewDecoder(configReader).Decode(&config)
+	_ = configReader.Close()
+	if err != nil {
+		return LoadTestConfig{}, xerrors.Errorf("read config file %q: %w", configPath, err)
+	}
+
+	err = config.Validate()
+	if err != nil {
+		return LoadTestConfig{}, xerrors.Errorf("validate config: %w", err)
+	}
+
+	return config, nil
+}
+
+type loadTestOutputFormat string
+
+const (
+	loadTestOutputFormatText loadTestOutputFormat = "text"
+	loadTestOutputFormatJSON loadTestOutputFormat = "json"
+	// TODO: html format
+)
+
+type loadTestOutput struct {
+	format loadTestOutputFormat
+	// Up to one path (the first path) will have the value "-" which signifies
+	// stdout.
+	path string
+}
+
+func parseLoadTestOutputs(outputs []string) ([]loadTestOutput, error) {
+	var stdoutFormat loadTestOutputFormat
+
+	validFormats := map[loadTestOutputFormat]struct{}{
+		loadTestOutputFormatText: {},
+		loadTestOutputFormatJSON: {},
+	}
+
+	var out []loadTestOutput
+	for i, o := range outputs {
+		parts := strings.SplitN(o, ":", 2)
+		format := loadTestOutputFormat(parts[0])
+		if _, ok := validFormats[format]; !ok {
+			return nil, xerrors.Errorf("invalid output format %q in output flag %d", parts[0], i)
+		}
+
+		if len(parts) == 1 {
+			if stdoutFormat != "" {
+				return nil, xerrors.Errorf("multiple output flags specified for stdout")
+			}
+			stdoutFormat = format
+			continue
+		}
+		if len(parts) != 2 {
+			return nil, xerrors.Errorf("invalid output flag %d: %q", i, o)
+		}
+
+		out = append(out, loadTestOutput{
+			format: format,
+			path:   parts[1],
+		})
+	}
+
+	// Default to --output text
+	if stdoutFormat == "" && len(out) == 0 {
+		stdoutFormat = loadTestOutputFormatText
+	}
+
+	if stdoutFormat != "" {
+		out = append([]loadTestOutput{{
+			format: stdoutFormat,
+			path:   "-",
+		}}, out...)
+	}
+
+	return out, nil
+}

cli/loadtest_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
@@ -17,6 +18,7 @@ import (
 	"github.com/coder/coder/coderd/coderdtest"
 	"github.com/coder/coder/coderd/httpapi"
 	"github.com/coder/coder/codersdk"
+	"github.com/coder/coder/loadtest/harness"
 	"github.com/coder/coder/loadtest/placebo"
 	"github.com/coder/coder/loadtest/workspacebuild"
 	"github.com/coder/coder/pty/ptytest"
@@ -133,4 +135,161 @@ func TestLoadTest(t *testing.T) {
 		<-done
 		cancelFunc()
 	})
+
+	t.Run("OutputFormats", func(t *testing.T) {
+		t.Parallel()
+
+		type outputFlag struct {
+			format string
+			path   string
+		}
+
+		dir := t.TempDir()
+
+		cases := []struct {
+			name        string
+			outputs     []outputFlag
+			errContains string
+		}{
+			{
+				name:    "Default",
+				outputs: []outputFlag{},
+			},
+			{
+				name:    "ExplicitText",
+				outputs: []outputFlag{{format: "text"}},
+			},
+			{
+				name: "JSON",
+				outputs: []outputFlag{
+					{
+						format: "json",
+						path:   filepath.Join(dir, "results.json"),
+					},
+				},
+			},
+			{
+				name: "TextAndJSON",
+				outputs: []outputFlag{
+					{
+						format: "text",
+					},
+					{
+						format: "json",
+						path:   filepath.Join(dir, "results.json"),
+					},
+				},
+			},
+			{
+				name: "TextAndJSON2",
+				outputs: []outputFlag{
+					{
+						format: "text",
+					},
+					{
+						format: "text",
+						path:   filepath.Join(dir, "results.txt"),
+					},
+					{
+						format: "json",
+						path:   filepath.Join(dir, "results.json"),
+					},
+				},
+			},
+		}
+
+		for _, c := range cases {
+			t.Run(c.name, func(t *testing.T) {
+				t.Parallel()
+
+				client := coderdtest.New(t, nil)
+				_ = coderdtest.CreateFirstUser(t, client)
+
+				config := cli.LoadTestConfig{
+					Strategy: cli.LoadTestStrategy{
+						Type: cli.LoadTestStrategyTypeLinear,
+					},
+					Tests: []cli.LoadTest{
+						{
+							Type:  cli.LoadTestTypePlacebo,
+							Count: 10,
+							Placebo: &placebo.Config{
+								Sleep: httpapi.Duration(10 * time.Millisecond),
+							},
+						},
+					},
+					Timeout: httpapi.Duration(testutil.WaitShort),
+				}
+
+				configBytes, err := json.Marshal(config)
+				require.NoError(t, err)
+
+				args := []string{"loadtest", "--config", "-"}
+				for _, output := range c.outputs {
+					flag := output.format
+					if output.path != "" {
+						flag += ":" + output.path
+					}
+					args = append(args, "--output", flag)
+				}
+
+				cmd, root := clitest.New(t, args...)
+				clitest.SetupConfig(t, client, root)
+				cmd.SetIn(bytes.NewReader(configBytes))
+				out := bytes.NewBuffer(nil)
+				cmd.SetOut(out)
+				pty := ptytest.New(t)
+				cmd.SetErr(pty.Output())
+
+				ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitLong)
+				defer cancelFunc()
+
+				done := make(chan any)
+				go func() {
+					errC := cmd.ExecuteContext(ctx)
+					if c.errContains != "" {
+						assert.Error(t, errC)
+						assert.Contains(t, errC.Error(), c.errContains)
+					} else {
+						assert.NoError(t, errC)
+					}
+					close(done)
+				}()
+
+				<-done
+
+				if c.errContains != "" {
+					return
+				}
+				if len(c.outputs) == 0 {
+					// This is the default output format when no flags are
+					// specified.
+					c.outputs = []outputFlag{{format: "text"}}
+				}
+				for i, output := range c.outputs {
+					msg := fmt.Sprintf("flag %d", i)
+					var b []byte
+					if output.path == "" {
+						b = out.Bytes()
+					} else {
+						b, err = os.ReadFile(output.path)
+						require.NoError(t, err, msg)
+					}
+
+					switch output.format {
+					case "text":
+						require.Contains(t, string(b), "Test results:", msg)
+						require.Contains(t, string(b), "Pass:  10", msg)
+					case "json":
+						var res harness.Results
+						err = json.Unmarshal(b, &res)
+						require.NoError(t, err, msg)
+						require.Equal(t, 10, res.TotalRuns, msg)
+						require.Equal(t, 10, res.TotalPass, msg)
+						require.Len(t, res.Runs, 10, msg)
+					}
+				}
+			})
+		}
+	})
 }

loadtest/harness/harness.go

@@ -3,6 +3,7 @@ package harness
 import (
 	"context"
 	"sync"
+	"time"
 
 	"github.com/hashicorp/go-multierror"
 	"golang.org/x/xerrors"
@@ -27,6 +28,7 @@ type TestHarness struct {
 	runs    []*TestRun
 	started bool
 	done    chan struct{}
+	elapsed time.Duration
 }
 
 // NewTestHarness creates a new TestHarness with the given ExecutionStrategy.
@@ -63,6 +65,13 @@ func (h *TestHarness) Run(ctx context.Context) (err error) {
 		}
 	}()
 
+	start := time.Now()
+	defer func() {
+		h.mut.Lock()
+		defer h.mut.Unlock()
+		h.elapsed = time.Since(start)
+	}()
+
 	err = h.strategy.Execute(ctx, h.runs)
 	//nolint:revive // we use named returns because we mutate it in a defer
 	return

loadtest/harness/results.go

@@ -1,25 +1,36 @@
 package harness
 
-import "time"
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/coder/coder/coderd/httpapi"
+)
 
 // Results is the full compiled results for a set of test runs.
 type Results struct {
-	TotalRuns int
+	TotalRuns int              `json:"total_runs"`
-	TotalPass int
+	TotalPass int              `json:"total_pass"`
-	TotalFail int
+	TotalFail int              `json:"total_fail"`
+	Elapsed   httpapi.Duration `json:"elapsed"`
+	ElapsedMS int64            `json:"elapsed_ms"`
 
-	Runs map[string]RunResult
+	Runs map[string]RunResult `json:"runs"`
 }
 
 // RunResult is the result of a single test run.
 type RunResult struct {
-	FullID    string
+	FullID     string           `json:"full_id"`
-	TestName  string
+	TestName   string           `json:"test_name"`
-	ID        string
+	ID         string           `json:"id"`
-	Logs      []byte
+	Logs       string           `json:"logs"`
-	Error     error
+	Error      error            `json:"error"`
-	StartedAt time.Time
+	StartedAt  time.Time        `json:"started_at"`
-	Duration  time.Duration
+	Duration   httpapi.Duration `json:"duration"`
+	DurationMS int64            `json:"duration_ms"`
 }
 
 // Results returns the results of the test run. Panics if the test run is not
@@ -32,13 +43,14 @@ func (r *TestRun) Result() RunResult {
 	}
 
 	return RunResult{
-		FullID:    r.FullID(),
+		FullID:     r.FullID(),
-		TestName:  r.testName,
+		TestName:   r.testName,
-		ID:        r.id,
+		ID:         r.id,
-		Logs:      r.logs.Bytes(),
+		Logs:       r.logs.String(),
-		Error:     r.err,
+		Error:      r.err,
-		StartedAt: r.started,
+		StartedAt:  r.started,
-		Duration:  r.duration,
+		Duration:   httpapi.Duration(r.duration),
+		DurationMS: r.duration.Milliseconds(),
 	}
 }
 
@@ -56,6 +68,8 @@ func (h *TestHarness) Results() Results {
 	results := Results{
 		TotalRuns: len(h.runs),
 		Runs:      make(map[string]RunResult, len(h.runs)),
+		Elapsed:   httpapi.Duration(h.elapsed),
+		ElapsedMS: h.elapsed.Milliseconds(),
 	}
 	for _, run := range h.runs {
 		runRes := run.Result()
@@ -70,3 +84,40 @@ func (h *TestHarness) Results() Results {
 
 	return results
 }
+
+// PrintText prints the results as human-readable text to the given writer.
+func (r *Results) PrintText(w io.Writer) {
+	var totalDuration time.Duration
+	for _, run := range r.Runs {
+		totalDuration += time.Duration(run.Duration)
+		if run.Error == nil {
+			continue
+		}
+
+		_, _ = fmt.Fprintf(w, "\n== FAIL: %s\n\n", run.FullID)
+		_, _ = fmt.Fprintf(w, "\tError: %s\n\n", run.Error)
+
+		// Print log lines indented.
+		_, _ = fmt.Fprintf(w, "\tLog:\n")
+		rd := bufio.NewReader(strings.NewReader(run.Logs))
+		for {
+			line, err := rd.ReadBytes('\n')
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				_, _ = fmt.Fprintf(w, "\n\tLOG PRINT ERROR: %+v\n", err)
+			}
+
+			_, _ = fmt.Fprintf(w, "\t\t%s", line)
+		}
+	}
+
+	_, _ = fmt.Fprintln(w, "\n\nTest results:")
+	_, _ = fmt.Fprintf(w, "\tPass:  %d\n", r.TotalPass)
+	_, _ = fmt.Fprintf(w, "\tFail:  %d\n", r.TotalFail)
+	_, _ = fmt.Fprintf(w, "\tTotal: %d\n", r.TotalRuns)
+	_, _ = fmt.Fprintln(w, "")
+	_, _ = fmt.Fprintf(w, "\tTotal duration: %s\n", time.Duration(r.Elapsed))
+	_, _ = fmt.Fprintf(w, "\tAvg. duration:  %s\n", totalDuration/time.Duration(r.TotalRuns))
+}