synced/coder
1
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2025-07-12 00:14:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: loadtest output formats (#4928)

Browse Source
This commit is contained in:
Dean Sheather
2022-11-08 03:26:50 +10:00
committed by GitHub
parent f9189772d7
commit 5f099ea488
4 changed files with 394 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

218
cli/loadtest.go
View File

@ -1,14 +1,13 @@
package cli
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
"github.com/spf13/cobra"
@ -21,44 +20,46 @@ import (
func loadtest() *cobra.Command {
var (
configPath string
configPath string
outputSpecs []string
)
cmd := &cobra.Command{
Use: "loadtest --config <path>",
Use: "loadtest --config <path> [--output json[:path]] [--output text[:path]]]",
Short: "Load test the Coder API",
// TODO: documentation and a JSON scheme file
Long: "Perform load tests against the Coder server. The load tests " +
"configurable via a JSON file.",
// TODO: documentation and a JSON schema file
Long: "Perform load tests against the Coder server. The load tests are configurable via a JSON file.",
Example: formatExamples(
example{
Description: "Run a loadtest with the given configuration file",
Command: "coder loadtest --config path/to/config.json",
},
example{
Description: "Run a loadtest, reading the configuration from stdin",
Command: "cat path/to/config.json | coder loadtest --config -",
},
example{
Description: "Run a loadtest outputting JSON results instead",
Command: "coder loadtest --config path/to/config.json --output json",
},
example{
Description: "Run a loadtest outputting JSON results to a file",
Command: "coder loadtest --config path/to/config.json --output json:path/to/results.json",
},
example{
Description: "Run a loadtest outputting text results to stdout and JSON results to a file",
Command: "coder loadtest --config path/to/config.json --output text --output json:path/to/results.json",
},
),
Hidden: true,
Args: cobra.ExactArgs(0),
RunE: func(cmd *cobra.Command, args []string) error {
if configPath == "" {
return xerrors.New("config is required")
}
var (
configReader io.ReadCloser
)
if configPath == "-" {
configReader = io.NopCloser(cmd.InOrStdin())
} else {
f, err := os.Open(configPath)
if err != nil {
return xerrors.Errorf("open config file %q: %w", configPath, err)
}
configReader = f
}
var config LoadTestConfig
err := json.NewDecoder(configReader).Decode(&config)
_ = configReader.Close()
config, err := loadLoadTestConfigFile(configPath, cmd.InOrStdin())
if err != nil {
return xerrors.Errorf("read config file %q: %w", configPath, err)
return err
}
err = config.Validate()
outputs, err := parseLoadTestOutputs(outputSpecs)
if err != nil {
return xerrors.Errorf("validate config: %w", err)
return err
}
client, err := CreateClient(cmd)
@ -117,52 +118,44 @@ func loadtest() *cobra.Command {
}
// TODO: live progress output
start := time.Now()
err = th.Run(testCtx)
if err != nil {
return xerrors.Errorf("run test harness (harness failure, not a test failure): %w", err)
}
elapsed := time.Since(start)
// Print the results.
// TODO: better result printing
// TODO: move result printing to the loadtest package, add multiple
// output formats (like HTML, JSON)
res := th.Results()
var totalDuration time.Duration
for _, run := range res.Runs {
totalDuration += run.Duration
if run.Error == nil {
continue
for _, output := range outputs {
var (
w = cmd.OutOrStdout()
c io.Closer
)
if output.path != "-" {
f, err := os.Create(output.path)
if err != nil {
return xerrors.Errorf("create output file: %w", err)
}
w, c = f, f
}
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\n== FAIL: %s\n\n", run.FullID)
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tError: %s\n\n", run.Error)
// Print log lines indented.
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tLog:\n")
rd := bufio.NewReader(bytes.NewBuffer(run.Logs))
for {
line, err := rd.ReadBytes('\n')
if err == io.EOF {
break
}
switch output.format {
case loadTestOutputFormatText:
res.PrintText(w)
case loadTestOutputFormatJSON:
err = json.NewEncoder(w).Encode(res)
if err != nil {
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\n\tLOG PRINT ERROR: %+v\n", err)
return xerrors.Errorf("encode JSON: %w", err)
}
}
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\t\t%s", line)
if c != nil {
err = c.Close()
if err != nil {
return xerrors.Errorf("close output file: %w", err)
}
}
}
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\n\nTest results:")
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tPass: %d\n", res.TotalPass)
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tFail: %d\n", res.TotalFail)
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tTotal: %d\n", res.TotalRuns)
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "")
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tTotal duration: %s\n", elapsed)
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tAvg. duration: %s\n", totalDuration/time.Duration(res.TotalRuns))
// Cleanup.
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\nCleaning up...")
err = th.Cleanup(cmd.Context())
@ -170,10 +163,111 @@ func loadtest() *cobra.Command {
return xerrors.Errorf("cleanup tests: %w", err)
}
if res.TotalFail > 0 {
return xerrors.New("load test failed, see above for more details")
}
return nil
},
}
cliflag.StringVarP(cmd.Flags(), &configPath, "config", "", "CODER_LOADTEST_CONFIG_PATH", "", "Path to the load test configuration file, or - to read from stdin.")
cliflag.StringArrayVarP(cmd.Flags(), &outputSpecs, "output", "", "CODER_LOADTEST_OUTPUTS", []string{"text"}, "Output formats, see usage for more information.")
return cmd
}
func loadLoadTestConfigFile(configPath string, stdin io.Reader) (LoadTestConfig, error) {
if configPath == "" {
return LoadTestConfig{}, xerrors.New("config is required")
}
var (
configReader io.ReadCloser
)
if configPath == "-" {
configReader = io.NopCloser(stdin)
} else {
f, err := os.Open(configPath)
if err != nil {
return LoadTestConfig{}, xerrors.Errorf("open config file %q: %w", configPath, err)
}
configReader = f
}
var config LoadTestConfig
err := json.NewDecoder(configReader).Decode(&config)
_ = configReader.Close()
if err != nil {
return LoadTestConfig{}, xerrors.Errorf("read config file %q: %w", configPath, err)
}
err = config.Validate()
if err != nil {
return LoadTestConfig{}, xerrors.Errorf("validate config: %w", err)
}
return config, nil
}
type loadTestOutputFormat string
const (
loadTestOutputFormatText loadTestOutputFormat = "text"
loadTestOutputFormatJSON loadTestOutputFormat = "json"
// TODO: html format
)
type loadTestOutput struct {
format loadTestOutputFormat
// Up to one path (the first path) will have the value "-" which signifies
// stdout.
path string
}
func parseLoadTestOutputs(outputs []string) ([]loadTestOutput, error) {
var stdoutFormat loadTestOutputFormat
validFormats := map[loadTestOutputFormat]struct{}{
loadTestOutputFormatText: {},
loadTestOutputFormatJSON: {},
}
var out []loadTestOutput
for i, o := range outputs {
parts := strings.SplitN(o, ":", 2)
format := loadTestOutputFormat(parts[0])
if _, ok := validFormats[format]; !ok {
return nil, xerrors.Errorf("invalid output format %q in output flag %d", parts[0], i)
}
if len(parts) == 1 {
if stdoutFormat != "" {
return nil, xerrors.Errorf("multiple output flags specified for stdout")
}
stdoutFormat = format
continue
}
if len(parts) != 2 {
return nil, xerrors.Errorf("invalid output flag %d: %q", i, o)
}
out = append(out, loadTestOutput{
format: format,
path: parts[1],
})
}
// Default to --output text
if stdoutFormat == "" && len(out) == 0 {
stdoutFormat = loadTestOutputFormatText
}
if stdoutFormat != "" {
out = append([]loadTestOutput{{
format: stdoutFormat,
path: "-",
}}, out...)
}
return out, nil
}

159
cli/loadtest_test.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"testing"
@ -17,6 +18,7 @@ import (
"github.com/coder/coder/coderd/coderdtest"
"github.com/coder/coder/coderd/httpapi"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/loadtest/harness"
"github.com/coder/coder/loadtest/placebo"
"github.com/coder/coder/loadtest/workspacebuild"
"github.com/coder/coder/pty/ptytest"
@ -133,4 +135,161 @@ func TestLoadTest(t *testing.T) {
<-done
cancelFunc()
})
t.Run("OutputFormats", func(t *testing.T) {
t.Parallel()
type outputFlag struct {
format string
path string
}
dir := t.TempDir()
cases := []struct {
name string
outputs []outputFlag
errContains string
}{
{
name: "Default",
outputs: []outputFlag{},
},
{
name: "ExplicitText",
outputs: []outputFlag{{format: "text"}},
},
{
name: "JSON",
outputs: []outputFlag{
{
format: "json",
path: filepath.Join(dir, "results.json"),
},
},
},
{
name: "TextAndJSON",
outputs: []outputFlag{
{
format: "text",
},
{
format: "json",
path: filepath.Join(dir, "results.json"),
},
},
},
{
name: "TextAndJSON2",
outputs: []outputFlag{
{
format: "text",
},
{
format: "text",
path: filepath.Join(dir, "results.txt"),
},
{
format: "json",
path: filepath.Join(dir, "results.json"),
},
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, nil)
_ = coderdtest.CreateFirstUser(t, client)
config := cli.LoadTestConfig{
Strategy: cli.LoadTestStrategy{
Type: cli.LoadTestStrategyTypeLinear,
},
Tests: []cli.LoadTest{
{
Type: cli.LoadTestTypePlacebo,
Count: 10,
Placebo: &placebo.Config{
Sleep: httpapi.Duration(10 * time.Millisecond),
},
},
},
Timeout: httpapi.Duration(testutil.WaitShort),
}
configBytes, err := json.Marshal(config)
require.NoError(t, err)
args := []string{"loadtest", "--config", "-"}
for _, output := range c.outputs {
flag := output.format
if output.path != "" {
flag += ":" + output.path
}
args = append(args, "--output", flag)
}
cmd, root := clitest.New(t, args...)
clitest.SetupConfig(t, client, root)
cmd.SetIn(bytes.NewReader(configBytes))
out := bytes.NewBuffer(nil)
cmd.SetOut(out)
pty := ptytest.New(t)
cmd.SetErr(pty.Output())
ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancelFunc()
done := make(chan any)
go func() {
errC := cmd.ExecuteContext(ctx)
if c.errContains != "" {
assert.Error(t, errC)
assert.Contains(t, errC.Error(), c.errContains)
} else {
assert.NoError(t, errC)
}
close(done)
}()
<-done
if c.errContains != "" {
return
}
if len(c.outputs) == 0 {
// This is the default output format when no flags are
// specified.
c.outputs = []outputFlag{{format: "text"}}
}
for i, output := range c.outputs {
msg := fmt.Sprintf("flag %d", i)
var b []byte
if output.path == "" {
b = out.Bytes()
} else {
b, err = os.ReadFile(output.path)
require.NoError(t, err, msg)
}
switch output.format {
case "text":
require.Contains(t, string(b), "Test results:", msg)
require.Contains(t, string(b), "Pass: 10", msg)
case "json":
var res harness.Results
err = json.Unmarshal(b, &res)
require.NoError(t, err, msg)
require.Equal(t, 10, res.TotalRuns, msg)
require.Equal(t, 10, res.TotalPass, msg)
require.Len(t, res.Runs, 10, msg)
}
}
})
}
})
}

9
loadtest/harness/harness.go
View File

@ -3,6 +3,7 @@ package harness
import (
"context"
"sync"
"time"
"github.com/hashicorp/go-multierror"
"golang.org/x/xerrors"
@ -27,6 +28,7 @@ type TestHarness struct {
runs []*TestRun
started bool
done chan struct{}
elapsed time.Duration
}
// NewTestHarness creates a new TestHarness with the given ExecutionStrategy.
@ -63,6 +65,13 @@ func (h *TestHarness) Run(ctx context.Context) (err error) {
}
}()
start := time.Now()
defer func() {
h.mut.Lock()
defer h.mut.Unlock()
h.elapsed = time.Since(start)
}()
err = h.strategy.Execute(ctx, h.runs)
//nolint:revive // we use named returns because we mutate it in a defer
return

89
loadtest/harness/results.go
View File

@ -1,25 +1,36 @@
package harness
import "time"
import (
"bufio"
"fmt"
"io"
"strings"
"time"
"github.com/coder/coder/coderd/httpapi"
)
// Results is the full compiled results for a set of test runs.
type Results struct {
TotalRuns int
TotalPass int
TotalFail int
TotalRuns int `json:"total_runs"`
TotalPass int `json:"total_pass"`
TotalFail int `json:"total_fail"`
Elapsed httpapi.Duration `json:"elapsed"`
ElapsedMS int64 `json:"elapsed_ms"`
Runs map[string]RunResult
Runs map[string]RunResult `json:"runs"`
}
// RunResult is the result of a single test run.
type RunResult struct {
FullID string
TestName string
ID string
Logs []byte
Error error
StartedAt time.Time
Duration time.Duration
FullID string `json:"full_id"`
TestName string `json:"test_name"`
ID string `json:"id"`
Logs string `json:"logs"`
Error error `json:"error"`
StartedAt time.Time `json:"started_at"`
Duration httpapi.Duration `json:"duration"`
DurationMS int64 `json:"duration_ms"`
}
// Results returns the results of the test run. Panics if the test run is not
@ -32,13 +43,14 @@ func (r *TestRun) Result() RunResult {
}
return RunResult{
FullID: r.FullID(),
TestName: r.testName,
ID: r.id,
Logs: r.logs.Bytes(),
Error: r.err,
StartedAt: r.started,
Duration: r.duration,
FullID: r.FullID(),
TestName: r.testName,
ID: r.id,
Logs: r.logs.String(),
Error: r.err,
StartedAt: r.started,
Duration: httpapi.Duration(r.duration),
DurationMS: r.duration.Milliseconds(),
}
}
@ -56,6 +68,8 @@ func (h *TestHarness) Results() Results {
results := Results{
TotalRuns: len(h.runs),
Runs: make(map[string]RunResult, len(h.runs)),
Elapsed: httpapi.Duration(h.elapsed),
ElapsedMS: h.elapsed.Milliseconds(),
}
for _, run := range h.runs {
runRes := run.Result()
@ -70,3 +84,40 @@ func (h *TestHarness) Results() Results {
return results
}
// PrintText prints the results as human-readable text to the given writer.
func (r *Results) PrintText(w io.Writer) {
var totalDuration time.Duration
for _, run := range r.Runs {
totalDuration += time.Duration(run.Duration)
if run.Error == nil {
continue
}
_, _ = fmt.Fprintf(w, "\n== FAIL: %s\n\n", run.FullID)
_, _ = fmt.Fprintf(w, "\tError: %s\n\n", run.Error)
// Print log lines indented.
_, _ = fmt.Fprintf(w, "\tLog:\n")
rd := bufio.NewReader(strings.NewReader(run.Logs))
for {
line, err := rd.ReadBytes('\n')
if err == io.EOF {
break
}
if err != nil {
_, _ = fmt.Fprintf(w, "\n\tLOG PRINT ERROR: %+v\n", err)
}
_, _ = fmt.Fprintf(w, "\t\t%s", line)
}
}
_, _ = fmt.Fprintln(w, "\n\nTest results:")
_, _ = fmt.Fprintf(w, "\tPass: %d\n", r.TotalPass)
_, _ = fmt.Fprintf(w, "\tFail: %d\n", r.TotalFail)
_, _ = fmt.Fprintf(w, "\tTotal: %d\n", r.TotalRuns)
_, _ = fmt.Fprintln(w, "")
_, _ = fmt.Fprintf(w, "\tTotal duration: %s\n", time.Duration(r.Elapsed))
_, _ = fmt.Fprintf(w, "\tAvg. duration: %s\n", totalDuration/time.Duration(r.TotalRuns))
}