package cli import ( "context" "encoding/json" "fmt" "io" "os" "strconv" "strings" "time" "github.com/spf13/cobra" "golang.org/x/xerrors" "github.com/coder/coder/cli/cliflag" "github.com/coder/coder/codersdk" "github.com/coder/coder/loadtest/harness" ) func loadtest() *cobra.Command { var ( configPath string outputSpecs []string ) cmd := &cobra.Command{ Use: "loadtest --config [--output json[:path]] [--output text[:path]]]", Short: "Load test the Coder API", // TODO: documentation and a JSON schema file Long: "Perform load tests against the Coder server. The load tests are configurable via a JSON file.", Example: formatExamples( example{ Description: "Run a loadtest with the given configuration file", Command: "coder loadtest --config path/to/config.json", }, example{ Description: "Run a loadtest, reading the configuration from stdin", Command: "cat path/to/config.json | coder loadtest --config -", }, example{ Description: "Run a loadtest outputting JSON results instead", Command: "coder loadtest --config path/to/config.json --output json", }, example{ Description: "Run a loadtest outputting JSON results to a file", Command: "coder loadtest --config path/to/config.json --output json:path/to/results.json", }, example{ Description: "Run a loadtest outputting text results to stdout and JSON results to a file", Command: "coder loadtest --config path/to/config.json --output text --output json:path/to/results.json", }, ), Hidden: true, Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) error { config, err := loadLoadTestConfigFile(configPath, cmd.InOrStdin()) if err != nil { return err } outputs, err := parseLoadTestOutputs(outputSpecs) if err != nil { return err } client, err := CreateClient(cmd) if err != nil { return err } me, err := client.User(cmd.Context(), codersdk.Me) if err != nil { return xerrors.Errorf("fetch current user: %w", err) } // Only owners can do loadtests. This isn't a very strong check but // there's not much else we can do. Ratelimits are enforced for // non-owners so hopefully that limits the damage if someone // disables this check and runs it against a non-owner account. ok := false for _, role := range me.Roles { if role.Name == "owner" { ok = true break } } if !ok { return xerrors.Errorf("Not logged in as site owner. Load testing is only available to site owners.") } // Disable ratelimits for future requests. client.BypassRatelimits = true // Prepare the test. strategy := config.Strategy.ExecutionStrategy() th := harness.NewTestHarness(strategy) for i, t := range config.Tests { name := fmt.Sprintf("%s-%d", t.Type, i) for j := 0; j < t.Count; j++ { id := strconv.Itoa(j) runner, err := t.NewRunner(client) if err != nil { return xerrors.Errorf("create %q runner for %s/%s: %w", t.Type, name, id, err) } th.AddRun(name, id, runner) } } _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Running load test...") testCtx := cmd.Context() if config.Timeout > 0 { var cancel func() testCtx, cancel = context.WithTimeout(testCtx, time.Duration(config.Timeout)) defer cancel() } // TODO: live progress output err = th.Run(testCtx) if err != nil { return xerrors.Errorf("run test harness (harness failure, not a test failure): %w", err) } // Print the results. res := th.Results() for _, output := range outputs { var ( w = cmd.OutOrStdout() c io.Closer ) if output.path != "-" { f, err := os.Create(output.path) if err != nil { return xerrors.Errorf("create output file: %w", err) } w, c = f, f } switch output.format { case loadTestOutputFormatText: res.PrintText(w) case loadTestOutputFormatJSON: err = json.NewEncoder(w).Encode(res) if err != nil { return xerrors.Errorf("encode JSON: %w", err) } } if c != nil { err = c.Close() if err != nil { return xerrors.Errorf("close output file: %w", err) } } } // Cleanup. _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\nCleaning up...") err = th.Cleanup(cmd.Context()) if err != nil { return xerrors.Errorf("cleanup tests: %w", err) } if res.TotalFail > 0 { return xerrors.New("load test failed, see above for more details") } return nil }, } cliflag.StringVarP(cmd.Flags(), &configPath, "config", "", "CODER_LOADTEST_CONFIG_PATH", "", "Path to the load test configuration file, or - to read from stdin.") cliflag.StringArrayVarP(cmd.Flags(), &outputSpecs, "output", "", "CODER_LOADTEST_OUTPUTS", []string{"text"}, "Output formats, see usage for more information.") return cmd } func loadLoadTestConfigFile(configPath string, stdin io.Reader) (LoadTestConfig, error) { if configPath == "" { return LoadTestConfig{}, xerrors.New("config is required") } var ( configReader io.ReadCloser ) if configPath == "-" { configReader = io.NopCloser(stdin) } else { f, err := os.Open(configPath) if err != nil { return LoadTestConfig{}, xerrors.Errorf("open config file %q: %w", configPath, err) } configReader = f } var config LoadTestConfig err := json.NewDecoder(configReader).Decode(&config) _ = configReader.Close() if err != nil { return LoadTestConfig{}, xerrors.Errorf("read config file %q: %w", configPath, err) } err = config.Validate() if err != nil { return LoadTestConfig{}, xerrors.Errorf("validate config: %w", err) } return config, nil } type loadTestOutputFormat string const ( loadTestOutputFormatText loadTestOutputFormat = "text" loadTestOutputFormatJSON loadTestOutputFormat = "json" // TODO: html format ) type loadTestOutput struct { format loadTestOutputFormat // Up to one path (the first path) will have the value "-" which signifies // stdout. path string } func parseLoadTestOutputs(outputs []string) ([]loadTestOutput, error) { var stdoutFormat loadTestOutputFormat validFormats := map[loadTestOutputFormat]struct{}{ loadTestOutputFormatText: {}, loadTestOutputFormatJSON: {}, } var out []loadTestOutput for i, o := range outputs { parts := strings.SplitN(o, ":", 2) format := loadTestOutputFormat(parts[0]) if _, ok := validFormats[format]; !ok { return nil, xerrors.Errorf("invalid output format %q in output flag %d", parts[0], i) } if len(parts) == 1 { if stdoutFormat != "" { return nil, xerrors.Errorf("multiple output flags specified for stdout") } stdoutFormat = format continue } if len(parts) != 2 { return nil, xerrors.Errorf("invalid output flag %d: %q", i, o) } out = append(out, loadTestOutput{ format: format, path: parts[1], }) } // Default to --output text if stdoutFormat == "" && len(out) == 0 { stdoutFormat = loadTestOutputFormatText } if stdoutFormat != "" { out = append([]loadTestOutput{{ format: stdoutFormat, path: "-", }}, out...) } return out, nil }