fix(scaletest): output error and trace instead of {} for json output (#10075)

scaletest/harness/results.go

@@ -2,11 +2,15 @@ package harness
 
 import (
 	"bufio"
+	"encoding/json"
 	"fmt"
 	"io"
+	"sort"
 	"strings"
 	"time"
 
+	"golang.org/x/exp/maps"
+
 	"github.com/coder/coder/v2/coderd/httpapi"
 )
 
@@ -33,6 +37,18 @@ type RunResult struct {
 	DurationMS int64            `json:"duration_ms"`
 }
 
+// MarshalJSON implements json.Marhshaler for RunResult.
+func (r RunResult) MarshalJSON() ([]byte, error) {
+	type alias RunResult
+	return json.Marshal(&struct {
+		alias
+		Error string `json:"error"`
+	}{
+		alias: alias(r),
+		Error: fmt.Sprintf("%+v", r.Error),
+	})
+}
+
 // Results returns the results of the test run. Panics if the test run is not
 // done yet.
 func (r *TestRun) Result() RunResult {
@@ -88,7 +104,10 @@ func (h *TestHarness) Results() Results {
 // PrintText prints the results as human-readable text to the given writer.
 func (r *Results) PrintText(w io.Writer) {
 	var totalDuration time.Duration
-	for _, run := range r.Runs {
+	keys := maps.Keys(r.Runs)
+	sort.Strings(keys)
+	for _, key := range keys {
+		run := r.Runs[key]
 		totalDuration += time.Duration(run.Duration)
 		if run.Error == nil {
 			continue

scaletest/harness/results_test.go

@@ -2,9 +2,15 @@ package harness_test
 
 import (
 	"bytes"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/xerrors"
 
@@ -12,9 +18,18 @@ import (
 	"github.com/coder/coder/v2/scaletest/harness"
 )
 
+type testError struct {
+	hidden error
+}
+
+func (e testError) Error() string {
+	return e.hidden.Error()
+}
+
 func Test_Results(t *testing.T) {
 	t.Parallel()
 
+	now := time.Date(2023, 10, 5, 12, 3, 56, 395813665, time.UTC)
 	results := harness.Results{
 		TotalRuns: 10,
 		TotalPass: 8,
@@ -26,7 +41,7 @@ func Test_Results(t *testing.T) {
 				ID:         "0",
 				Logs:       "test-0/0 log line 1\ntest-0/0 log line 2",
 				Error:      xerrors.New("test-0/0 error"),
-				StartedAt:  time.Now(),
+				StartedAt:  now,
 				Duration:   httpapi.Duration(time.Second),
 				DurationMS: 1000,
 			},
@@ -36,7 +51,17 @@ func Test_Results(t *testing.T) {
 				ID:         "1",
 				Logs:       "test-0/1 log line 1\ntest-0/1 log line 2",
 				Error:      nil,
-				StartedAt:  time.Now(),
+				StartedAt:  now.Add(333 * time.Millisecond),
+				Duration:   httpapi.Duration(time.Second),
+				DurationMS: 1000,
+			},
+			"test-0/2": {
+				FullID:     "test-0/2",
+				TestName:   "test-0",
+				ID:         "2",
+				Logs:       "test-0/2 log line 1\ntest-0/2 log line 2",
+				Error:      testError{hidden: xerrors.New("test-0/2 error")},
+				StartedAt:  now.Add(666 * time.Millisecond),
 				Duration:   httpapi.Duration(time.Second),
 				DurationMS: 1000,
 			},
@@ -45,7 +70,7 @@ func Test_Results(t *testing.T) {
 		ElapsedMS: 1000,
 	}
 
-	expected := `
+	wantText := `
 == FAIL: test-0/0
 
 	Error: test-0/0 error
@@ -53,6 +78,13 @@ func Test_Results(t *testing.T) {
 	Log:
 		test-0/0 log line 1
 
+== FAIL: test-0/2
+
+	Error: test-0/2 error
+
+	Log:
+		test-0/2 log line 1
+
 
 Test results:
 	Pass:  8
@@ -60,11 +92,63 @@ Test results:
 	Total: 10
 
 	Total duration: 1s
-	Avg. duration:  200ms
+	Avg. duration:  300ms
 `
+	wantJSON := `{
+	"total_runs": 10,
+	"total_pass": 8,
+	"total_fail": 2,
+	"elapsed": "1s",
+	"elapsed_ms": 1000,
+	"runs": {
+		"test-0/0": {
+			"full_id": "test-0/0",
+			"test_name": "test-0",
+			"id": "0",
+			"logs": "test-0/0 log line 1\ntest-0/0 log line 2",
+			"started_at": "2023-10-05T12:03:56.395813665Z",
+			"duration": "1s",
+			"duration_ms": 1000,
+			"error": "test-0/0 error:\n    github.com/coder/coder/v2/scaletest/harness_test.Test_Results\n        [working_directory]/results_test.go:43"
+		},
+		"test-0/1": {
+			"full_id": "test-0/1",
+			"test_name": "test-0",
+			"id": "1",
+			"logs": "test-0/1 log line 1\ntest-0/1 log line 2",
+			"started_at": "2023-10-05T12:03:56.728813665Z",
+			"duration": "1s",
+			"duration_ms": 1000,
+			"error": "\u003cnil\u003e"
+		},
+		"test-0/2": {
+			"full_id": "test-0/2",
+			"test_name": "test-0",
+			"id": "2",
+			"logs": "test-0/2 log line 1\ntest-0/2 log line 2",
+			"started_at": "2023-10-05T12:03:57.061813665Z",
+			"duration": "1s",
+			"duration_ms": 1000,
+			"error": "test-0/2 error"
+		}
+	}
+}
+`
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+	wd = filepath.ToSlash(wd) // Hello there Windows, my friend...
+	wantJSON = strings.Replace(wantJSON, "[working_directory]", wd, 1)
 
 	out := bytes.NewBuffer(nil)
 	results.PrintText(out)
 
-	require.Equal(t, expected, out.String())
+	assert.Empty(t, cmp.Diff(wantText, out.String()), "text result does not match (-want +got)")
+
+	out.Reset()
+	enc := json.NewEncoder(out)
+	enc.SetIndent("", "\t")
+	err = enc.Encode(results)
+	require.NoError(t, err)
+
+	assert.Empty(t, cmp.Diff(wantJSON, out.String()), "JSON result does not match (-want +got)")
 }