mirror of
https://github.com/googleforgames/open-match.git
synced 2025-03-25 13:24:18 +00:00
Add metric support to the scale tests (#1042)
This commit is contained in:
Scott Redig
GitHub
3
Makefile
3
Makefile
@ -358,10 +358,11 @@ install-scale-chart: install-chart-prerequisite build/toolchain/bin/helm$(EXE_EX
|
||||
--set open-match-customize.evaluator.enabled=true \
|
||||
--set open-match-customize.function.image=openmatch-scale-mmf \
|
||||
--set global.telemetry.grafana.enabled=true \
|
||||
--set global.telemetry.jaeger.enabled=true \
|
||||
--set global.telemetry.jaeger.enabled=false \
|
||||
--set global.telemetry.prometheus.enabled=true
|
||||
$(HELM) template $(OPEN_MATCH_HELM_NAME)-scale install/helm/open-match $(HELM_TEMPLATE_FLAGS) $(HELM_IMAGE_FLAGS) -f install/helm/open-match/values-production.yaml \
|
||||
--set open-match-core.enabled=false \
|
||||
--set global.telemetry.prometheus.enabled=true \
|
||||
--set open-match-scale.enabled=true | $(KUBECTL) apply -f -
|
||||
|
||||
# install-ci-chart will install open-match-core with pool based mmf for end-to-end in-cluster test.
|
||||
|
||||
@ -16,8 +16,10 @@ package main
|
||||
|
||||
import (
|
||||
"open-match.dev/open-match/examples/scale/backend"
|
||||
"open-match.dev/open-match/internal/app"
|
||||
"open-match.dev/open-match/internal/config"
|
||||
)
|
||||
|
||||
func main() {
|
||||
backend.Run()
|
||||
app.RunApplication("scale", config.Read, backend.BindService)
|
||||
}
|
||||
|
||||
@ -16,8 +16,10 @@ package main
|
||||
|
||||
import (
|
||||
"open-match.dev/open-match/examples/scale/frontend"
|
||||
"open-match.dev/open-match/internal/app"
|
||||
"open-match.dev/open-match/internal/config"
|
||||
)
|
||||
|
||||
func main() {
|
||||
frontend.Run()
|
||||
app.RunApplication("scale", config.Read, frontend.BindService)
|
||||
}
|
||||
|
||||
@ -27,8 +27,8 @@ import (
|
||||
"open-match.dev/open-match/examples/scale/profiles"
|
||||
"open-match.dev/open-match/examples/scale/scenarios"
|
||||
"open-match.dev/open-match/internal/config"
|
||||
"open-match.dev/open-match/internal/logging"
|
||||
"open-match.dev/open-match/internal/rpc"
|
||||
"open-match.dev/open-match/internal/telemetry"
|
||||
"open-match.dev/open-match/pkg/pb"
|
||||
)
|
||||
|
||||
@ -40,19 +40,27 @@ var (
|
||||
|
||||
activeScenario = scenarios.ActiveScenario
|
||||
statProcessor = scenarios.NewStatProcessor()
|
||||
|
||||
mIterations = telemetry.Counter("scale_backend_iterations", "fetch match iterations")
|
||||
mFetchMatchCalls = telemetry.Counter("scale_backend_fetch_match_calls", "fetch match calls")
|
||||
mFetchMatchSuccesses = telemetry.Counter("scale_backend_fetch_match_successes", "fetch match successes")
|
||||
mFetchMatchErrors = telemetry.Counter("scale_backend_fetch_match_errors", "fetch match errors")
|
||||
mMatchesReturned = telemetry.Counter("scale_backend_matches_returned", "matches returned")
|
||||
mSumTicketsReturned = telemetry.Counter("scale_backend_sum_tickets_returned", "tickets in matches returned")
|
||||
mMatchesAssigned = telemetry.Counter("scale_backend_matches_assigned", "matches assigned")
|
||||
mMatchAssignsFailed = telemetry.Counter("scale_backend_match_assigns_failed", "match assigns failed")
|
||||
mTicketsDeleted = telemetry.Counter("scale_backend_tickets_deleted", "tickets deleted")
|
||||
mTicketDeletesFailed = telemetry.Counter("scale_backend_ticket_deletes_failed", "ticket deletes failed")
|
||||
)
|
||||
|
||||
// Run triggers execution of functions that continuously fetch, assign and
|
||||
// delete matches.
|
||||
func Run() {
|
||||
cfg, err := config.Read()
|
||||
if err != nil {
|
||||
logger.WithFields(logrus.Fields{
|
||||
"error": err.Error(),
|
||||
}).Fatalf("cannot read configuration.")
|
||||
}
|
||||
func BindService(p *rpc.ServerParams, cfg config.View) error {
|
||||
go run(cfg)
|
||||
return nil
|
||||
}
|
||||
|
||||
logging.ConfigureLogging(cfg)
|
||||
func run(cfg config.View) {
|
||||
beConn, err := rpc.GRPCClientFromConfig(cfg, "api.backend")
|
||||
if err != nil {
|
||||
logger.Fatalf("failed to connect to Open Match Backend, got %v", err)
|
||||
@ -84,18 +92,19 @@ func Run() {
|
||||
wg.Add(1)
|
||||
go func(wg *sync.WaitGroup, p *pb.MatchProfile) {
|
||||
defer wg.Done()
|
||||
run(fe, be, p)
|
||||
runIteration(fe, be, p)
|
||||
}(&wg, p)
|
||||
}
|
||||
|
||||
// Wait for all profiles to complete before proceeding.
|
||||
wg.Wait()
|
||||
statProcessor.SetStat("TimeElapsed", time.Since(startTime).String())
|
||||
telemetry.RecordUnitMeasurement(context.Background(), mIterations)
|
||||
statProcessor.Log(w)
|
||||
}
|
||||
}
|
||||
|
||||
func run(fe pb.FrontendClient, be pb.BackendClient, p *pb.MatchProfile) {
|
||||
func runIteration(fe pb.FrontendClient, be pb.BackendClient, p *pb.MatchProfile) {
|
||||
ctx, span := trace.StartSpan(context.Background(), "scale.backend/FetchMatches")
|
||||
defer span.End()
|
||||
|
||||
@ -108,32 +117,38 @@ func run(fe pb.FrontendClient, be pb.BackendClient, p *pb.MatchProfile) {
|
||||
Profiles: []*pb.MatchProfile{p},
|
||||
}
|
||||
|
||||
telemetry.RecordUnitMeasurement(ctx, mFetchMatchCalls)
|
||||
stream, err := be.FetchMatches(ctx, req)
|
||||
if err != nil {
|
||||
telemetry.RecordUnitMeasurement(ctx, mFetchMatchErrors)
|
||||
statProcessor.RecordError("failed to get available stream client", err)
|
||||
return
|
||||
}
|
||||
|
||||
processMatches(fe, be, stream)
|
||||
processMatches(ctx, fe, be, stream)
|
||||
}
|
||||
|
||||
func processMatches(fe pb.FrontendClient, be pb.BackendClient, stream pb.Backend_FetchMatchesClient) {
|
||||
func processMatches(ctx context.Context, fe pb.FrontendClient, be pb.BackendClient, stream pb.Backend_FetchMatchesClient) {
|
||||
for {
|
||||
// Pull the Match
|
||||
resp, err := stream.Recv()
|
||||
if err == io.EOF {
|
||||
telemetry.RecordUnitMeasurement(ctx, mFetchMatchSuccesses)
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
telemetry.RecordUnitMeasurement(ctx, mFetchMatchErrors)
|
||||
statProcessor.RecordError("failed to get matches from stream client", err)
|
||||
return
|
||||
}
|
||||
|
||||
telemetry.RecordUnitMeasurement(ctx, mMatchesReturned)
|
||||
statProcessor.IncrementStat("MatchCount", 1)
|
||||
|
||||
ids := []string{}
|
||||
for _, t := range resp.GetMatch().Tickets {
|
||||
telemetry.RecordUnitMeasurement(ctx, mSumTicketsReturned)
|
||||
ids = append(ids, t.GetId())
|
||||
}
|
||||
// Assign Tickets
|
||||
@ -144,10 +159,12 @@ func processMatches(fe pb.FrontendClient, be pb.BackendClient, stream pb.Backend
|
||||
Connection: fmt.Sprintf("%d.%d.%d.%d:2222", rand.Intn(256), rand.Intn(256), rand.Intn(256), rand.Intn(256)),
|
||||
},
|
||||
}); err != nil {
|
||||
telemetry.RecordUnitMeasurement(ctx, mMatchAssignsFailed)
|
||||
statProcessor.RecordError("failed to assign tickets", err)
|
||||
continue
|
||||
}
|
||||
|
||||
telemetry.RecordUnitMeasurement(ctx, mMatchesAssigned)
|
||||
statProcessor.IncrementStat("Assigned", len(ids))
|
||||
}
|
||||
|
||||
@ -159,10 +176,12 @@ func processMatches(fe pb.FrontendClient, be pb.BackendClient, stream pb.Backend
|
||||
}
|
||||
|
||||
if _, err := fe.DeleteTicket(context.Background(), req); err != nil {
|
||||
telemetry.RecordUnitMeasurement(ctx, mTicketDeletesFailed)
|
||||
statProcessor.RecordError("failed to delete tickets", err)
|
||||
continue
|
||||
}
|
||||
|
||||
telemetry.RecordUnitMeasurement(ctx, mTicketsDeleted)
|
||||
statProcessor.IncrementStat("Deleted", 1)
|
||||
}
|
||||
}
|
||||
|
||||
@ -25,8 +25,8 @@ import (
|
||||
"open-match.dev/open-match/examples/scale/scenarios"
|
||||
"open-match.dev/open-match/examples/scale/tickets"
|
||||
"open-match.dev/open-match/internal/config"
|
||||
"open-match.dev/open-match/internal/logging"
|
||||
"open-match.dev/open-match/internal/rpc"
|
||||
"open-match.dev/open-match/internal/telemetry"
|
||||
"open-match.dev/open-match/pkg/pb"
|
||||
)
|
||||
|
||||
@ -40,35 +40,28 @@ var (
|
||||
numOfRoutineCreate = 8
|
||||
|
||||
totalCreated uint32
|
||||
|
||||
mTicketsCreated = telemetry.Counter("scale_frontend_tickets_created", "tickets created")
|
||||
mTicketCreationsFailed = telemetry.Counter("scale_frontend_ticket_creations_failed", "tickets created")
|
||||
)
|
||||
|
||||
// Run triggers execution of the scale frontend component that creates
|
||||
// tickets at scale in Open Match.
|
||||
func Run() {
|
||||
cfg, err := config.Read()
|
||||
if err != nil {
|
||||
logger.WithFields(logrus.Fields{
|
||||
"error": err.Error(),
|
||||
}).Fatal("cannot read configuration.")
|
||||
}
|
||||
func BindService(p *rpc.ServerParams, cfg config.View) error {
|
||||
go run(cfg)
|
||||
|
||||
logging.ConfigureLogging(cfg)
|
||||
return nil
|
||||
}
|
||||
|
||||
func run(cfg config.View) {
|
||||
conn, err := rpc.GRPCClientFromConfig(cfg, "api.frontend")
|
||||
if err != nil {
|
||||
logger.WithFields(logrus.Fields{
|
||||
"error": err.Error(),
|
||||
}).Fatal("failed to get Frontend connection")
|
||||
}
|
||||
|
||||
defer conn.Close()
|
||||
fe := pb.NewFrontendClient(conn)
|
||||
|
||||
create(cfg, fe)
|
||||
|
||||
select {}
|
||||
}
|
||||
|
||||
func create(cfg config.View, fe pb.FrontendClient) {
|
||||
w := logger.Writer()
|
||||
defer w.Close()
|
||||
|
||||
@ -130,10 +123,13 @@ func createPerCycle(wg *sync.WaitGroup, fe pb.FrontendClient, ticketPerRoutine i
|
||||
|
||||
time.Sleep(timeLeft / time.Duration(ticketsLeft))
|
||||
|
||||
if _, err := fe.CreateTicket(ctx, req); err != nil {
|
||||
if _, err := fe.CreateTicket(ctx, req); err == nil {
|
||||
cycleCreated++
|
||||
telemetry.RecordUnitMeasurement(ctx, mTicketsCreated)
|
||||
} else {
|
||||
statProcessor.RecordError("failed to create a ticket", err)
|
||||
telemetry.RecordUnitMeasurement(ctx, mTicketCreationsFailed)
|
||||
}
|
||||
cycleCreated++
|
||||
}
|
||||
|
||||
atomic.AddUint32(&totalCreated, uint32(cycleCreated))
|
||||
|
||||
@ -52,6 +52,7 @@ spec:
|
||||
namespace: {{ .Release.Namespace }}
|
||||
annotations:
|
||||
{{- include "openmatch.chartmeta" . | nindent 8 }}
|
||||
{{- include "prometheus.annotations" (dict "port" .Values.scaleBackend.httpPort "prometheus" .Values.global.telemetry.prometheus) | nindent 8 }}
|
||||
labels:
|
||||
app: {{ template "openmatch.name" . }}
|
||||
component: scaleBackend
|
||||
|
||||
@ -33,6 +33,57 @@ data:
|
||||
hostname: "{{ .Values.frontend.hostName }}"
|
||||
grpcport: "{{ .Values.frontend.grpcPort }}"
|
||||
httpport: "{{ .Values.frontend.httpPort }}"
|
||||
scale:
|
||||
httpport: "51509"
|
||||
{{- if .Values.global.tls.enabled }}
|
||||
tls:
|
||||
trustedCertificatePath: "{{.Values.global.tls.rootca.mountPath}}/public.cert"
|
||||
certificatefile: "{{.Values.global.tls.server.mountPath}}/public.cert"
|
||||
privatekey: "{{.Values.global.tls.server.mountPath}}/private.key"
|
||||
rootcertificatefile: "{{.Values.global.tls.rootca.mountPath}}/public.cert"
|
||||
{{- end }}
|
||||
|
||||
logging:
|
||||
level: debug
|
||||
{{- if .Values.global.telemetry.stackdriverMetrics.enabled }}
|
||||
format: stackdriver
|
||||
{{- else }}
|
||||
format: text
|
||||
{{- end }}
|
||||
rpc: {{ .Values.global.logging.rpc.enabled }}
|
||||
# Open Match applies the exponential backoff strategy for its retryable gRPC calls.
|
||||
# The settings below are the default backoff configuration used in Open Match.
|
||||
# See https://github.com/cenkalti/backoff/blob/v3/exponential.go for detailed explanations
|
||||
backoff:
|
||||
# The initial retry interval (in milliseconds)
|
||||
initialInterval: 100ms
|
||||
# maxInterval caps the maximum time elapsed for a retry interval
|
||||
maxInterval: 500ms
|
||||
# The next retry interval is multiplied by this multiplier
|
||||
multiplier: 1.5
|
||||
# Randomize the retry interval
|
||||
randFactor: 0.5
|
||||
# maxElapsedTime caps the retry time (in milliseconds)
|
||||
maxElapsedTime: 3000ms
|
||||
|
||||
telemetry:
|
||||
zpages:
|
||||
enable: "{{ .Values.global.telemetry.zpages.enabled }}"
|
||||
jaeger:
|
||||
enable: "{{ .Values.global.telemetry.jaeger.enabled }}"
|
||||
samplerFraction: {{ .Values.global.telemetry.jaeger.samplerFraction }}
|
||||
agentEndpoint: "{{ .Values.global.telemetry.jaeger.agentEndpoint }}"
|
||||
collectorEndpoint: "{{ .Values.global.telemetry.jaeger.collectorEndpoint }}"
|
||||
prometheus:
|
||||
enable: "{{ .Values.global.telemetry.prometheus.enabled }}"
|
||||
endpoint: "{{ .Values.global.telemetry.prometheus.endpoint }}"
|
||||
serviceDiscovery: "{{ .Values.global.telemetry.prometheus.serviceDiscovery }}"
|
||||
stackdriverMetrics:
|
||||
enable: "{{ .Values.global.telemetry.stackdriverMetrics.enabled }}"
|
||||
gcpProjectId: "{{ .Values.global.gcpProjectId }}"
|
||||
prefix: "{{ .Values.global.telemetry.stackdriverMetrics.prefix }}"
|
||||
reportingPeriod: "{{ .Values.global.telemetry.reportingPeriod }}"
|
||||
|
||||
matchmaker_config_override.yaml: |-
|
||||
testConfig:
|
||||
profile: "{{ .Values.testConfig.profile }}"
|
||||
@ -54,3 +105,4 @@ data:
|
||||
rangeSize: "{{ .Values.testConfig.multipool.rangeSize }}"
|
||||
rangeOverlap: "{{ .Values.testConfig.multipool.rangeOverlap }}"
|
||||
characterCount: "{{ .Values.testConfig.multipool.characterCount }}"
|
||||
|
||||
|
||||
@ -52,6 +52,7 @@ spec:
|
||||
namespace: {{ .Release.Namespace }}
|
||||
annotations:
|
||||
{{- include "openmatch.chartmeta" . | nindent 8 }}
|
||||
{{- include "prometheus.annotations" (dict "port" .Values.scaleFrontend.httpPort "prometheus" .Values.global.telemetry.prometheus) | nindent 8 }}
|
||||
labels:
|
||||
app: {{ template "openmatch.name" . }}
|
||||
component: scaleFrontend
|
||||
@ -70,4 +71,4 @@ spec:
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: {{ .Values.scaleFrontend.httpPort }}
|
||||
{{- include "openmatch.container.common" . | nindent 8 }}
|
||||
{{- include "openmatch.container.common" . | nindent 8 }}
|
||||
|
||||
@ -20,7 +20,7 @@ scaleFrontend:
|
||||
|
||||
scaleBackend:
|
||||
hostName: om-scale-backend
|
||||
httpPort: 51510
|
||||
httpPort: 51509
|
||||
replicas: 1
|
||||
image: openmatch-scale-backend
|
||||
|
||||
|
||||
@ -0,0 +1,840 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"gnetId": null,
|
||||
"graphTooltip": 0,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 16,
|
||||
"title": "Iterations",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"id": 4,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"options": {},
|
||||
"percentage": false,
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(scale_backend_iterations[5m]))",
|
||||
"format": "time_series",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "Iterations per second",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [
|
||||
{
|
||||
"colorMode": "ok",
|
||||
"fill": true,
|
||||
"line": true,
|
||||
"op": "gt",
|
||||
"value": 1,
|
||||
"yaxis": "left"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Fetch Match Iterations",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": "0",
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"id": 14,
|
||||
"panels": [],
|
||||
"title": "Tickets",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"description": "",
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 10
|
||||
},
|
||||
"id": 2,
|
||||
"legend": {
|
||||
"avg": true,
|
||||
"current": true,
|
||||
"hideEmpty": false,
|
||||
"hideZero": false,
|
||||
"max": true,
|
||||
"min": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"options": {},
|
||||
"percentage": false,
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"repeat": null,
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(scale_frontend_tickets_created[5m]))",
|
||||
"format": "time_series",
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "Tickets Created per second",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(scale_frontend_ticket_creations_failed[5m]))",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "Ticket Creations Failed per second",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Scale Frontend Ticket Creation",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": "0",
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 10
|
||||
},
|
||||
"id": 12,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"options": {},
|
||||
"percentage": false,
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(scale_backend_sum_tickets_returned[5m]))",
|
||||
"format": "time_series",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "Backend Tickets in Matches pers second",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Tickets In Matches",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": "0",
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 19
|
||||
},
|
||||
"id": 22,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"options": {},
|
||||
"percentage": false,
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(scale_backend_tickets_deleted[5m]))",
|
||||
"format": "time_series",
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "Backend Tickets Deleted per second",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(scale_backend_ticket_deletes_failed[5m]))",
|
||||
"format": "time_series",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "Backend Ticket Deletions Failed per second",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Ticket Deletion",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": "0",
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 27
|
||||
},
|
||||
"id": 18,
|
||||
"panels": [],
|
||||
"title": "Fetch MatchCalls",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 28
|
||||
},
|
||||
"id": 6,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"options": {},
|
||||
"percentage": false,
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(scale_backend_fetch_match_calls[5m]))",
|
||||
"format": "time_series",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "Fetch Match Calls Started per second",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Fetch Match Calls Started",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"decimals": null,
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": "0",
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 28
|
||||
},
|
||||
"id": 19,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"options": {},
|
||||
"percentage": false,
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(scale_backend_fetch_match_successes[5m]))",
|
||||
"format": "time_series",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "Fetch Match Calls Succeeding per second",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(scale_backend_fetch_match_errors[5m]))",
|
||||
"format": "time_series",
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "Fetch Match Calls Ending In Errors per second",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Fetch Match Results",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": "0",
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 36
|
||||
},
|
||||
"id": 21,
|
||||
"panels": [],
|
||||
"title": "Matches",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 37
|
||||
},
|
||||
"id": 8,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"options": {},
|
||||
"percentage": false,
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(scale_backend_matches_returned[5m]))",
|
||||
"format": "time_series",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "Matches Returned From Fetch Matches per second",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Matches Made",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": "0",
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 37
|
||||
},
|
||||
"id": 10,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"options": {},
|
||||
"percentage": false,
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(scale_backend_matches_assigned[5m]))",
|
||||
"format": "time_series",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "Matches Assigned per second",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(scale_backend_match_assigns_failed[5m]))",
|
||||
"format": "time_series",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "Match Assignments Failed per second",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Match Assignment",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": "0",
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"refresh": "",
|
||||
"schemaVersion": 18,
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"refresh_intervals": [
|
||||
"1s",
|
||||
"5s",
|
||||
"10s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"30m",
|
||||
"1h",
|
||||
"2h",
|
||||
"1d"
|
||||
],
|
||||
"time_options": [
|
||||
"5m",
|
||||
"15m",
|
||||
"1h",
|
||||
"6h",
|
||||
"12h",
|
||||
"24h",
|
||||
"2d",
|
||||
"7d",
|
||||
"30d"
|
||||
]
|
||||
},
|
||||
"timezone": "",
|
||||
"title": "Scale",
|
||||
"uid": "PCNBQKPWk",
|
||||
"version": 1
|
||||
}
|
||||
@ -68,6 +68,10 @@ data:
|
||||
swaggerui:
|
||||
hostname: "{{ .Values.swaggerui.hostName }}"
|
||||
httpport: "{{ .Values.swaggerui.httpPort }}"
|
||||
scale-frontend:
|
||||
httpport: "51509"
|
||||
scale-backend:
|
||||
httpport: "51510"
|
||||
{{- if .Values.global.tls.enabled }}
|
||||
tls:
|
||||
trustedCertificatePath: "{{.Values.global.tls.rootca.mountPath}}/public.cert"
|
||||
|
||||
Reference in New Issue
Block a user