2022-06-02 07:48:46 -06:00
{
"annotations" : {
"list" : [
]
} ,
"editable" : true ,
"gnetId" : null ,
"graphTooltip" : 0 ,
"hideControls" : false ,
"links" : [
{
"asDropdown" : true ,
"icon" : "external link" ,
"includeVars" : true ,
"keepTime" : true ,
"tags" : [
"tempo"
] ,
"targetBlank" : false ,
"title" : "Tempo Dashboards" ,
"type" : "dashboards"
}
] ,
"panels" : [
{
"aliasColors" : {
} ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "thresholds"
} ,
"mappings" : [
] ,
"max" : 1 ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "yellow" ,
"value" : null
} ,
{
"color" : "yellow" ,
"value" : 0.999
} ,
{
"color" : "green" ,
"value" : 1
}
]
} ,
"unit" : "percentunit"
}
} ,
"fill" : 1 ,
"gridPos" : {
"h" : 8 ,
"w" : 10 ,
"x" : 0 ,
"y" : 0
} ,
"id" : 1 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [
] ,
"nullPointMode" : "null as zero" ,
"options" : {
"displayMode" : "basic" ,
"orientation" : "horizontal" ,
"reduceOptions" : {
"calcs" : [
"lastNotNull"
] ,
"fields" : "" ,
"values" : false
}
} ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [
] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2023-04-11 10:12:55 +02:00
"expr" : "(\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_status_replicas_updated{cluster=~\"$cluster\", namespace=~\"$namespace\",statefulset=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"},\n \"tempo_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n /\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"tempo_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n) and (\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"tempo_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n > 0\n)\n" ,
2022-06-02 07:48:46 -06:00
"format" : null ,
"intervalFactor" : null ,
2023-04-11 10:12:55 +02:00
"legendFormat" : "{{tempo_service}}" ,
2022-06-02 07:48:46 -06:00
"legendLink" : null ,
"step" : null
} ,
{
2023-04-11 10:12:55 +02:00
"expr" : "(\n sum by(tempo_service) (\n label_replace(\n kube_deployment_status_replicas_updated{cluster=~\"$cluster\", namespace=~\"$namespace\",deployment=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"},\n \"tempo_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n /\n sum by(tempo_service) (\n label_replace(\n kube_deployment_spec_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"tempo_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n) and (\n sum by(tempo_service) (\n label_replace(\n kube_deployment_spec_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"tempo_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n > 0\n)\n" ,
2022-06-02 07:48:46 -06:00
"format" : null ,
"intervalFactor" : null ,
2023-04-11 10:12:55 +02:00
"legendFormat" : "{{tempo_service}}" ,
2022-06-02 07:48:46 -06:00
"legendLink" : null ,
"step" : null
}
] ,
"thresholds" : [
] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Rollout progress" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "bargauge" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [
]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"aliasColors" : {
} ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "thresholds"
} ,
"decimals" : 1 ,
"noValue" : "" ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
}
]
} ,
"unit" : "percentunit"
} ,
"overrides" : [
]
} ,
"fill" : 1 ,
"gridPos" : {
"h" : 4 ,
"w" : 2 ,
"x" : 10 ,
"y" : 0
} ,
"id" : 2 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [
] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [
] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-21 10:58:37 +02:00
"expr" : "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n" ,
2022-06-02 07:48:46 -06:00
"format" : null ,
"instant" : false ,
"interval" : "" ,
"intervalFactor" : null ,
"legendFormat" : "" ,
"legendLink" : null ,
"step" : null
}
] ,
"thresholds" : [
] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Writes - 2xx" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "stat" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [
]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"aliasColors" : {
} ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "thresholds"
} ,
"decimals" : 1 ,
"noValue" : "" ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "orange" ,
2024-04-05 16:57:25 +00:00
"value" : 0.2
2022-06-02 07:48:46 -06:00
} ,
{
"color" : "red" ,
"value" : 0.5
}
]
} ,
"unit" : "percentunit"
} ,
"overrides" : [
]
} ,
"fill" : 1 ,
"gridPos" : {
"h" : 4 ,
"w" : 2 ,
"x" : 12 ,
"y" : 0
} ,
"id" : 3 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [
] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [
] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-21 10:58:37 +02:00
"expr" : "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n" ,
2022-06-02 07:48:46 -06:00
"format" : null ,
"instant" : false ,
"interval" : "" ,
"intervalFactor" : null ,
"legendFormat" : "" ,
"legendLink" : null ,
"step" : null
}
] ,
"thresholds" : [
] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Writes - 4xx" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "stat" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [
]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"aliasColors" : {
} ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "thresholds"
} ,
"decimals" : 1 ,
"noValue" : "" ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 0.01
}
]
} ,
"unit" : "percentunit"
} ,
"overrides" : [
]
} ,
"fill" : 1 ,
"gridPos" : {
"h" : 4 ,
"w" : 2 ,
"x" : 14 ,
"y" : 0
} ,
"id" : 4 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [
] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [
] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-21 10:58:37 +02:00
"expr" : "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n" ,
2022-06-02 07:48:46 -06:00
"format" : null ,
"instant" : false ,
"interval" : "" ,
"intervalFactor" : null ,
"legendFormat" : "" ,
"legendLink" : null ,
"step" : null
}
] ,
"thresholds" : [
] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Writes - 5xx" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "stat" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [
]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"aliasColors" : {
} ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "thresholds"
} ,
"decimals" : 1 ,
"noValue" : "" ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "orange" ,
2024-04-05 16:57:25 +00:00
"value" : 0.2
2022-06-02 07:48:46 -06:00
} ,
{
"color" : "red" ,
"value" : 0.5
}
]
} ,
"unit" : "s"
} ,
"overrides" : [
]
} ,
"fill" : 1 ,
"gridPos" : {
"h" : 4 ,
"w" : 8 ,
"x" : 16 ,
"y" : 0
} ,
"id" : 5 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [
] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [
] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-21 10:58:37 +02:00
"expr" : "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}))\n" ,
2022-06-02 07:48:46 -06:00
"format" : null ,
"instant" : false ,
"interval" : "" ,
"intervalFactor" : null ,
"legendFormat" : "" ,
"legendLink" : null ,
"step" : null
}
] ,
"thresholds" : [
] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Writes 99th latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "stat" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [
]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"aliasColors" : {
} ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "thresholds"
} ,
"decimals" : 1 ,
"noValue" : "" ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
}
]
} ,
"unit" : "percentunit"
} ,
"overrides" : [
]
} ,
"fill" : 1 ,
"gridPos" : {
"h" : 4 ,
"w" : 2 ,
"x" : 10 ,
"y" : 4
} ,
"id" : 6 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [
] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [
] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-21 10:58:37 +02:00
"expr" : "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}[$__rate_interval]))\n" ,
2022-06-02 07:48:46 -06:00
"format" : null ,
"instant" : false ,
"interval" : "" ,
"intervalFactor" : null ,
"legendFormat" : "" ,
"legendLink" : null ,
"step" : null
}
] ,
"thresholds" : [
] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Reads - 2xx" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "stat" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [
]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"aliasColors" : {
} ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "thresholds"
} ,
"decimals" : 1 ,
"noValue" : "" ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "orange" ,
"value" : 0.01
} ,
{
"color" : "red" ,
2024-04-05 16:57:25 +00:00
"value" : 0.05
2022-06-02 07:48:46 -06:00
}
]
} ,
"unit" : "percentunit"
} ,
"overrides" : [
]
} ,
"fill" : 1 ,
"gridPos" : {
"h" : 4 ,
"w" : 2 ,
"x" : 12 ,
"y" : 4
} ,
"id" : 7 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [
] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [
] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-21 10:58:37 +02:00
"expr" : "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}[$__rate_interval]))\n" ,
2022-06-02 07:48:46 -06:00
"format" : null ,
"instant" : false ,
"interval" : "" ,
"intervalFactor" : null ,
"legendFormat" : "" ,
"legendLink" : null ,
"step" : null
}
] ,
"thresholds" : [
] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Reads - 4xx" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "stat" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [
]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"aliasColors" : {
} ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "thresholds"
} ,
"decimals" : 1 ,
"noValue" : "" ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 0.01
}
]
} ,
"unit" : "percentunit"
} ,
"overrides" : [
]
} ,
"fill" : 1 ,
"gridPos" : {
"h" : 4 ,
"w" : 2 ,
"x" : 14 ,
"y" : 4
} ,
"id" : 8 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [
] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [
] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-21 10:58:37 +02:00
"expr" : "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}[$__rate_interval]))\n" ,
2022-06-02 07:48:46 -06:00
"format" : null ,
"instant" : false ,
"interval" : "" ,
"intervalFactor" : null ,
"legendFormat" : "" ,
"legendLink" : null ,
"step" : null
}
] ,
"thresholds" : [
] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Reads - 5xx" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "stat" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [
]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"aliasColors" : {
} ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "thresholds"
} ,
"decimals" : 1 ,
"noValue" : "" ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "orange" ,
"value" : 1
} ,
{
"color" : "red" ,
"value" : 2.5
}
]
} ,
"unit" : "s"
} ,
"overrides" : [
]
} ,
"fill" : 1 ,
"gridPos" : {
"h" : 4 ,
"w" : 8 ,
"x" : 16 ,
"y" : 4
} ,
"id" : 9 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [
] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [
] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-21 10:58:37 +02:00
"expr" : "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}))\n" ,
2022-06-02 07:48:46 -06:00
"format" : null ,
"instant" : false ,
"interval" : "" ,
"intervalFactor" : null ,
"legendFormat" : "" ,
"legendLink" : null ,
"step" : null
}
] ,
"thresholds" : [
] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Reads 99th latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "stat" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [
]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"aliasColors" : {
} ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "thresholds"
} ,
"decimals" : 0 ,
"noValue" : "All healthy" ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "orange" ,
"value" : 1
} ,
{
"color" : "red" ,
"value" : 2
}
]
} ,
"unit" : "short"
} ,
"overrides" : [
]
} ,
"fill" : 1 ,
"gridPos" : {
"h" : 8 ,
"w" : 10 ,
"x" : 0 ,
"y" : 8
} ,
"id" : 10 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [
] ,
"nullPointMode" : "null as zero" ,
"options" : {
"text" : {
"titleSize" : 14 ,
"valueSize" : 14
}
} ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [
] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"expr" : "kube_deployment_status_replicas_unavailable{cluster=~\"$cluster\", namespace=~\"$namespace\", deployment=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"}\n> 0\n" ,
"format" : null ,
"instant" : true ,
"interval" : "" ,
"intervalFactor" : null ,
"legendFormat" : "{{deployment}}" ,
"legendLink" : null ,
"step" : null
} ,
{
"expr" : "kube_statefulset_status_replicas_current{cluster=~\"$cluster\", namespace=~\"$namespace\", statefulset=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"} -\nkube_statefulset_status_replicas_ready {cluster=~\"$cluster\", namespace=~\"$namespace\", statefulset=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"}\n> 0\n" ,
"format" : null ,
"instant" : true ,
"interval" : "" ,
"intervalFactor" : null ,
"legendFormat" : "{{statefulset}}" ,
"legendLink" : null ,
"step" : null
}
] ,
"thresholds" : [
] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Unhealthy pods" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "stat" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [
]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"datasource" : "$datasource" ,
"fieldConfig" : {
"overrides" : [
{
"matcher" : {
"id" : "byRegexp" ,
"options" : "r.*"
} ,
"properties" : [
{
"id" : "custom.align" ,
"value" : "center"
}
]
}
]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 6 ,
"x" : 10 ,
"y" : 8
} ,
"id" : 11 ,
"targets" : [
{
"expr" : "count by(container, version) (\n label_replace(\n kube_pod_container_info{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"},\n \"version\", \"$1\", \"image\", \".*:(.+)-.*\"\n )\n)\n" ,
"instant" : true ,
"legendFormat" : "" ,
"refId" : "A"
}
] ,
"title" : "Pods count per version" ,
"transformations" : [
{
"id" : "labelsToFields" ,
"options" : {
"valueLabel" : "version"
}
} ,
{
"id" : "organize" ,
"options" : {
"excludeByName" : {
"Time" : true
}
}
} ,
{
"id" : "sortBy" ,
"options" : {
"fields" : {
} ,
"sort" : [
{
"field" : "container"
}
]
}
}
] ,
"type" : "table"
} ,
{
"aliasColors" : {
} ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fill" : 1 ,
"gridPos" : {
"h" : 8 ,
"w" : 8 ,
"x" : 16 ,
"y" : 8
} ,
"id" : 12 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [
] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [
] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-21 10:58:37 +02:00
"expr" : "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}))[1h:])\n)\n" ,
2022-06-02 07:48:46 -06:00
"format" : "time_series" ,
"interval" : "1m" ,
"intervalFactor" : 2 ,
"legendFormat" : "writes" ,
"legendLink" : null ,
"step" : 10
} ,
{
2022-10-21 10:58:37 +02:00
"expr" : "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}))[1h:])\n)\n" ,
2022-06-02 07:48:46 -06:00
"format" : "time_series" ,
"interval" : "1m" ,
"intervalFactor" : 2 ,
"legendFormat" : "reads" ,
"legendLink" : null ,
"step" : 10
}
] ,
"thresholds" : [
] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Latency vs 24h ago" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [
]
} ,
"yaxes" : [
{
"format" : "percentunit" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
}
] ,
2023-04-01 00:21:00 +05:30
"refresh" : "" ,
2022-06-02 07:48:46 -06:00
"rows" : null ,
"schemaVersion" : 27 ,
"style" : "dark" ,
"tags" : [
"tempo"
] ,
"templating" : {
"list" : [
{
"current" : {
"text" : "default" ,
"value" : "default"
} ,
"hide" : 0 ,
"label" : "Data Source" ,
"name" : "datasource" ,
"options" : [
] ,
"query" : "prometheus" ,
"refresh" : 1 ,
"regex" : "" ,
"type" : "datasource"
} ,
{
2022-11-28 14:32:26 +01:00
"allValue" : null ,
2022-06-02 07:48:46 -06:00
"current" : {
"selected" : true ,
"text" : "All" ,
"value" : "$__all"
} ,
"datasource" : "$datasource" ,
"hide" : 0 ,
"includeAll" : false ,
"label" : "cluster" ,
"multi" : true ,
"name" : "cluster" ,
"options" : [
] ,
"query" : "label_values(tempo_build_info, cluster)" ,
"refresh" : 1 ,
"regex" : "" ,
"sort" : 2 ,
"tagValuesQuery" : "" ,
"tags" : [
] ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
} ,
{
"allValue" : null ,
"current" : {
"selected" : true ,
"text" : "All" ,
"value" : "$__all"
} ,
"datasource" : "$datasource" ,
"hide" : 0 ,
"includeAll" : false ,
"label" : "namespace" ,
"multi" : true ,
"name" : "namespace" ,
"options" : [
] ,
2022-09-29 22:41:16 +05:30
"query" : "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)" ,
2022-06-02 07:48:46 -06:00
"refresh" : 1 ,
"regex" : "" ,
"sort" : 2 ,
"tagValuesQuery" : "" ,
"tags" : [
] ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
}
]
} ,
"time" : {
"from" : "now-1h" ,
"to" : "now"
} ,
"timepicker" : {
"refresh_intervals" : [
"5s" ,
"10s" ,
"30s" ,
"1m" ,
"5m" ,
"15m" ,
"30m" ,
"1h" ,
"2h" ,
"1d"
] ,
"time_options" : [
"5m" ,
"15m" ,
"1h" ,
"6h" ,
"12h" ,
"24h" ,
"2d" ,
"7d" ,
"30d"
]
} ,
"timezone" : "utc" ,
"title" : "Rollout progress" ,
"uid" : "217b16a8c5966b32c770225dea289b19" ,
"version" : 0
2024-04-05 16:57:25 +00:00
}