{
 "annotations": {
  "list": [

  ]
 },
 "editable": true,
 "gnetId": null,
 "graphTooltip": 0,
 "hideControls": false,
 "links": [
  {
   "asDropdown": true,
   "icon": "external link",
   "includeVars": true,
   "keepTime": true,
   "tags": [
    "tempo"
   ],
   "targetBlank": false,
   "title": "Tempo Dashboards",
   "type": "dashboards"
  }
 ],
 "panels": [
  {
   "aliasColors": {

   },
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "$datasource",
   "fieldConfig": {
    "defaults": {
     "color": {
      "mode": "thresholds"
     },
     "mappings": [

     ],
     "max": 1,
     "min": 0,
     "thresholds": {
      "mode": "absolute",
      "steps": [
       {
        "color": "yellow",
        "value": null
       },
       {
        "color": "yellow",
        "value": 0.999
       },
       {
        "color": "green",
        "value": 1
       }
      ]
     },
     "unit": "percentunit"
    }
   },
   "fill": 1,
   "gridPos": {
    "h": 8,
    "w": 10,
    "x": 0,
    "y": 0
   },
   "id": 1,
   "legend": {
    "avg": false,
    "current": false,
    "max": false,
    "min": false,
    "show": true,
    "total": false,
    "values": false
   },
   "lines": true,
   "linewidth": 1,
   "links": [

   ],
   "nullPointMode": "null as zero",
   "options": {
    "displayMode": "basic",
    "orientation": "horizontal",
    "reduceOptions": {
     "calcs": [
      "lastNotNull"
     ],
     "fields": "",
     "values": false
    }
   },
   "percentage": false,
   "pointradius": 5,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [

   ],
   "spaceLength": 10,
   "span": 6,
   "stack": false,
   "steppedLine": false,
   "targets": [
    {
     "expr": "(\n  sum by(tempo_service) (\n    label_replace(\n      kube_statefulset_status_replicas_updated{cluster=~\"$cluster\", namespace=~\"$namespace\",statefulset=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"},\n      \"tempo_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n    )\n  )\n  /\n  sum by(tempo_service) (\n    label_replace(\n      kube_statefulset_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n      \"tempo_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n    )\n  )\n) and (\n  sum by(tempo_service) (\n    label_replace(\n      kube_statefulset_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n      \"tempo_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n    )\n  )\n  > 0\n)\n",
     "format": null,
     "intervalFactor": null,
     "legendFormat": "{{tempo_service}}",
     "legendLink": null,
     "step": null
    },
    {
     "expr": "(\n  sum by(tempo_service) (\n    label_replace(\n      kube_deployment_status_replicas_updated{cluster=~\"$cluster\", namespace=~\"$namespace\",deployment=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"},\n      \"tempo_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n    )\n  )\n  /\n  sum by(tempo_service) (\n    label_replace(\n      kube_deployment_spec_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n      \"tempo_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n    )\n  )\n) and (\n  sum by(tempo_service) (\n    label_replace(\n      kube_deployment_spec_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n      \"tempo_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n    )\n  )\n  > 0\n)\n",
     "format": null,
     "intervalFactor": null,
     "legendFormat": "{{tempo_service}}",
     "legendLink": null,
     "step": null
    }
   ],
   "thresholds": [

   ],
   "timeFrom": null,
   "timeShift": null,
   "title": "Rollout progress",
   "tooltip": {
    "shared": true,
    "sort": 2,
    "value_type": "individual"
   },
   "type": "bargauge",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": [

    ]
   },
   "yaxes": [
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": 0,
     "show": true
    },
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": false
    }
   ]
  },
  {
   "aliasColors": {

   },
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "$datasource",
   "fieldConfig": {
    "defaults": {
     "color": {
      "mode": "thresholds"
     },
     "decimals": 1,
     "noValue": "",
     "thresholds": {
      "mode": "absolute",
      "steps": [
       {
        "color": "green",
        "value": null
       }
      ]
     },
     "unit": "percentunit"
    },
    "overrides": [

    ]
   },
   "fill": 1,
   "gridPos": {
    "h": 4,
    "w": 2,
    "x": 10,
    "y": 0
   },
   "id": 2,
   "legend": {
    "avg": false,
    "current": false,
    "max": false,
    "min": false,
    "show": true,
    "total": false,
    "values": false
   },
   "lines": true,
   "linewidth": 1,
   "links": [

   ],
   "nullPointMode": "null as zero",
   "percentage": false,
   "pointradius": 5,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [

   ],
   "spaceLength": 10,
   "span": 6,
   "stack": false,
   "steppedLine": false,
   "targets": [
    {
     "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n",
     "format": null,
     "instant": false,
     "interval": "",
     "intervalFactor": null,
     "legendFormat": "",
     "legendLink": null,
     "step": null
    }
   ],
   "thresholds": [

   ],
   "timeFrom": null,
   "timeShift": null,
   "title": "Writes - 2xx",
   "tooltip": {
    "shared": true,
    "sort": 2,
    "value_type": "individual"
   },
   "type": "stat",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": [

    ]
   },
   "yaxes": [
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": 0,
     "show": true
    },
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": false
    }
   ]
  },
  {
   "aliasColors": {

   },
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "$datasource",
   "fieldConfig": {
    "defaults": {
     "color": {
      "mode": "thresholds"
     },
     "decimals": 1,
     "noValue": "",
     "thresholds": {
      "mode": "absolute",
      "steps": [
       {
        "color": "green",
        "value": null
       },
       {
        "color": "orange",
        "value": 0.2
       },
       {
        "color": "red",
        "value": 0.5
       }
      ]
     },
     "unit": "percentunit"
    },
    "overrides": [

    ]
   },
   "fill": 1,
   "gridPos": {
    "h": 4,
    "w": 2,
    "x": 12,
    "y": 0
   },
   "id": 3,
   "legend": {
    "avg": false,
    "current": false,
    "max": false,
    "min": false,
    "show": true,
    "total": false,
    "values": false
   },
   "lines": true,
   "linewidth": 1,
   "links": [

   ],
   "nullPointMode": "null as zero",
   "percentage": false,
   "pointradius": 5,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [

   ],
   "spaceLength": 10,
   "span": 6,
   "stack": false,
   "steppedLine": false,
   "targets": [
    {
     "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n",
     "format": null,
     "instant": false,
     "interval": "",
     "intervalFactor": null,
     "legendFormat": "",
     "legendLink": null,
     "step": null
    }
   ],
   "thresholds": [

   ],
   "timeFrom": null,
   "timeShift": null,
   "title": "Writes - 4xx",
   "tooltip": {
    "shared": true,
    "sort": 2,
    "value_type": "individual"
   },
   "type": "stat",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": [

    ]
   },
   "yaxes": [
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": 0,
     "show": true
    },
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": false
    }
   ]
  },
  {
   "aliasColors": {

   },
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "$datasource",
   "fieldConfig": {
    "defaults": {
     "color": {
      "mode": "thresholds"
     },
     "decimals": 1,
     "noValue": "",
     "thresholds": {
      "mode": "absolute",
      "steps": [
       {
        "color": "green",
        "value": null
       },
       {
        "color": "red",
        "value": 0.01
       }
      ]
     },
     "unit": "percentunit"
    },
    "overrides": [

    ]
   },
   "fill": 1,
   "gridPos": {
    "h": 4,
    "w": 2,
    "x": 14,
    "y": 0
   },
   "id": 4,
   "legend": {
    "avg": false,
    "current": false,
    "max": false,
    "min": false,
    "show": true,
    "total": false,
    "values": false
   },
   "lines": true,
   "linewidth": 1,
   "links": [

   ],
   "nullPointMode": "null as zero",
   "percentage": false,
   "pointradius": 5,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [

   ],
   "spaceLength": 10,
   "span": 6,
   "stack": false,
   "steppedLine": false,
   "targets": [
    {
     "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n",
     "format": null,
     "instant": false,
     "interval": "",
     "intervalFactor": null,
     "legendFormat": "",
     "legendLink": null,
     "step": null
    }
   ],
   "thresholds": [

   ],
   "timeFrom": null,
   "timeShift": null,
   "title": "Writes - 5xx",
   "tooltip": {
    "shared": true,
    "sort": 2,
    "value_type": "individual"
   },
   "type": "stat",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": [

    ]
   },
   "yaxes": [
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": 0,
     "show": true
    },
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": false
    }
   ]
  },
  {
   "aliasColors": {

   },
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "$datasource",
   "fieldConfig": {
    "defaults": {
     "color": {
      "mode": "thresholds"
     },
     "decimals": 1,
     "noValue": "",
     "thresholds": {
      "mode": "absolute",
      "steps": [
       {
        "color": "green",
        "value": null
       },
       {
        "color": "orange",
        "value": 0.2
       },
       {
        "color": "red",
        "value": 0.5
       }
      ]
     },
     "unit": "s"
    },
    "overrides": [

    ]
   },
   "fill": 1,
   "gridPos": {
    "h": 4,
    "w": 8,
    "x": 16,
    "y": 0
   },
   "id": 5,
   "legend": {
    "avg": false,
    "current": false,
    "max": false,
    "min": false,
    "show": true,
    "total": false,
    "values": false
   },
   "lines": true,
   "linewidth": 1,
   "links": [

   ],
   "nullPointMode": "null as zero",
   "percentage": false,
   "pointradius": 5,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [

   ],
   "spaceLength": 10,
   "span": 6,
   "stack": false,
   "steppedLine": false,
   "targets": [
    {
     "expr": "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}))\n",
     "format": null,
     "instant": false,
     "interval": "",
     "intervalFactor": null,
     "legendFormat": "",
     "legendLink": null,
     "step": null
    }
   ],
   "thresholds": [

   ],
   "timeFrom": null,
   "timeShift": null,
   "title": "Writes 99th latency",
   "tooltip": {
    "shared": true,
    "sort": 2,
    "value_type": "individual"
   },
   "type": "stat",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": [

    ]
   },
   "yaxes": [
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": 0,
     "show": true
    },
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": false
    }
   ]
  },
  {
   "aliasColors": {

   },
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "$datasource",
   "fieldConfig": {
    "defaults": {
     "color": {
      "mode": "thresholds"
     },
     "decimals": 1,
     "noValue": "",
     "thresholds": {
      "mode": "absolute",
      "steps": [
       {
        "color": "green",
        "value": null
       }
      ]
     },
     "unit": "percentunit"
    },
    "overrides": [

    ]
   },
   "fill": 1,
   "gridPos": {
    "h": 4,
    "w": 2,
    "x": 10,
    "y": 4
   },
   "id": 6,
   "legend": {
    "avg": false,
    "current": false,
    "max": false,
    "min": false,
    "show": true,
    "total": false,
    "values": false
   },
   "lines": true,
   "linewidth": 1,
   "links": [

   ],
   "nullPointMode": "null as zero",
   "percentage": false,
   "pointradius": 5,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [

   ],
   "spaceLength": 10,
   "span": 6,
   "stack": false,
   "steppedLine": false,
   "targets": [
    {
     "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}[$__rate_interval]))\n",
     "format": null,
     "instant": false,
     "interval": "",
     "intervalFactor": null,
     "legendFormat": "",
     "legendLink": null,
     "step": null
    }
   ],
   "thresholds": [

   ],
   "timeFrom": null,
   "timeShift": null,
   "title": "Reads - 2xx",
   "tooltip": {
    "shared": true,
    "sort": 2,
    "value_type": "individual"
   },
   "type": "stat",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": [

    ]
   },
   "yaxes": [
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": 0,
     "show": true
    },
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": false
    }
   ]
  },
  {
   "aliasColors": {

   },
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "$datasource",
   "fieldConfig": {
    "defaults": {
     "color": {
      "mode": "thresholds"
     },
     "decimals": 1,
     "noValue": "",
     "thresholds": {
      "mode": "absolute",
      "steps": [
       {
        "color": "green",
        "value": null
       },
       {
        "color": "orange",
        "value": 0.01
       },
       {
        "color": "red",
        "value": 0.05
       }
      ]
     },
     "unit": "percentunit"
    },
    "overrides": [

    ]
   },
   "fill": 1,
   "gridPos": {
    "h": 4,
    "w": 2,
    "x": 12,
    "y": 4
   },
   "id": 7,
   "legend": {
    "avg": false,
    "current": false,
    "max": false,
    "min": false,
    "show": true,
    "total": false,
    "values": false
   },
   "lines": true,
   "linewidth": 1,
   "links": [

   ],
   "nullPointMode": "null as zero",
   "percentage": false,
   "pointradius": 5,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [

   ],
   "spaceLength": 10,
   "span": 6,
   "stack": false,
   "steppedLine": false,
   "targets": [
    {
     "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}[$__rate_interval]))\n",
     "format": null,
     "instant": false,
     "interval": "",
     "intervalFactor": null,
     "legendFormat": "",
     "legendLink": null,
     "step": null
    }
   ],
   "thresholds": [

   ],
   "timeFrom": null,
   "timeShift": null,
   "title": "Reads - 4xx",
   "tooltip": {
    "shared": true,
    "sort": 2,
    "value_type": "individual"
   },
   "type": "stat",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": [

    ]
   },
   "yaxes": [
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": 0,
     "show": true
    },
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": false
    }
   ]
  },
  {
   "aliasColors": {

   },
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "$datasource",
   "fieldConfig": {
    "defaults": {
     "color": {
      "mode": "thresholds"
     },
     "decimals": 1,
     "noValue": "",
     "thresholds": {
      "mode": "absolute",
      "steps": [
       {
        "color": "green",
        "value": null
       },
       {
        "color": "red",
        "value": 0.01
       }
      ]
     },
     "unit": "percentunit"
    },
    "overrides": [

    ]
   },
   "fill": 1,
   "gridPos": {
    "h": 4,
    "w": 2,
    "x": 14,
    "y": 4
   },
   "id": 8,
   "legend": {
    "avg": false,
    "current": false,
    "max": false,
    "min": false,
    "show": true,
    "total": false,
    "values": false
   },
   "lines": true,
   "linewidth": 1,
   "links": [

   ],
   "nullPointMode": "null as zero",
   "percentage": false,
   "pointradius": 5,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [

   ],
   "spaceLength": 10,
   "span": 6,
   "stack": false,
   "steppedLine": false,
   "targets": [
    {
     "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}[$__rate_interval]))\n",
     "format": null,
     "instant": false,
     "interval": "",
     "intervalFactor": null,
     "legendFormat": "",
     "legendLink": null,
     "step": null
    }
   ],
   "thresholds": [

   ],
   "timeFrom": null,
   "timeShift": null,
   "title": "Reads - 5xx",
   "tooltip": {
    "shared": true,
    "sort": 2,
    "value_type": "individual"
   },
   "type": "stat",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": [

    ]
   },
   "yaxes": [
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": 0,
     "show": true
    },
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": false
    }
   ]
  },
  {
   "aliasColors": {

   },
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "$datasource",
   "fieldConfig": {
    "defaults": {
     "color": {
      "mode": "thresholds"
     },
     "decimals": 1,
     "noValue": "",
     "thresholds": {
      "mode": "absolute",
      "steps": [
       {
        "color": "green",
        "value": null
       },
       {
        "color": "orange",
        "value": 1
       },
       {
        "color": "red",
        "value": 2.5
       }
      ]
     },
     "unit": "s"
    },
    "overrides": [

    ]
   },
   "fill": 1,
   "gridPos": {
    "h": 4,
    "w": 8,
    "x": 16,
    "y": 4
   },
   "id": 9,
   "legend": {
    "avg": false,
    "current": false,
    "max": false,
    "min": false,
    "show": true,
    "total": false,
    "values": false
   },
   "lines": true,
   "linewidth": 1,
   "links": [

   ],
   "nullPointMode": "null as zero",
   "percentage": false,
   "pointradius": 5,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [

   ],
   "spaceLength": 10,
   "span": 6,
   "stack": false,
   "steppedLine": false,
   "targets": [
    {
     "expr": "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}))\n",
     "format": null,
     "instant": false,
     "interval": "",
     "intervalFactor": null,
     "legendFormat": "",
     "legendLink": null,
     "step": null
    }
   ],
   "thresholds": [

   ],
   "timeFrom": null,
   "timeShift": null,
   "title": "Reads 99th latency",
   "tooltip": {
    "shared": true,
    "sort": 2,
    "value_type": "individual"
   },
   "type": "stat",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": [

    ]
   },
   "yaxes": [
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": 0,
     "show": true
    },
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": false
    }
   ]
  },
  {
   "aliasColors": {

   },
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "$datasource",
   "fieldConfig": {
    "defaults": {
     "color": {
      "mode": "thresholds"
     },
     "decimals": 0,
     "noValue": "All healthy",
     "thresholds": {
      "mode": "absolute",
      "steps": [
       {
        "color": "green",
        "value": null
       },
       {
        "color": "orange",
        "value": 1
       },
       {
        "color": "red",
        "value": 2
       }
      ]
     },
     "unit": "short"
    },
    "overrides": [

    ]
   },
   "fill": 1,
   "gridPos": {
    "h": 8,
    "w": 10,
    "x": 0,
    "y": 8
   },
   "id": 10,
   "legend": {
    "avg": false,
    "current": false,
    "max": false,
    "min": false,
    "show": true,
    "total": false,
    "values": false
   },
   "lines": true,
   "linewidth": 1,
   "links": [

   ],
   "nullPointMode": "null as zero",
   "options": {
    "text": {
     "titleSize": 14,
     "valueSize": 14
    }
   },
   "percentage": false,
   "pointradius": 5,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [

   ],
   "spaceLength": 10,
   "span": 6,
   "stack": false,
   "steppedLine": false,
   "targets": [
    {
     "expr": "kube_deployment_status_replicas_unavailable{cluster=~\"$cluster\", namespace=~\"$namespace\", deployment=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"}\n> 0\n",
     "format": null,
     "instant": true,
     "interval": "",
     "intervalFactor": null,
     "legendFormat": "{{deployment}}",
     "legendLink": null,
     "step": null
    },
    {
     "expr": "kube_statefulset_status_replicas_current{cluster=~\"$cluster\", namespace=~\"$namespace\", statefulset=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"} -\nkube_statefulset_status_replicas_ready {cluster=~\"$cluster\", namespace=~\"$namespace\", statefulset=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"}\n> 0\n",
     "format": null,
     "instant": true,
     "interval": "",
     "intervalFactor": null,
     "legendFormat": "{{statefulset}}",
     "legendLink": null,
     "step": null
    }
   ],
   "thresholds": [

   ],
   "timeFrom": null,
   "timeShift": null,
   "title": "Unhealthy pods",
   "tooltip": {
    "shared": true,
    "sort": 2,
    "value_type": "individual"
   },
   "type": "stat",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": [

    ]
   },
   "yaxes": [
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": 0,
     "show": true
    },
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": false
    }
   ]
  },
  {
   "datasource": "$datasource",
   "fieldConfig": {
    "overrides": [
     {
      "matcher": {
       "id": "byRegexp",
       "options": "r.*"
      },
      "properties": [
       {
        "id": "custom.align",
        "value": "center"
       }
      ]
     }
    ]
   },
   "gridPos": {
    "h": 8,
    "w": 6,
    "x": 10,
    "y": 8
   },
   "id": 11,
   "targets": [
    {
     "expr": "count by(container, version) (\n  label_replace(\n    kube_pod_container_info{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"},\n    \"version\", \"$1\", \"image\", \".*:(.+)-.*\"\n  )\n)\n",
     "instant": true,
     "legendFormat": "",
     "refId": "A"
    }
   ],
   "title": "Pods count per version",
   "transformations": [
    {
     "id": "labelsToFields",
     "options": {
      "valueLabel": "version"
     }
    },
    {
     "id": "organize",
     "options": {
      "excludeByName": {
       "Time": true
      }
     }
    },
    {
     "id": "sortBy",
     "options": {
      "fields": {

      },
      "sort": [
       {
        "field": "container"
       }
      ]
     }
    }
   ],
   "type": "table"
  },
  {
   "aliasColors": {

   },
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "$datasource",
   "fill": 1,
   "gridPos": {
    "h": 8,
    "w": 8,
    "x": 16,
    "y": 8
   },
   "id": 12,
   "legend": {
    "avg": false,
    "current": false,
    "max": false,
    "min": false,
    "show": true,
    "total": false,
    "values": false
   },
   "lines": true,
   "linewidth": 1,
   "links": [

   ],
   "nullPointMode": "null as zero",
   "percentage": false,
   "pointradius": 5,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [

   ],
   "spaceLength": 10,
   "span": 6,
   "stack": false,
   "steppedLine": false,
   "targets": [
    {
     "expr": "1 - (\n  avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"} offset 24h))[1h:])\n  /\n  avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}))[1h:])\n)\n",
     "format": "time_series",
     "interval": "1m",
     "intervalFactor": 2,
     "legendFormat": "writes",
     "legendLink": null,
     "step": 10
    },
    {
     "expr": "1 - (\n  avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"} offset 24h))[1h:])\n  /\n  avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}))[1h:])\n)\n",
     "format": "time_series",
     "interval": "1m",
     "intervalFactor": 2,
     "legendFormat": "reads",
     "legendLink": null,
     "step": 10
    }
   ],
   "thresholds": [

   ],
   "timeFrom": null,
   "timeShift": null,
   "title": "Latency vs 24h ago",
   "tooltip": {
    "shared": true,
    "sort": 2,
    "value_type": "individual"
   },
   "type": "graph",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": [

    ]
   },
   "yaxes": [
    {
     "format": "percentunit",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": true
    },
    {
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": false
    }
   ]
  }
 ],
 "refresh": "",
 "rows": null,
 "schemaVersion": 27,
 "style": "dark",
 "tags": [
  "tempo"
 ],
 "templating": {
  "list": [
   {
    "current": {
     "text": "default",
     "value": "default"
    },
    "hide": 0,
    "label": "Data Source",
    "name": "datasource",
    "options": [

    ],
    "query": "prometheus",
    "refresh": 1,
    "regex": "",
    "type": "datasource"
   },
   {
    "allValue": null,
    "current": {
     "selected": true,
     "text": "All",
     "value": "$__all"
    },
    "datasource": "$datasource",
    "hide": 0,
    "includeAll": false,
    "label": "cluster",
    "multi": true,
    "name": "cluster",
    "options": [

    ],
    "query": "label_values(tempo_build_info, cluster)",
    "refresh": 1,
    "regex": "",
    "sort": 2,
    "tagValuesQuery": "",
    "tags": [

    ],
    "tagsQuery": "",
    "type": "query",
    "useTags": false
   },
   {
    "allValue": null,
    "current": {
     "selected": true,
     "text": "All",
     "value": "$__all"
    },
    "datasource": "$datasource",
    "hide": 0,
    "includeAll": false,
    "label": "namespace",
    "multi": true,
    "name": "namespace",
    "options": [

    ],
    "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)",
    "refresh": 1,
    "regex": "",
    "sort": 2,
    "tagValuesQuery": "",
    "tags": [

    ],
    "tagsQuery": "",
    "type": "query",
    "useTags": false
   }
  ]
 },
 "time": {
  "from": "now-1h",
  "to": "now"
 },
 "timepicker": {
  "refresh_intervals": [
   "5s",
   "10s",
   "30s",
   "1m",
   "5m",
   "15m",
   "30m",
   "1h",
   "2h",
   "1d"
  ],
  "time_options": [
   "5m",
   "15m",
   "1h",
   "6h",
   "12h",
   "24h",
   "2d",
   "7d",
   "30d"
  ]
 },
 "timezone": "utc",
 "title": "Rollout progress",
 "uid": "217b16a8c5966b32c770225dea289b19",
 "version": 0
}