Increase length of time compactions have to fail (#4315)

* increase length of time compactions have to fail Signed-off-by: Joe Elliott <number101010@gmail.com> * gen Signed-off-by: Joe Elliott <number101010@gmail.com> --------- Signed-off-by: Joe Elliott <number101010@gmail.com>
2025-03-15 09:20:00 +00:00 · 2024-11-12 13:25:41 -05:00
parent 2bc0b62c91
commit 29dbbd28f3
2 changed files with 2 additions and 2 deletions
--- a/operations/tempo-mixin-compiled/alerts.yaml
+++ b/operations/tempo-mixin-compiled/alerts.yaml
@ -36,7 +36,7 @@
    "expr": |
      sum by (cluster, namespace) (increase(tempodb_compaction_errors_total{}[1h])) > 2 and
      sum by (cluster, namespace) (increase(tempodb_compaction_errors_total{}[5m])) > 0
-    "for": "5m"
+    "for": "1h"
    "labels":
      "severity": "critical"
  - "alert": "TempoIngesterFlushesUnhealthy"
--- a/operations/tempo-mixin/alerts.libsonnet
+++ b/operations/tempo-mixin/alerts.libsonnet
@ -50,7 +50,7 @@
          },
          {
            alert: 'TempoCompactionsFailing',
-            'for': '5m',
+            'for': '1h',
            expr: |||
              sum by (%s) (increase(tempodb_compaction_errors_total{}[1h])) > %s and
              sum by (%s) (increase(tempodb_compaction_errors_total{}[5m])) > 0