Prometheus Metrics

1
apiVersion: monitoring.coreos.com/v1
2
kind: ServiceMonitor
3
metadata:
4
  name: cluster-readiness-engine-metrics-monitor
5
  namespace: cluster-readiness-engine
6
spec:
7
  endpoints:
8
    - path: /metrics
9
      port: https
10
      scheme: https
11
      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
12
      tlsConfig:
13
        insecureSkipVerify: true  # Use cert-manager in production
14
  selector:
15
    matchLabels:
16
      control-plane: manager

1
apiVersion: monitoring.coreos.com/v1
2
kind: PrometheusRule
3
metadata:
4
  name: cre-alerts
5
  labels:
6
    app.kubernetes.io/name: cluster-readiness-engine
7
spec:
8
  groups:
9
    - name: burnin.rules
10
      rules:
11
        # Alert when hardware failure rate exceeds threshold
12
        - alert: CREHighHardwareFailureRate
13
          expr: |
14
            sum(rate(cre_hardware_failures_detected_total[1h])) by (namespace) > 0.5
15
          for: 10m
16
          labels:
17
            severity: critical
18
          annotations:
19
            summary: "High hardware failure rate in namespace {{ $labels.namespace }}"
20
            description: >
21
              More than 0.5 hardware failures per second detected in
22
              namespace {{ $labels.namespace }} over the last hour.
23
24
        # Alert when goodput drops below acceptable threshold
25
        - alert: CRELowGoodput
26
          expr: |
27
            cre_goodput_ratio < 0.75
28
            and cre_goodput_ratio > 0
29
          for: 15m
30
          labels:
31
            severity: warning
32
          annotations:
33
            summary: "Low goodput for measurement {{ $labels.measurement }}"
34
            description: >
35
              Goodput ratio for {{ $labels.measurement }} in
36
              {{ $labels.namespace }} is {{ $value | humanize }},
37
              below the 75% threshold. Check for frequent checkpointing
38
              or rescheduling overhead.
39
40
        # Alert when a job appears stuck (in_progress for too long)
41
        - alert: CREJobStuck
42
          expr: |
43
            cre_job_status{status="in_progress"} == 1
44
            unless on(namespace, job) (
45
              increase(cre_reconcile_total[30m]) > 0
46
            )
47
          for: 30m
48
          labels:
49
            severity: warning
50
          annotations:
51
            summary: "CRE job {{ $labels.job }} appears stuck"
52
            description: >
53
              Job {{ $labels.job }} in namespace {{ $labels.namespace }}
54
              has been in_progress for at least 30 minutes with no
55
              reconciliation activity.