Best Practice: Alerting on Symptoms, Not Causes

# slo-definitions.yaml (store in version control)
services:
  payment-service:
    slos:
      - name: availability
        description: "Availability of the Payment Service"
        target: 99.9  # 99.9% = 8.6 hours of downtime/year allowed
        window: 30d
        metric:
          good_events: "http_requests_total{service='payment',code!~'5..'}"
          total_events: "http_requests_total{service='payment'}"

      - name: latency-p99
        description: "p99 latency for payment requests"
        target: 99.0  # 99% of requests under 500ms
        window: 30d
        metric:
          good_events: "http_request_duration_seconds_bucket{service='payment',le='0.5'}"
          total_events: "http_requests_total{service='payment'}"

      - name: error-rate
        description: "Error rate of all requests"
        target: 99.9  # < 0.1% error rate
        window: 30d
        metric:
          good_events: "http_requests_total{service='payment',code!~'5..'}"
          total_events: "http_requests_total{service='payment'}"

# prometheus-alerts.yaml
groups:
  - name: payment-service-slo
    rules:
      # Fast-Burn Alert: Page immediately when error budget burns 5x faster
      - alert: PaymentServiceHighErrorBudgetBurn
        expr: |
          (
            rate(http_requests_total{service="payment",code=~"5.."}[1h])
            /
            rate(http_requests_total{service="payment"}[1h])
          ) > (5 * 0.001)  # 5x burn rate, SLO target = 99.9% -> 0.1% error budget
        for: 2m
        labels:
          severity: critical
          team: payment
        annotations:
          summary: "Payment Service high error budget burn rate"
          description: "Error budget burning 5x faster than allowed. At this rate, monthly budget exhausted in {{ $value | humanizeDuration }}."
          runbook_url: "https://wiki.company.com/runbooks/payment-service/error-budget-burn"

      # Slow-Burn Alert: Create ticket (no page) when budget burns slowly
      - alert: PaymentServiceSlowErrorBudgetBurn
        expr: |
          (
            rate(http_requests_total{service="payment",code=~"5.."}[6h])
            /
            rate(http_requests_total{service="payment"}[6h])
          ) > (2 * 0.001)  # 2x burn rate over 6 hours
        for: 60m
        labels:
          severity: warning
          team: payment
        annotations:
          summary: "Payment Service slow error budget burn"
          description: "Error budget burning 2x expected rate. Review and address within 24 hours."
          runbook_url: "https://wiki.company.com/runbooks/payment-service/slow-error-burn"

      # Latency SLO Alert
      - alert: PaymentServiceHighLatency
        expr: |
          histogram_quantile(0.99,
            rate(http_request_duration_seconds_bucket{service="payment"}[5m])
          ) > 0.5  # p99 > 500ms
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Payment Service p99 latency above SLO"
          description: "p99 latency is {{ $value | humanizeDuration }}, exceeding 500ms SLO threshold."
          runbook_url: "https://wiki.company.com/runbooks/payment-service/high-latency"

# Symptom-based alert: HTTP 5xx Error Rate
resource "aws_cloudwatch_metric_alarm" "payment_error_rate" {
  alarm_name          = "payment-service-5xx-error-rate"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 2
  datapoints_to_alarm = 2
  threshold           = 10
  treat_missing_data  = "notBreaching"

  metric_query {
    id          = "error_rate"
    expression  = "errors / total * 100"
    label       = "Error Rate %"
    return_data = true
  }

  metric_query {
    id = "errors"
    metric {
      metric_name = "HTTPCode_Target_5XX_Count"
      namespace   = "AWS/ApplicationELB"
      period      = 60
      stat        = "Sum"
      dimensions = {
        LoadBalancer = aws_lb.main.arn_suffix
        TargetGroup  = aws_lb_target_group.app.arn_suffix
      }
    }
  }

  metric_query {
    id = "total"
    metric {
      metric_name = "RequestCount"
      namespace   = "AWS/ApplicationELB"
      period      = 60
      stat        = "Sum"
      dimensions = {
        LoadBalancer = aws_lb.main.arn_suffix
        TargetGroup  = aws_lb_target_group.app.arn_suffix
      }
    }
  }

  alarm_description = <<-EOT
    Payment Service 5xx Error Rate > 10%

    Symptom: Users are receiving HTTP 5xx errors from the Payment Service.
    Runbook: https://wiki.company.com/runbooks/payment-service/5xx-errors
    Dashboard: https://monitoring.company.com/d/payment-service
    Escalation: payment-oncall@company.com → platform-team@company.com
  EOT

  alarm_actions = [aws_sns_topic.payment_oncall.arn]
  ok_actions    = [aws_sns_topic.payment_oncall.arn]
}

# Latency Alert (Symptom: Slow response times for users)
resource "aws_cloudwatch_metric_alarm" "payment_latency_p99" {
  alarm_name          = "payment-service-p99-latency"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 3
  datapoints_to_alarm = 2
  metric_name         = "TargetResponseTime"
  namespace           = "AWS/ApplicationELB"
  period              = 60
  extended_statistic  = "p99"
  threshold           = 0.5  # 500ms

  dimensions = {
    LoadBalancer = aws_lb.main.arn_suffix
    TargetGroup  = aws_lb_target_group.app.arn_suffix
  }

  alarm_description = <<-EOT
    Payment Service p99 Latency > 500ms

    Symptom: 1% of users experience response times > 500ms (SLO violation).
    Runbook: https://wiki.company.com/runbooks/payment-service/high-latency
  EOT

  alarm_actions = [aws_sns_topic.payment_oncall.arn]
}

#!/bin/bash
# alert-audit.sh – Check existing alerts for quality

# List all CloudWatch Alarms
aws cloudwatch describe-alarms --state-value OK,ALARM,INSUFFICIENT_DATA \
  --query 'MetricAlarms[].{
    Name: AlarmName,
    Metric: MetricName,
    Description: AlarmDescription,
    Actions: AlarmActions
  }' \
  --output table

# For each alarm, check:
# 1. Is the metric symptom-based? (not CPU, Memory)
# 2. Does the description contain a runbook URL?
# 3. Are there AlarmActions (SNS Topic)?
# 4. Was the alarm triggered in the last 90 days? Was it actionable?