Best Practice: Circuit Breaker, Timeouts & Bulkheads

import pybreaker
import httpx
import asyncio
import logging
from datetime import datetime

# Circuit Breaker konfigurieren
payment_gateway_cb = pybreaker.CircuitBreaker(
    fail_max=5,          # Nach 5 Fehlern: OPEN
    reset_timeout=30,    # Nach 30s: HALF-OPEN (ein Test-Request)
    name="payment-gateway"
)

# Event-Listener für Logging
@payment_gateway_cb.on_state_change
def log_state_change(cb, old_state, new_state):
    logging.warning(f"CircuitBreaker[{cb.name}]: {old_state} -> {new_state}")

async def charge_card(card_token: str, amount: float) -> dict:
    """Zahlung mit Circuit Breaker und Timeout."""
    try:
        # Circuit Breaker wrapping + Timeout
        async with httpx.AsyncClient(timeout=httpx.Timeout(3.0)) as client:
            response = await payment_gateway_cb.call_async(
                client.post,
                "https://payment-gateway.example.com/charge",
                json={"card_token": card_token, "amount": amount}
            )
            return response.json()

    except pybreaker.CircuitBreakerError:
        # Circuit ist OPEN: Sofortige Ablehnung, keine Wartezeit
        logging.warning("Payment gateway circuit open – fast failing")
        raise ServiceUnavailableError("Payment gateway temporarily unavailable")

    except httpx.TimeoutException:
        # Timeout überschritten
        raise ServiceTimeoutError("Payment gateway timeout after 3s")

# Retry mit Exponential Backoff + Jitter
async def charge_with_retry(card_token: str, amount: float) -> dict:
    max_attempts = 3
    for attempt in range(max_attempts):
        try:
            return await charge_card(card_token, amount)
        except ServiceTimeoutError:
            if attempt == max_attempts - 1:
                raise
            # Exponential Backoff + Jitter
            wait = (2 ** attempt) + (asyncio.get_event_loop().time() % 1)
            await asyncio.sleep(wait)
    raise ServiceUnavailableError("Max retry attempts exceeded")

// application.yml
resilience4j:
  circuitbreaker:
    instances:
      payment-gateway:
        registerHealthIndicator: true
        slidingWindowSize: 10
        minimumNumberOfCalls: 5
        permittedNumberOfCallsInHalfOpenState: 2
        automaticTransitionFromOpenToHalfOpenEnabled: true
        waitDurationInOpenState: 30s
        failureRateThreshold: 50       # 50% Fehlerrate → OPEN
        slowCallDurationThreshold: 2s
        slowCallRateThreshold: 80
  retry:
    instances:
      payment-gateway:
        maxAttempts: 3
        waitDuration: 100ms
        enableExponentialBackoff: true
        exponentialBackoffMultiplier: 2
        randomizedWaitFactor: 0.5      # Jitter ±50%
  bulkhead:
    instances:
      payment-gateway:
        maxConcurrentCalls: 10         # Max. gleichzeitige Calls
        maxWaitDuration: 100ms

// PaymentService.java
@Service
public class PaymentService {

    @CircuitBreaker(name = "payment-gateway", fallbackMethod = "paymentFallback")
    @Retry(name = "payment-gateway")
    @Bulkhead(name = "payment-gateway")
    public ChargeResult chargeCard(String cardToken, BigDecimal amount) {
        return paymentGatewayClient.charge(cardToken, amount);
    }

    public ChargeResult paymentFallback(String cardToken, BigDecimal amount,
                                         CallNotPermittedException ex) {
        // Circuit offen: Offline-Queue für spätere Verarbeitung
        offlineQueue.enqueue(cardToken, amount);
        return ChargeResult.queued("Payment queued for processing");
    }
}

apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: payment-gateway
  namespace: payment
spec:
  host: payment-gateway.payment.svc.cluster.local
  trafficPolicy:
    connectionPool:
      tcp:
        maxConnections: 100
        connectTimeout: 3s
      http:
        http1MaxPendingRequests: 50
        http2MaxRequests: 100
        maxRequestsPerConnection: 1000
        maxRetries: 3
        retryOn: "5xx,gateway-error,connect-failure,retriable-4xx"
        retryRemoteStatuses: "500,502,503"

    outlierDetection:
      consecutive5xxErrors: 5         # 5 Fehler → Ausschluss
      interval: 10s                   # Bewertungsfenster
      baseEjectionTime: 30s           # Minimum Ausschlusszeit
      maxEjectionPercent: 50          # Max. 50% der Hosts ausschließen
      splitExternalLocalOriginErrors: true

resource "aws_lb" "api" {
  name               = "payment-api-alb"
  internal           = false
  load_balancer_type = "application"
  security_groups    = [aws_security_group.alb.id]
  subnets            = var.public_subnet_ids
  idle_timeout       = 30    # Für REST APIs: 30s; Nicht Default 60s

  tags = var.mandatory_tags
}

resource "aws_lb_target_group" "api" {
  name                 = "payment-api-tg"
  port                 = 8080
  protocol             = "HTTP"
  vpc_id               = var.vpc_id
  deregistration_delay = 30  # Graceful shutdown Fenster

  health_check {
    enabled             = true
    path                = "/health/ready"
    interval            = 15
    timeout             = 5
    healthy_threshold   = 2
    unhealthy_threshold = 3
    matcher             = "200"
  }
}