Split View: 서킷 브레이커와 복원력 패턴 실전 가이드 — Resilience4j, Istio, 장애 격리 전략

              Failure rate >= threshold (failureRateThreshold)
    +---------------------------------------------+
    |                                             |
    v                                             |
+----------+                                 +----------+
|          |     Trial call success rate >= threshold     |          |
|   OPEN   | <- - - - - - - - - - - - - - -- |  CLOSED  |
| (blocked)|                                 | (normal) |
+----------+                                 +----------+
     |                                            ^
     | waitDurationInOpenState elapsed             |
     v                                            |
+--------------+    Trial call success             |
|  HALF-OPEN   | ---------------------------------+
| (trial mode) |
+--------------+
     |
     | Trial call failure
     v
+----------+
|   OPEN   |  (blocked again)
+----------+

// build.gradle (Spring Boot 3.x)
dependencies {
    implementation 'io.github.resilience4j:resilience4j-spring-boot3:2.2.0'
    implementation 'io.github.resilience4j:resilience4j-micrometer:2.2.0'
    implementation 'org.springframework.boot:spring-boot-starter-actuator'
    implementation 'org.springframework.boot:spring-boot-starter-aop'
}

resilience4j:
  circuitbreaker:
    instances:
      paymentService:
        registerHealthIndicator: true
        slidingWindowType: COUNT_BASED
        slidingWindowSize: 10
        minimumNumberOfCalls: 5
        failureRateThreshold: 50
        waitDurationInOpenState: 10s
        permittedNumberOfCallsInHalfOpenState: 3
        slowCallDurationThreshold: 2s
        slowCallRateThreshold: 80
        recordExceptions:
          - java.io.IOException
          - java.util.concurrent.TimeoutException
          - org.springframework.web.client.HttpServerErrorException
        ignoreExceptions:
          - com.example.BusinessException

  retry:
    instances:
      paymentService:
        maxAttempts: 3
        waitDuration: 500ms
        enableExponentialBackoff: true
        exponentialBackoffMultiplier: 2
        retryExceptions:
          - java.io.IOException
          - java.util.concurrent.TimeoutException

  bulkhead:
    instances:
      paymentService:
        maxConcurrentCalls: 20
        maxWaitDuration: 500ms

  timelimiter:
    instances:
      paymentService:
        timeoutDuration: 3s
        cancelRunningFuture: true

  ratelimiter:
    instances:
      paymentService:
        limitRefreshPeriod: 1s
        limitForPeriod: 50
        timeoutDuration: 0s

@Service
@Slf4j
public class PaymentService {

    private final PaymentGatewayClient paymentGatewayClient;
    private final PaymentCacheService paymentCacheService;

    public PaymentService(PaymentGatewayClient paymentGatewayClient,
                          PaymentCacheService paymentCacheService) {
        this.paymentGatewayClient = paymentGatewayClient;
        this.paymentCacheService = paymentCacheService;
    }

    @CircuitBreaker(name = "paymentService", fallbackMethod = "paymentFallback")
    @Bulkhead(name = "paymentService")
    @Retry(name = "paymentService")
    @TimeLimiter(name = "paymentService")
    public CompletableFuture<PaymentResponse> processPayment(PaymentRequest request) {
        return CompletableFuture.supplyAsync(() -> {
            log.info("Processing payment: orderId={}", request.getOrderId());
            return paymentGatewayClient.charge(request);
        });
    }

    // Fallback method: called when circuit is OPEN or exception occurs
    private CompletableFuture<PaymentResponse> paymentFallback(
            PaymentRequest request, Throwable throwable) {
        log.warn("Payment service fallback triggered: orderId={}, reason={}",
                request.getOrderId(), throwable.getMessage());

        if (throwable instanceof CallNotPermittedException) {
            // Circuit is open - save to queue for async processing
            return CompletableFuture.completedFuture(
                PaymentResponse.queued(request.getOrderId(),
                    "Payment service temporarily unavailable. Order has been queued.")
            );
        }

        // Other exceptions - try returning cached result
        return CompletableFuture.completedFuture(
            paymentCacheService.getCachedResponse(request.getOrderId())
                .orElse(PaymentResponse.error(request.getOrderId(),
                    "An error occurred during payment processing. Please try again later."))
        );
    }
}

Retry ( CircuitBreaker ( RateLimiter ( TimeLimiter ( Bulkhead ( Function ) ) ) ) )

apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: payment-service-circuit-breaker
  namespace: production
spec:
  host: payment-service.production.svc.cluster.local
  trafficPolicy:
    connectionPool:
      tcp:
        maxConnections: 100 # Maximum TCP connections
        connectTimeout: 3s # TCP connection timeout
      http:
        h2UpgradePolicy: DEFAULT
        http1MaxPendingRequests: 50 # Max pending HTTP requests
        http2MaxRequests: 100 # Max active HTTP/2 requests
        maxRequestsPerConnection: 10 # Max requests per connection
        maxRetries: 3 # Max retries
    outlierDetection:
      consecutive5xxErrors: 5 # Remove after 5 consecutive 5xx errors
      interval: 10s # Analysis interval
      baseEjectionTime: 30s # Minimum ejection time
      maxEjectionPercent: 50 # Max ejection percentage (50%)
      minHealthPercent: 30 # Minimum healthy instance percentage

# ThreadPool Bulkhead configuration
resilience4j:
  thread-pool-bulkhead:
    instances:
      inventoryService:
        maxThreadPoolSize: 10
        coreThreadPoolSize: 5
        queueCapacity: 20
        keepAliveDuration: 100ms
        writableStackTraceEnabled: true

@Service
public class OrderOrchestrator {

    @Bulkhead(name = "paymentService", type = Bulkhead.Type.SEMAPHORE)
    public PaymentResult processPayment(Order order) {
        return paymentClient.charge(order.getPaymentInfo());
    }

    @Bulkhead(name = "inventoryService", type = Bulkhead.Type.THREADPOOL)
    public CompletableFuture<InventoryResult> reserveInventory(Order order) {
        return CompletableFuture.supplyAsync(() ->
            inventoryClient.reserve(order.getItems()));
    }

    @Bulkhead(name = "notificationService", type = Bulkhead.Type.SEMAPHORE)
    public void sendNotification(Order order) {
        notificationClient.send(order.getUserId(), "Your order has been received.");
    }
}

@Configuration
public class ResilienceConfig {

    @Bean
    public Supplier<String> resilientSupplier(
            CircuitBreakerRegistry circuitBreakerRegistry,
            RetryRegistry retryRegistry,
            BulkheadRegistry bulkheadRegistry,
            RateLimiterRegistry rateLimiterRegistry) {

        CircuitBreaker circuitBreaker = circuitBreakerRegistry
                .circuitBreaker("externalApi");
        Retry retry = retryRegistry.retry("externalApi");
        Bulkhead bulkhead = bulkheadRegistry.bulkhead("externalApi");
        RateLimiter rateLimiter = rateLimiterRegistry
                .rateLimiter("externalApi");

        // Decorator chaining: applied from inside to outside
        Supplier<String> decoratedSupplier = Decorators
                .ofSupplier(() -> externalApiClient.call())
                .withBulkhead(bulkhead)           // 1. Concurrent call limit
                .withRateLimiter(rateLimiter)      // 2. Rate limit
                .withCircuitBreaker(circuitBreaker) // 3. Failure detection/blocking
                .withRetry(retry)                   // 4. Retry
                .withFallback(Arrays.asList(
                    CallNotPermittedException.class,
                    BulkheadFullException.class,
                    RequestNotPermitted.class),
                    throwable -> "Fallback Response")
                .decorate();

        return decoratedSupplier;
    }
}

@Service
@Slf4j
public class ProductRecommendationService {

    private final RecommendationEngine primaryEngine;
    private final RecommendationEngine secondaryEngine;
    private final RedisTemplate<String, List<Product>> cache;

    @CircuitBreaker(name = "recommendation",
                    fallbackMethod = "secondaryRecommendation")
    public List<Product> getRecommendations(String userId) {
        return primaryEngine.recommend(userId);
    }

    // 1st fallback: Use secondary recommendation engine
    private List<Product> secondaryRecommendation(
            String userId, Throwable t) {
        log.warn("Primary recommendation engine failure, switching to secondary: {}", t.getMessage());
        try {
            return secondaryEngine.recommend(userId);
        } catch (Exception e) {
            return cachedRecommendation(userId, e);
        }
    }

    // 2nd fallback: Return cached recommendation results
    private List<Product> cachedRecommendation(
            String userId, Throwable t) {
        log.warn("Secondary recommendation engine also failed, checking cache: {}", t.getMessage());
        List<Product> cached = cache.opsForValue()
                .get("recommendation:" + userId);
        if (cached != null && !cached.isEmpty()) {
            return cached;
        }
        return defaultRecommendation(userId, t);
    }

    // 3rd fallback: Return default popular products list
    private List<Product> defaultRecommendation(
            String userId, Throwable t) {
        log.warn("No cache available, returning default popular products");
        return List.of(
            Product.popular("BEST-001", "Bestseller Product A"),
            Product.popular("BEST-002", "Bestseller Product B"),
            Product.popular("BEST-003", "Bestseller Product C")
        );
    }
}

// Remove
// implementation 'org.springframework.cloud:spring-cloud-starter-netflix-hystrix'

// Add
implementation 'io.github.resilience4j:resilience4j-spring-boot3:2.2.0'
implementation 'io.github.resilience4j:resilience4j-micrometer:2.2.0'

resilience4j:
  retry:
    instances:
      paymentGateway:
        maxAttempts: 3
        waitDuration: 1s
        enableExponentialBackoff: true
        exponentialBackoffMultiplier: 2
        enableRandomizedWait: true # Enable jitter
        randomizedWaitFactor: 0.5 # Randomize within 50% range

#!/bin/bash
# circuit-breaker-recovery.sh
# Circuit breaker failure recovery procedure script

echo "===== Check Circuit Breaker Status ====="
# Check circuit breaker state via Actuator endpoint
curl -s http://localhost:8080/actuator/circuitbreakers | jq '.circuitBreakers'

echo ""
echo "===== Downstream Service Health Check ====="
curl -s http://payment-service:8080/actuator/health | jq '.status'
curl -s http://inventory-service:8080/actuator/health | jq '.status'

echo ""
echo "===== Force Close Circuit Breaker (after confirming downstream recovery) ====="
# WARNING: Execute only after downstream service is fully recovered
# curl -X POST http://localhost:8080/actuator/circuitbreakers/paymentService/close

echo ""
echo "===== Check Current Metrics ====="
curl -s http://localhost:8080/actuator/metrics/resilience4j.circuitbreaker.state | jq '.'
curl -s http://localhost:8080/actuator/metrics/resilience4j.circuitbreaker.failure.rate | jq '.'

echo ""
echo "===== Check Istio Outlier Detection Status ====="
kubectl get destinationrules -n production
kubectl describe destinationrule payment-service-circuit-breaker -n production

# Prometheus scrape configuration
scrape_configs:
  - job_name: 'spring-boot-resilience4j'
    metrics_path: '/actuator/prometheus'
    scrape_interval: 5s
    static_configs:
      - targets: ['payment-service:8080']
        labels:
          application: 'payment-service'

# Grafana Alert Rule (provisioning)
groups:
  - name: circuit-breaker-alerts
    rules:
      - alert: CircuitBreakerOpen
        expr: resilience4j_circuitbreaker_state{state="open"} == 1
        for: 10s
        labels:
          severity: critical
        annotations:
          summary: 'Circuit Breaker OPEN - {{ $labels.name }}'
          description: >
            {{ $labels.application }}'s {{ $labels.name }}
            circuit breaker is in OPEN state.
            Check downstream service status immediately.

      - alert: HighFailureRate
        expr: resilience4j_circuitbreaker_failure_rate > 40
        for: 30s
        labels:
          severity: warning
        annotations:
          summary: 'High failure rate detected - {{ $labels.name }}'
          description: >
            {{ $labels.name }} failure rate is {{ $value }}%.
            Identify the cause before the circuit opens.

# Check service status within Istio mesh
istioctl proxy-config cluster <pod-name> -n production | grep outlier

# Check Envoy statistics
kubectl exec -it <pod-name> -n production -c istio-proxy -- \
  curl localhost:15000/stats | grep outlier_detection

# Access Kiali dashboard
istioctl dashboard kiali