Circuit Breaker 패턴과 Resilience4j 실전 구현 가이드: 장애 전파 차단부터 복구까지

                     실패율 >= 임계값
         ┌─────────────────────────────────────┐
         │                                     │
         ▼                                     │
    ┌──────────┐                          ┌──────────┐
    │          │    waitDuration 경과      │          │
    │   OPEN   │ ─────────────────────>   │  CLOSED  │
    │  (차단)   │                          │  (정상)   │
    └──────────┘                          └──────────┘
         │                                     ▲
         │ waitDuration 경과                    │
         ▼                                     │ 시험 호출 성공률 >= 임계값
    ┌──────────────┐                           │
    │  HALF-OPEN   │ ──────────────────────────┘
    │  (시험 허용)  │
    └──────────────┘
         │
         │ 시험 호출 실패율 >= 임계값
         │
         ▼
    ┌──────────┐
    │   OPEN   │  (다시 차단)
    └──────────┘

외부(먼저 평가) ──────────────────────────────────> 내부(마지막 평가)
Retry -> CircuitBreaker -> RateLimiter -> TimeLimiter -> Bulkhead

// build.gradle.kts (Spring Boot 3.3+ / Resilience4j 2.2+)
plugins {
    id("org.springframework.boot") version "3.3.5"
    id("io.spring.dependency-management") version "1.1.6"
    kotlin("jvm") version "1.9.25"
    kotlin("plugin.spring") version "1.9.25"
}

dependencies {
    // Resilience4j Spring Boot 3 스타터
    implementation("io.github.resilience4j:resilience4j-spring-boot3:2.2.0")

    // 개별 모듈 (스타터에 포함되지만 명시적 선언 권장)
    implementation("io.github.resilience4j:resilience4j-circuitbreaker")
    implementation("io.github.resilience4j:resilience4j-retry")
    implementation("io.github.resilience4j:resilience4j-bulkhead")
    implementation("io.github.resilience4j:resilience4j-ratelimiter")
    implementation("io.github.resilience4j:resilience4j-timelimiter")

    // Micrometer + Prometheus (모니터링)
    implementation("io.github.resilience4j:resilience4j-micrometer")
    implementation("io.micrometer:micrometer-registry-prometheus")

    // Spring Boot Actuator
    implementation("org.springframework.boot:spring-boot-starter-actuator")
    implementation("org.springframework.boot:spring-boot-starter-aop")
    implementation("org.springframework.boot:spring-boot-starter-web")

    // Kotlin Coroutines (선택)
    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core")
    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-reactor")

    testImplementation("org.springframework.boot:spring-boot-starter-test")
}

# application.yml - Resilience4j 통합 설정
resilience4j:
  circuitbreaker:
    configs:
      default:
        registerHealthIndicator: true
        slidingWindowType: COUNT_BASED
        slidingWindowSize: 10
        minimumNumberOfCalls: 5
        failureRateThreshold: 50
        slowCallRateThreshold: 80
        slowCallDurationThreshold: 3s
        waitDurationInOpenState: 30s
        permittedNumberOfCallsInHalfOpenState: 3
        automaticTransitionFromOpenToHalfOpenEnabled: true
        recordExceptions:
          - java.io.IOException
          - java.util.concurrent.TimeoutException
          - org.springframework.web.client.HttpServerErrorException
        ignoreExceptions:
          - com.example.order.exception.BusinessValidationException
    instances:
      paymentGateway:
        baseConfig: default
        failureRateThreshold: 40
        waitDurationInOpenState: 60s
        slidingWindowSize: 20
      inventoryService:
        baseConfig: default
        failureRateThreshold: 60
        slowCallDurationThreshold: 5s
      notificationService:
        baseConfig: default
        failureRateThreshold: 70
        waitDurationInOpenState: 15s

  retry:
    configs:
      default:
        maxAttempts: 3
        waitDuration: 1s
        enableExponentialBackoff: true
        exponentialBackoffMultiplier: 2.0
        exponentialMaxWaitDuration: 10s
        retryExceptions:
          - java.io.IOException
          - java.util.concurrent.TimeoutException
        ignoreExceptions:
          - com.example.order.exception.BusinessValidationException
    instances:
      paymentGateway:
        baseConfig: default
        maxAttempts: 2
        waitDuration: 2s
      inventoryService:
        baseConfig: default
        maxAttempts: 4
      notificationService:
        baseConfig: default
        maxAttempts: 5
        waitDuration: 500ms

  bulkhead:
    configs:
      default:
        maxConcurrentCalls: 25
        maxWaitDuration: 500ms
    instances:
      paymentGateway:
        baseConfig: default
        maxConcurrentCalls: 15
      inventoryService:
        baseConfig: default
        maxConcurrentCalls: 30
      notificationService:
        baseConfig: default
        maxConcurrentCalls: 50

  ratelimiter:
    configs:
      default:
        limitForPeriod: 100
        limitRefreshPeriod: 1s
        timeoutDuration: 500ms
    instances:
      paymentGateway:
        baseConfig: default
        limitForPeriod: 50
      inventoryService:
        baseConfig: default
        limitForPeriod: 200

  timelimiter:
    configs:
      default:
        timeoutDuration: 5s
        cancelRunningFuture: true
    instances:
      paymentGateway:
        baseConfig: default
        timeoutDuration: 10s
      inventoryService:
        baseConfig: default
        timeoutDuration: 3s

# Actuator 메트릭 노출
management:
  endpoints:
    web:
      exposure:
        include: health,metrics,prometheus,circuitbreakers,retries
  endpoint:
    health:
      show-details: always
  health:
    circuitbreakers:
      enabled: true
  metrics:
    distribution:
      percentiles-histogram:
        resilience4j.circuitbreaker.calls: true
        resilience4j.retry.calls: true
    tags:
      application: order-service

// PaymentGatewayClient.kt
@Service
class PaymentGatewayClient(
    private val restClient: RestClient,
    private val paymentRetryQueue: PaymentRetryQueue,
    private val paymentCacheStore: PaymentCacheStore,
) {
    companion object {
        private val log = LoggerFactory.getLogger(PaymentGatewayClient::class.java)
        const val CB_NAME = "paymentGateway"
    }

    @CircuitBreaker(name = CB_NAME, fallbackMethod = "paymentFallback")
    @Retry(name = CB_NAME)
    @Bulkhead(name = CB_NAME)
    fun processPayment(request: PaymentRequest): PaymentResponse {
        log.info("Calling payment gateway for orderId={}", request.orderId)

        val response = restClient.post()
            .uri("https://payment-api.internal/v2/charges")
            .contentType(MediaType.APPLICATION_JSON)
            .body(request)
            .retrieve()
            .body(PaymentResponse::class.java)
            ?: throw PaymentGatewayException("Empty response from payment gateway")

        log.info("Payment processed: orderId={}, txId={}", request.orderId, response.transactionId)
        return response
    }

    /**
     * 폴백 메서드: CircuitBreaker OPEN 또는 예외 발생 시 호출된다.
     * 메서드 시그니처는 원본과 동일 + 마지막 파라미터로 Exception을 받아야 한다.
     */
    private fun paymentFallback(request: PaymentRequest, ex: Exception): PaymentResponse {
        log.warn(
            "Payment fallback activated: orderId={}, reason={}",
            request.orderId, ex.message
        )

        return when (ex) {
            is CallNotPermittedException -> {
                // CircuitBreaker OPEN 상태: 큐에 넣고 비동기 처리
                paymentRetryQueue.enqueue(request)
                PaymentResponse(
                    orderId = request.orderId,
                    status = PaymentStatus.QUEUED,
                    message = "결제가 대기열에 등록되었습니다. 잠시 후 처리됩니다.",
                    transactionId = null,
                )
            }
            is BulkheadFullException -> {
                // Bulkhead 포화: 즉시 재시도 유도
                PaymentResponse(
                    orderId = request.orderId,
                    status = PaymentStatus.RETRY_LATER,
                    message = "현재 결제 요청이 많습니다. 잠시 후 다시 시도해주세요.",
                    transactionId = null,
                )
            }
            else -> {
                // 기타 예외: 캐시된 결제 정보가 있으면 반환
                val cached = paymentCacheStore.getLastSuccess(request.orderId)
                if (cached != null) {
                    log.info("Returning cached payment for orderId={}", request.orderId)
                    cached.copy(status = PaymentStatus.CACHED)
                } else {
                    paymentRetryQueue.enqueue(request)
                    PaymentResponse(
                        orderId = request.orderId,
                        status = PaymentStatus.PENDING,
                        message = "결제 처리 중 오류가 발생했습니다. 자동 재시도됩니다.",
                        transactionId = null,
                    )
                }
            }
        }
    }
}

// InventoryServiceClient.java
@Service
@Slf4j
public class InventoryServiceClient {

    private final CircuitBreaker circuitBreaker;
    private final Retry retry;
    private final Bulkhead bulkhead;
    private final RestClient restClient;

    public InventoryServiceClient(
            CircuitBreakerRegistry cbRegistry,
            RetryRegistry retryRegistry,
            BulkheadRegistry bulkheadRegistry,
            RestClient.Builder restClientBuilder) {

        this.circuitBreaker = cbRegistry.circuitBreaker("inventoryService");
        this.retry = retryRegistry.retry("inventoryService");
        this.bulkhead = bulkheadRegistry.bulkhead("inventoryService");
        this.restClient = restClientBuilder
                .baseUrl("https://inventory-api.internal")
                .build();

        // 이벤트 리스너 등록
        registerEventListeners();
    }

    public InventoryResponse checkStock(String productId, int quantity) {
        // 데코레이터 체인: Bulkhead -> CircuitBreaker -> Retry -> 실제 호출
        Supplier<InventoryResponse> decorated = Decorators
                .ofSupplier(() -> doCheckStock(productId, quantity))
                .withBulkhead(bulkhead)
                .withCircuitBreaker(circuitBreaker)
                .withRetry(retry)
                .withFallback(
                    List.of(
                        CallNotPermittedException.class,
                        BulkheadFullException.class,
                        IOException.class
                    ),
                    ex -> stockFallback(productId, quantity, ex)
                )
                .decorate();

        return decorated.get();
    }

    private InventoryResponse doCheckStock(String productId, int quantity) {
        return restClient.get()
                .uri("/v1/stock/{productId}?qty={qty}", productId, quantity)
                .retrieve()
                .body(InventoryResponse.class);
    }

    private InventoryResponse stockFallback(
            String productId, int quantity, Throwable ex) {
        log.warn("Inventory fallback: productId={}, reason={}", productId, ex.getMessage());
        // 재고가 불확실할 때는 주문을 수락하되 비동기 검증 예약
        return InventoryResponse.builder()
                .productId(productId)
                .available(true)
                .reservationStatus(ReservationStatus.TENTATIVE)
                .message("재고 확인 지연: 잠정 승인 후 비동기 검증 예정")
                .build();
    }

    private void registerEventListeners() {
        circuitBreaker.getEventPublisher()
            .onStateTransition(event -> {
                log.warn("[CircuitBreaker] {} state: {} -> {}",
                    event.getCircuitBreakerName(),
                    event.getStateTransition().getFromState(),
                    event.getStateTransition().getToState());
            })
            .onError(event ->
                log.error("[CircuitBreaker] {} error: {} ({}ms)",
                    event.getCircuitBreakerName(),
                    event.getThrowable().getMessage(),
                    event.getElapsedDuration().toMillis())
            )
            .onSuccess(event ->
                log.debug("[CircuitBreaker] {} success ({}ms)",
                    event.getCircuitBreakerName(),
                    event.getElapsedDuration().toMillis())
            )
            .onCallNotPermitted(event ->
                log.warn("[CircuitBreaker] {} call not permitted (OPEN state)",
                    event.getCircuitBreakerName())
            );

        retry.getEventPublisher()
            .onRetry(event ->
                log.info("[Retry] {} attempt #{} (wait: {}ms)",
                    event.getName(),
                    event.getNumberOfRetryAttempts(),
                    event.getWaitInterval().toMillis())
            );
    }
}

// RetryConfig를 프로그래밍 방식으로 커스터마이징
@Configuration
class ResilienceConfig {

    @Bean
    fun customRetryConfig(): RetryConfig {
        return RetryConfig.custom<RetryConfig>()
            .maxAttempts(4)
            .intervalFunction(
                // 지수 백오프 + 지터: 1s, 2s(+jitter), 4s(+jitter), 8s(+jitter)
                IntervalFunction.ofExponentialRandomBackoff(
                    Duration.ofSeconds(1),   // 초기 대기 시간
                    2.0,                     // 배수
                    Duration.ofSeconds(15)   // 최대 대기 시간
                )
            )
            .retryOnException { ex ->
                // 재시도 대상 예외 판별
                when (ex) {
                    is IOException -> true
                    is TimeoutException -> true
                    is HttpServerErrorException -> true
                    is ConnectException -> true
                    else -> false
                }
            }
            .ignoreExceptions(
                BusinessValidationException::class.java,
                IllegalArgumentException::class.java
            )
            .failAfterMaxAttempts(true) // 최대 재시도 후 MaxRetriesExceededException 발생
            .build()
    }

    @Bean
    fun retryRegistry(customRetryConfig: RetryConfig): RetryRegistry {
        return RetryRegistry.of(customRetryConfig)
    }
}

# ThreadPoolBulkhead 설정 예시
resilience4j:
  thread-pool-bulkhead:
    instances:
      heavyProcessing:
        maxThreadPoolSize: 10
        coreThreadPoolSize: 5
        queueCapacity: 20
        keepAliveDuration: 100ms
        writableStackTraceEnabled: true

// RateLimiter와 CircuitBreaker 조합
@Service
@Slf4j
public class ExternalApiClient {

    private final RestClient restClient;

    @CircuitBreaker(name = "externalApi", fallbackMethod = "apiFallback")
    @RateLimiter(name = "externalApi")
    @Retry(name = "externalApi")
    public ApiResponse callExternalApi(ApiRequest request) {
        log.debug("Calling external API: endpoint={}", request.getEndpoint());

        return restClient.post()
                .uri(request.getEndpoint())
                .body(request.getPayload())
                .retrieve()
                .body(ApiResponse.class);
    }

    private ApiResponse apiFallback(ApiRequest request, RequestNotPermitted ex) {
        // RateLimiter에 의해 거부된 경우
        log.warn("Rate limit exceeded for external API: {}", request.getEndpoint());
        return ApiResponse.rateLimited(
                "요청 한도를 초과했습니다. " +
                "limitForPeriod 설정을 확인하거나 잠시 후 다시 시도하세요."
        );
    }

    private ApiResponse apiFallback(ApiRequest request, Exception ex) {
        // 기타 예외 (CircuitBreaker OPEN, 네트워크 오류 등)
        log.warn("External API fallback: endpoint={}, reason={}",
                request.getEndpoint(), ex.getMessage());
        return ApiResponse.error("외부 API 호출에 실패했습니다: " + ex.getMessage());
    }
}

# CircuitBreaker 상태 확인 (0=CLOSED, 1=OPEN, 2=HALF_OPEN, 3=DISABLED, 4=FORCED_OPEN)
resilience4j_circuitbreaker_state{name="paymentGateway"}

# 실패율 (%)
resilience4j_circuitbreaker_failure_rate{name="paymentGateway"}

# 느린 호출 비율 (%)
resilience4j_circuitbreaker_slow_call_rate{name="paymentGateway"}

# 호출 통계 (kind: successful, failed, ignored, not_permitted)
rate(resilience4j_circuitbreaker_calls_seconds_count{name="paymentGateway"}[5m])

# 호출 지연 시간 분포 (히스토그램)
histogram_quantile(0.95,
  rate(resilience4j_circuitbreaker_calls_seconds_bucket{name="paymentGateway"}[5m])
)

# Retry 재시도 횟수
increase(resilience4j_retry_calls_total{name="paymentGateway", kind="successful_with_retry"}[1h])
increase(resilience4j_retry_calls_total{name="paymentGateway", kind="failed_with_retry"}[1h])

# Bulkhead 가용 동시 호출 수
resilience4j_bulkhead_available_concurrent_calls{name="paymentGateway"}

# RateLimiter 가용 허용 수
resilience4j_ratelimiter_available_permissions{name="externalApi"}

resilience4j_circuitbreaker_state{application="order-service"}

resilience4j_circuitbreaker_failure_rate{application="order-service", name=~".*"}

sum by (name, kind) (
  rate(resilience4j_circuitbreaker_calls_seconds_count{application="order-service"}[5m])
)

histogram_quantile(0.95,
  sum by (le, name) (
    rate(resilience4j_circuitbreaker_calls_seconds_bucket{application="order-service"}[5m])
  )
)

resilience4j_bulkhead_max_allowed_concurrent_calls{application="order-service"}
- resilience4j_bulkhead_available_concurrent_calls{application="order-service"}

# prometheus-alerts.yml
groups:
  - name: resilience4j_alerts
    rules:
      - alert: CircuitBreakerOpen
        expr: resilience4j_circuitbreaker_state == 1
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: 'CircuitBreaker OPEN: {{ $labels.name }}'
          description: >
            서비스 {{ $labels.application }}의
            {{ $labels.name }} 서킷 브레이커가 OPEN 상태입니다.
            다운스트림 서비스 장애를 확인하세요.

      - alert: HighFailureRate
        expr: resilience4j_circuitbreaker_failure_rate > 30
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: 'High failure rate: {{ $labels.name }} ({{ $value }}%)'
          description: >
            {{ $labels.name }}의 실패율이 {{ $value }}%로
            경고 임계값(30%)을 초과했습니다.

      - alert: BulkheadSaturation
        expr: >
          (resilience4j_bulkhead_max_allowed_concurrent_calls
          - resilience4j_bulkhead_available_concurrent_calls)
          / resilience4j_bulkhead_max_allowed_concurrent_calls > 0.8
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: 'Bulkhead 80% saturated: {{ $labels.name }}'

      - alert: ExcessiveRetries
        expr: >
          rate(resilience4j_retry_calls_total{kind="failed_with_retry"}[5m])
          / rate(resilience4j_retry_calls_total[5m]) > 0.5
        for: 3m
        labels:
          severity: warning
        annotations:
          summary: 'Retry 실패율 50% 초과: {{ $labels.name }}'

// 원본 메서드
@CircuitBreaker(name = "svc", fallbackMethod = "fallback")
public OrderResponse getOrder(String orderId, boolean includeDetails) { ... }

// 올바른 폴백 (파라미터 동일 + Exception 추가)
private OrderResponse fallback(String orderId, boolean includeDetails, Exception ex) { ... }

// 잘못된 폴백 - 컴파일은 되지만 런타임에 매칭 실패
private OrderResponse fallback(String orderId, Exception ex) { ... }  // 파라미터 누락
private void fallback(String orderId, boolean includeDetails, Exception ex) { ... }  // 반환타입 불일치

// Actuator 엔드포인트로 상태 강제 전환
// POST /actuator/circuitbreakers/{name}/force-close
circuitBreakerRegistry.circuitBreaker("inventoryService")
    .transitionToForcedOpenState();  // 또는 transitionToClosedState()

// DynamicCircuitBreakerFactory.kt
@Component
class DynamicCircuitBreakerFactory(
    private val circuitBreakerRegistry: CircuitBreakerRegistry,
    private val meterRegistry: MeterRegistry,
) {
    private val log = LoggerFactory.getLogger(javaClass)

    /**
     * 서비스 이름 기반으로 CircuitBreaker를 동적 생성한다.
     * 이미 존재하면 기존 인스턴스를 반환한다.
     */
    fun getOrCreate(
        serviceName: String,
        tier: ServiceTier = ServiceTier.STANDARD,
    ): CircuitBreaker {
        return circuitBreakerRegistry.circuitBreaker(serviceName) {
            buildConfigForTier(tier)
        }.also { cb ->
            registerMetrics(cb)
            log.info(
                "CircuitBreaker created/retrieved: name={}, tier={}, state={}",
                serviceName, tier, cb.state
            )
        }
    }

    private fun buildConfigForTier(tier: ServiceTier): CircuitBreakerConfig {
        return when (tier) {
            ServiceTier.CRITICAL -> CircuitBreakerConfig.custom()
                .failureRateThreshold(30f)
                .slowCallRateThreshold(60f)
                .slowCallDurationThreshold(Duration.ofSeconds(2))
                .waitDurationInOpenState(Duration.ofSeconds(60))
                .slidingWindowSize(20)
                .minimumNumberOfCalls(10)
                .permittedNumberOfCallsInHalfOpenState(5)
                .automaticTransitionFromOpenToHalfOpenEnabled(true)
                .build()

            ServiceTier.STANDARD -> CircuitBreakerConfig.custom()
                .failureRateThreshold(50f)
                .slowCallRateThreshold(80f)
                .slowCallDurationThreshold(Duration.ofSeconds(3))
                .waitDurationInOpenState(Duration.ofSeconds(30))
                .slidingWindowSize(10)
                .minimumNumberOfCalls(5)
                .permittedNumberOfCallsInHalfOpenState(3)
                .automaticTransitionFromOpenToHalfOpenEnabled(true)
                .build()

            ServiceTier.BEST_EFFORT -> CircuitBreakerConfig.custom()
                .failureRateThreshold(70f)
                .slowCallRateThreshold(90f)
                .slowCallDurationThreshold(Duration.ofSeconds(5))
                .waitDurationInOpenState(Duration.ofSeconds(15))
                .slidingWindowSize(5)
                .minimumNumberOfCalls(3)
                .permittedNumberOfCallsInHalfOpenState(2)
                .automaticTransitionFromOpenToHalfOpenEnabled(true)
                .build()
        }
    }

    private fun registerMetrics(cb: CircuitBreaker) {
        TaggedCircuitBreakerMetrics.ofCircuitBreakerRegistry(
            circuitBreakerRegistry
        ).bindTo(meterRegistry)
    }

    enum class ServiceTier {
        CRITICAL,    // 결제, 인증 등 핵심 서비스
        STANDARD,    // 재고, 배송 등 일반 서비스
        BEST_EFFORT  // 알림, 추천 등 비핵심 서비스
    }
}

// CircuitBreakerIntegrationTest.java
@SpringBootTest
@AutoConfigureMockMvc
class CircuitBreakerIntegrationTest {

    @Autowired
    private CircuitBreakerRegistry circuitBreakerRegistry;

    @Autowired
    private MockMvc mockMvc;

    @MockBean
    private RestClient paymentRestClient;

    @Test
    @DisplayName("실패율 임계값 초과 시 CircuitBreaker가 OPEN으로 전이된다")
    void shouldTransitionToOpenOnFailureThreshold() {
        CircuitBreaker cb = circuitBreakerRegistry.circuitBreaker("paymentGateway");
        cb.reset(); // 테스트 격리를 위해 상태 초기화

        assertThat(cb.getState()).isEqualTo(CircuitBreaker.State.CLOSED);

        // slidingWindowSize=20, failureRateThreshold=40 일 때
        // minimumNumberOfCalls=10 이상 호출 후 40% 이상 실패해야 OPEN
        // 10번 호출 중 5번 실패 = 50% 실패율 -> OPEN 전이
        for (int i = 0; i < 5; i++) {
            cb.onSuccess(100, TimeUnit.MILLISECONDS);
        }
        for (int i = 0; i < 5; i++) {
            cb.onError(100, TimeUnit.MILLISECONDS, new IOException("connection refused"));
        }

        assertThat(cb.getState()).isEqualTo(CircuitBreaker.State.OPEN);
        assertThat(cb.getMetrics().getFailureRate()).isGreaterThanOrEqualTo(40f);
    }

    @Test
    @DisplayName("OPEN 상태에서 waitDuration 경과 후 HALF-OPEN으로 전이된다")
    void shouldTransitionToHalfOpenAfterWaitDuration() throws Exception {
        CircuitBreaker cb = circuitBreakerRegistry.circuitBreaker("paymentGateway");
        cb.reset();
        cb.transitionToOpenState();

        assertThat(cb.getState()).isEqualTo(CircuitBreaker.State.OPEN);

        // waitDurationInOpenState 경과를 시뮬레이션
        // (테스트에서는 짧은 waitDuration 설정을 사용하거나 직접 전이)
        cb.transitionToHalfOpenState();

        assertThat(cb.getState()).isEqualTo(CircuitBreaker.State.HALF_OPEN);
    }

    @Test
    @DisplayName("HALF-OPEN 시험 호출 성공 시 CLOSED로 복귀한다")
    void shouldTransitionToClosedOnSuccessfulTrialCalls() {
        CircuitBreaker cb = circuitBreakerRegistry.circuitBreaker("paymentGateway");
        cb.reset();
        cb.transitionToOpenState();
        cb.transitionToHalfOpenState();

        // permittedNumberOfCallsInHalfOpenState=5 만큼 성공
        for (int i = 0; i < 5; i++) {
            cb.onSuccess(50, TimeUnit.MILLISECONDS);
        }

        assertThat(cb.getState()).isEqualTo(CircuitBreaker.State.CLOSED);
    }

    @Test
    @DisplayName("폴백 메서드가 CircuitBreaker OPEN 시 정상 호출된다")
    void shouldInvokeFallbackWhenCircuitIsOpen() throws Exception {
        // 강제로 서킷을 OPEN으로 전환
        CircuitBreaker cb = circuitBreakerRegistry.circuitBreaker("paymentGateway");
        cb.transitionToForcedOpenState();

        mockMvc.perform(post("/api/v1/orders")
                .contentType(MediaType.APPLICATION_JSON)
                .content("{\"productId\": \"P001\", \"quantity\": 1}"))
            .andExpect(status().isOk())
            .andExpect(jsonPath("$.payment.status").value("QUEUED"))
            .andExpect(jsonPath("$.payment.message").exists());

        // 테스트 후 상태 복원
        cb.transitionToClosedState();
    }
}