Split View: Circuit Breaker 패턴과 Resilience4j 실전 구현 가이드: 장애 전파 차단부터 복구까지

                     Failure rate >= threshold
         ┌─────────────────────────────────────┐
         │                                     │
         ▼                                     │
    ┌──────────┐                          ┌──────────┐
    │          │    waitDuration elapsed   │          │
    │   OPEN   │ ─────────────────────>   │  CLOSED  │
    │ (blocked)│                          │ (normal) │
    └──────────┘                          └──────────┘
         │                                     ▲
         │ waitDuration elapsed                │
         ▼                                     │ Trial call success rate >= threshold
    ┌──────────────┐                           │
    │  HALF-OPEN   │ ──────────────────────────┘
    │(trial allowed)│
    └──────────────┘
         │
         │ Trial call failure rate >= threshold
         │
         ▼
    ┌──────────┐
    │   OPEN   │  (blocked again)
    └──────────┘

Outer (evaluated first) ──────────────────────────────────> Inner (evaluated last)
Retry -> CircuitBreaker -> RateLimiter -> TimeLimiter -> Bulkhead

// build.gradle.kts (Spring Boot 3.3+ / Resilience4j 2.2+)
plugins {
    id("org.springframework.boot") version "3.3.5"
    id("io.spring.dependency-management") version "1.1.6"
    kotlin("jvm") version "1.9.25"
    kotlin("plugin.spring") version "1.9.25"
}

dependencies {
    // Resilience4j Spring Boot 3 Starter
    implementation("io.github.resilience4j:resilience4j-spring-boot3:2.2.0")

    // Individual modules (included in starter but explicit declaration recommended)
    implementation("io.github.resilience4j:resilience4j-circuitbreaker")
    implementation("io.github.resilience4j:resilience4j-retry")
    implementation("io.github.resilience4j:resilience4j-bulkhead")
    implementation("io.github.resilience4j:resilience4j-ratelimiter")
    implementation("io.github.resilience4j:resilience4j-timelimiter")

    // Micrometer + Prometheus (monitoring)
    implementation("io.github.resilience4j:resilience4j-micrometer")
    implementation("io.micrometer:micrometer-registry-prometheus")

    // Spring Boot Actuator
    implementation("org.springframework.boot:spring-boot-starter-actuator")
    implementation("org.springframework.boot:spring-boot-starter-aop")
    implementation("org.springframework.boot:spring-boot-starter-web")

    // Kotlin Coroutines (optional)
    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core")
    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-reactor")

    testImplementation("org.springframework.boot:spring-boot-starter-test")
}

# application.yml - Resilience4j integrated configuration
resilience4j:
  circuitbreaker:
    configs:
      default:
        registerHealthIndicator: true
        slidingWindowType: COUNT_BASED
        slidingWindowSize: 10
        minimumNumberOfCalls: 5
        failureRateThreshold: 50
        slowCallRateThreshold: 80
        slowCallDurationThreshold: 3s
        waitDurationInOpenState: 30s
        permittedNumberOfCallsInHalfOpenState: 3
        automaticTransitionFromOpenToHalfOpenEnabled: true
        recordExceptions:
          - java.io.IOException
          - java.util.concurrent.TimeoutException
          - org.springframework.web.client.HttpServerErrorException
        ignoreExceptions:
          - com.example.order.exception.BusinessValidationException
    instances:
      paymentGateway:
        baseConfig: default
        failureRateThreshold: 40
        waitDurationInOpenState: 60s
        slidingWindowSize: 20
      inventoryService:
        baseConfig: default
        failureRateThreshold: 60
        slowCallDurationThreshold: 5s
      notificationService:
        baseConfig: default
        failureRateThreshold: 70
        waitDurationInOpenState: 15s

  retry:
    configs:
      default:
        maxAttempts: 3
        waitDuration: 1s
        enableExponentialBackoff: true
        exponentialBackoffMultiplier: 2.0
        exponentialMaxWaitDuration: 10s
        retryExceptions:
          - java.io.IOException
          - java.util.concurrent.TimeoutException
        ignoreExceptions:
          - com.example.order.exception.BusinessValidationException
    instances:
      paymentGateway:
        baseConfig: default
        maxAttempts: 2
        waitDuration: 2s
      inventoryService:
        baseConfig: default
        maxAttempts: 4
      notificationService:
        baseConfig: default
        maxAttempts: 5
        waitDuration: 500ms

  bulkhead:
    configs:
      default:
        maxConcurrentCalls: 25
        maxWaitDuration: 500ms
    instances:
      paymentGateway:
        baseConfig: default
        maxConcurrentCalls: 15
      inventoryService:
        baseConfig: default
        maxConcurrentCalls: 30
      notificationService:
        baseConfig: default
        maxConcurrentCalls: 50

  ratelimiter:
    configs:
      default:
        limitForPeriod: 100
        limitRefreshPeriod: 1s
        timeoutDuration: 500ms
    instances:
      paymentGateway:
        baseConfig: default
        limitForPeriod: 50
      inventoryService:
        baseConfig: default
        limitForPeriod: 200

  timelimiter:
    configs:
      default:
        timeoutDuration: 5s
        cancelRunningFuture: true
    instances:
      paymentGateway:
        baseConfig: default
        timeoutDuration: 10s
      inventoryService:
        baseConfig: default
        timeoutDuration: 3s

# Actuator metric exposure
management:
  endpoints:
    web:
      exposure:
        include: health,metrics,prometheus,circuitbreakers,retries
  endpoint:
    health:
      show-details: always
  health:
    circuitbreakers:
      enabled: true
  metrics:
    distribution:
      percentiles-histogram:
        resilience4j.circuitbreaker.calls: true
        resilience4j.retry.calls: true
    tags:
      application: order-service

// PaymentGatewayClient.kt
@Service
class PaymentGatewayClient(
    private val restClient: RestClient,
    private val paymentRetryQueue: PaymentRetryQueue,
    private val paymentCacheStore: PaymentCacheStore,
) {
    companion object {
        private val log = LoggerFactory.getLogger(PaymentGatewayClient::class.java)
        const val CB_NAME = "paymentGateway"
    }

    @CircuitBreaker(name = CB_NAME, fallbackMethod = "paymentFallback")
    @Retry(name = CB_NAME)
    @Bulkhead(name = CB_NAME)
    fun processPayment(request: PaymentRequest): PaymentResponse {
        log.info("Calling payment gateway for orderId={}", request.orderId)

        val response = restClient.post()
            .uri("https://payment-api.internal/v2/charges")
            .contentType(MediaType.APPLICATION_JSON)
            .body(request)
            .retrieve()
            .body(PaymentResponse::class.java)
            ?: throw PaymentGatewayException("Empty response from payment gateway")

        log.info("Payment processed: orderId={}, txId={}", request.orderId, response.transactionId)
        return response
    }

    /**
     * Fallback method: Called when CircuitBreaker is OPEN or an exception occurs.
     * The method signature must match the original + accept an Exception as the last parameter.
     */
    private fun paymentFallback(request: PaymentRequest, ex: Exception): PaymentResponse {
        log.warn(
            "Payment fallback activated: orderId={}, reason={}",
            request.orderId, ex.message
        )

        return when (ex) {
            is CallNotPermittedException -> {
                // CircuitBreaker OPEN state: enqueue for async processing
                paymentRetryQueue.enqueue(request)
                PaymentResponse(
                    orderId = request.orderId,
                    status = PaymentStatus.QUEUED,
                    message = "Payment has been queued. It will be processed shortly.",
                    transactionId = null,
                )
            }
            is BulkheadFullException -> {
                // Bulkhead saturated: prompt immediate retry
                PaymentResponse(
                    orderId = request.orderId,
                    status = PaymentStatus.RETRY_LATER,
                    message = "Too many payment requests at the moment. Please try again shortly.",
                    transactionId = null,
                )
            }
            else -> {
                // Other exceptions: return cached payment info if available
                val cached = paymentCacheStore.getLastSuccess(request.orderId)
                if (cached != null) {
                    log.info("Returning cached payment for orderId={}", request.orderId)
                    cached.copy(status = PaymentStatus.CACHED)
                } else {
                    paymentRetryQueue.enqueue(request)
                    PaymentResponse(
                        orderId = request.orderId,
                        status = PaymentStatus.PENDING,
                        message = "An error occurred during payment processing. Automatic retry in progress.",
                        transactionId = null,
                    )
                }
            }
        }
    }
}

// InventoryServiceClient.java
@Service
@Slf4j
public class InventoryServiceClient {

    private final CircuitBreaker circuitBreaker;
    private final Retry retry;
    private final Bulkhead bulkhead;
    private final RestClient restClient;

    public InventoryServiceClient(
            CircuitBreakerRegistry cbRegistry,
            RetryRegistry retryRegistry,
            BulkheadRegistry bulkheadRegistry,
            RestClient.Builder restClientBuilder) {

        this.circuitBreaker = cbRegistry.circuitBreaker("inventoryService");
        this.retry = retryRegistry.retry("inventoryService");
        this.bulkhead = bulkheadRegistry.bulkhead("inventoryService");
        this.restClient = restClientBuilder
                .baseUrl("https://inventory-api.internal")
                .build();

        // Register event listeners
        registerEventListeners();
    }

    public InventoryResponse checkStock(String productId, int quantity) {
        // Decorator chain: Bulkhead -> CircuitBreaker -> Retry -> actual call
        Supplier<InventoryResponse> decorated = Decorators
                .ofSupplier(() -> doCheckStock(productId, quantity))
                .withBulkhead(bulkhead)
                .withCircuitBreaker(circuitBreaker)
                .withRetry(retry)
                .withFallback(
                    List.of(
                        CallNotPermittedException.class,
                        BulkheadFullException.class,
                        IOException.class
                    ),
                    ex -> stockFallback(productId, quantity, ex)
                )
                .decorate();

        return decorated.get();
    }

    private InventoryResponse doCheckStock(String productId, int quantity) {
        return restClient.get()
                .uri("/v1/stock/{productId}?qty={qty}", productId, quantity)
                .retrieve()
                .body(InventoryResponse.class);
    }

    private InventoryResponse stockFallback(
            String productId, int quantity, Throwable ex) {
        log.warn("Inventory fallback: productId={}, reason={}", productId, ex.getMessage());
        // When stock is uncertain, accept the order but schedule async verification
        return InventoryResponse.builder()
                .productId(productId)
                .available(true)
                .reservationStatus(ReservationStatus.TENTATIVE)
                .message("Stock check delayed: tentative approval with async verification scheduled")
                .build();
    }

    private void registerEventListeners() {
        circuitBreaker.getEventPublisher()
            .onStateTransition(event -> {
                log.warn("[CircuitBreaker] {} state: {} -> {}",
                    event.getCircuitBreakerName(),
                    event.getStateTransition().getFromState(),
                    event.getStateTransition().getToState());
            })
            .onError(event ->
                log.error("[CircuitBreaker] {} error: {} ({}ms)",
                    event.getCircuitBreakerName(),
                    event.getThrowable().getMessage(),
                    event.getElapsedDuration().toMillis())
            )
            .onSuccess(event ->
                log.debug("[CircuitBreaker] {} success ({}ms)",
                    event.getCircuitBreakerName(),
                    event.getElapsedDuration().toMillis())
            )
            .onCallNotPermitted(event ->
                log.warn("[CircuitBreaker] {} call not permitted (OPEN state)",
                    event.getCircuitBreakerName())
            );

        retry.getEventPublisher()
            .onRetry(event ->
                log.info("[Retry] {} attempt #{} (wait: {}ms)",
                    event.getName(),
                    event.getNumberOfRetryAttempts(),
                    event.getWaitInterval().toMillis())
            );
    }
}

// Programmatic RetryConfig customization
@Configuration
class ResilienceConfig {

    @Bean
    fun customRetryConfig(): RetryConfig {
        return RetryConfig.custom<RetryConfig>()
            .maxAttempts(4)
            .intervalFunction(
                // Exponential backoff + jitter: 1s, 2s(+jitter), 4s(+jitter), 8s(+jitter)
                IntervalFunction.ofExponentialRandomBackoff(
                    Duration.ofSeconds(1),   // initial wait duration
                    2.0,                     // multiplier
                    Duration.ofSeconds(15)   // max wait duration
                )
            )
            .retryOnException { ex ->
                // Determine retry-eligible exceptions
                when (ex) {
                    is IOException -> true
                    is TimeoutException -> true
                    is HttpServerErrorException -> true
                    is ConnectException -> true
                    else -> false
                }
            }
            .ignoreExceptions(
                BusinessValidationException::class.java,
                IllegalArgumentException::class.java
            )
            .failAfterMaxAttempts(true) // Throw MaxRetriesExceededException after max retries
            .build()
    }

    @Bean
    fun retryRegistry(customRetryConfig: RetryConfig): RetryRegistry {
        return RetryRegistry.of(customRetryConfig)
    }
}

# ThreadPoolBulkhead configuration example
resilience4j:
  thread-pool-bulkhead:
    instances:
      heavyProcessing:
        maxThreadPoolSize: 10
        coreThreadPoolSize: 5
        queueCapacity: 20
        keepAliveDuration: 100ms
        writableStackTraceEnabled: true

// RateLimiter and CircuitBreaker combination
@Service
@Slf4j
public class ExternalApiClient {

    private final RestClient restClient;

    @CircuitBreaker(name = "externalApi", fallbackMethod = "apiFallback")
    @RateLimiter(name = "externalApi")
    @Retry(name = "externalApi")
    public ApiResponse callExternalApi(ApiRequest request) {
        log.debug("Calling external API: endpoint={}", request.getEndpoint());

        return restClient.post()
                .uri(request.getEndpoint())
                .body(request.getPayload())
                .retrieve()
                .body(ApiResponse.class);
    }

    private ApiResponse apiFallback(ApiRequest request, RequestNotPermitted ex) {
        // Rejected by RateLimiter
        log.warn("Rate limit exceeded for external API: {}", request.getEndpoint());
        return ApiResponse.rateLimited(
                "Request limit exceeded. " +
                "Check limitForPeriod settings or try again later."
        );
    }

    private ApiResponse apiFallback(ApiRequest request, Exception ex) {
        // Other exceptions (CircuitBreaker OPEN, network errors, etc.)
        log.warn("External API fallback: endpoint={}, reason={}",
                request.getEndpoint(), ex.getMessage());
        return ApiResponse.error("External API call failed: " + ex.getMessage());
    }
}

# CircuitBreaker state check (0=CLOSED, 1=OPEN, 2=HALF_OPEN, 3=DISABLED, 4=FORCED_OPEN)
resilience4j_circuitbreaker_state{name="paymentGateway"}

# Failure rate (%)
resilience4j_circuitbreaker_failure_rate{name="paymentGateway"}

# Slow call rate (%)
resilience4j_circuitbreaker_slow_call_rate{name="paymentGateway"}

# Call statistics (kind: successful, failed, ignored, not_permitted)
rate(resilience4j_circuitbreaker_calls_seconds_count{name="paymentGateway"}[5m])

# Call latency distribution (histogram)
histogram_quantile(0.95,
  rate(resilience4j_circuitbreaker_calls_seconds_bucket{name="paymentGateway"}[5m])
)

# Retry count
increase(resilience4j_retry_calls_total{name="paymentGateway", kind="successful_with_retry"}[1h])
increase(resilience4j_retry_calls_total{name="paymentGateway", kind="failed_with_retry"}[1h])

# Bulkhead available concurrent calls
resilience4j_bulkhead_available_concurrent_calls{name="paymentGateway"}

# RateLimiter available permissions
resilience4j_ratelimiter_available_permissions{name="externalApi"}

resilience4j_circuitbreaker_state{application="order-service"}

resilience4j_circuitbreaker_failure_rate{application="order-service", name=~".*"}

sum by (name, kind) (
  rate(resilience4j_circuitbreaker_calls_seconds_count{application="order-service"}[5m])
)

histogram_quantile(0.95,
  sum by (le, name) (
    rate(resilience4j_circuitbreaker_calls_seconds_bucket{application="order-service"}[5m])
  )
)

resilience4j_bulkhead_max_allowed_concurrent_calls{application="order-service"}
- resilience4j_bulkhead_available_concurrent_calls{application="order-service"}

# prometheus-alerts.yml
groups:
  - name: resilience4j_alerts
    rules:
      - alert: CircuitBreakerOpen
        expr: resilience4j_circuitbreaker_state == 1
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: 'CircuitBreaker OPEN: {{ $labels.name }}'
          description: >
            The {{ $labels.name }} circuit breaker in service
            {{ $labels.application }} is in OPEN state.
            Check downstream service failures.

      - alert: HighFailureRate
        expr: resilience4j_circuitbreaker_failure_rate > 30
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: 'High failure rate: {{ $labels.name }} ({{ $value }}%)'
          description: >
            The failure rate of {{ $labels.name }} is {{ $value }}%,
            exceeding the warning threshold (30%).

      - alert: BulkheadSaturation
        expr: >
          (resilience4j_bulkhead_max_allowed_concurrent_calls
          - resilience4j_bulkhead_available_concurrent_calls)
          / resilience4j_bulkhead_max_allowed_concurrent_calls > 0.8
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: 'Bulkhead 80% saturated: {{ $labels.name }}'

      - alert: ExcessiveRetries
        expr: >
          rate(resilience4j_retry_calls_total{kind="failed_with_retry"}[5m])
          / rate(resilience4j_retry_calls_total[5m]) > 0.5
        for: 3m
        labels:
          severity: warning
        annotations:
          summary: 'Retry failure rate exceeds 50%: {{ $labels.name }}'

// Original method
@CircuitBreaker(name = "svc", fallbackMethod = "fallback")
public OrderResponse getOrder(String orderId, boolean includeDetails) { ... }

// Correct fallback (same parameters + Exception added)
private OrderResponse fallback(String orderId, boolean includeDetails, Exception ex) { ... }

// Incorrect fallback - compiles but fails to match at runtime
private OrderResponse fallback(String orderId, Exception ex) { ... }  // Missing parameter
private void fallback(String orderId, boolean includeDetails, Exception ex) { ... }  // Return type mismatch

// Force state transition via Actuator endpoint
// POST /actuator/circuitbreakers/{name}/force-close
circuitBreakerRegistry.circuitBreaker("inventoryService")
    .transitionToForcedOpenState();  // or transitionToClosedState()

// DynamicCircuitBreakerFactory.kt
@Component
class DynamicCircuitBreakerFactory(
    private val circuitBreakerRegistry: CircuitBreakerRegistry,
    private val meterRegistry: MeterRegistry,
) {
    private val log = LoggerFactory.getLogger(javaClass)

    /**
     * Dynamically creates a CircuitBreaker based on service name.
     * Returns an existing instance if one already exists.
     */
    fun getOrCreate(
        serviceName: String,
        tier: ServiceTier = ServiceTier.STANDARD,
    ): CircuitBreaker {
        return circuitBreakerRegistry.circuitBreaker(serviceName) {
            buildConfigForTier(tier)
        }.also { cb ->
            registerMetrics(cb)
            log.info(
                "CircuitBreaker created/retrieved: name={}, tier={}, state={}",
                serviceName, tier, cb.state
            )
        }
    }

    private fun buildConfigForTier(tier: ServiceTier): CircuitBreakerConfig {
        return when (tier) {
            ServiceTier.CRITICAL -> CircuitBreakerConfig.custom()
                .failureRateThreshold(30f)
                .slowCallRateThreshold(60f)
                .slowCallDurationThreshold(Duration.ofSeconds(2))
                .waitDurationInOpenState(Duration.ofSeconds(60))
                .slidingWindowSize(20)
                .minimumNumberOfCalls(10)
                .permittedNumberOfCallsInHalfOpenState(5)
                .automaticTransitionFromOpenToHalfOpenEnabled(true)
                .build()

            ServiceTier.STANDARD -> CircuitBreakerConfig.custom()
                .failureRateThreshold(50f)
                .slowCallRateThreshold(80f)
                .slowCallDurationThreshold(Duration.ofSeconds(3))
                .waitDurationInOpenState(Duration.ofSeconds(30))
                .slidingWindowSize(10)
                .minimumNumberOfCalls(5)
                .permittedNumberOfCallsInHalfOpenState(3)
                .automaticTransitionFromOpenToHalfOpenEnabled(true)
                .build()

            ServiceTier.BEST_EFFORT -> CircuitBreakerConfig.custom()
                .failureRateThreshold(70f)
                .slowCallRateThreshold(90f)
                .slowCallDurationThreshold(Duration.ofSeconds(5))
                .waitDurationInOpenState(Duration.ofSeconds(15))
                .slidingWindowSize(5)
                .minimumNumberOfCalls(3)
                .permittedNumberOfCallsInHalfOpenState(2)
                .automaticTransitionFromOpenToHalfOpenEnabled(true)
                .build()
        }
    }

    private fun registerMetrics(cb: CircuitBreaker) {
        TaggedCircuitBreakerMetrics.ofCircuitBreakerRegistry(
            circuitBreakerRegistry
        ).bindTo(meterRegistry)
    }

    enum class ServiceTier {
        CRITICAL,    // Core services like payment, authentication
        STANDARD,    // General services like inventory, shipping
        BEST_EFFORT  // Non-critical services like notifications, recommendations
    }
}

// CircuitBreakerIntegrationTest.java
@SpringBootTest
@AutoConfigureMockMvc
class CircuitBreakerIntegrationTest {

    @Autowired
    private CircuitBreakerRegistry circuitBreakerRegistry;

    @Autowired
    private MockMvc mockMvc;

    @MockBean
    private RestClient paymentRestClient;

    @Test
    @DisplayName("CircuitBreaker transitions to OPEN when failure rate threshold is exceeded")
    void shouldTransitionToOpenOnFailureThreshold() {
        CircuitBreaker cb = circuitBreakerRegistry.circuitBreaker("paymentGateway");
        cb.reset(); // Reset state for test isolation

        assertThat(cb.getState()).isEqualTo(CircuitBreaker.State.CLOSED);

        // With slidingWindowSize=20, failureRateThreshold=40
        // Must call minimumNumberOfCalls=10 or more, then 40%+ failure -> OPEN
        // 10 calls with 5 failures = 50% failure rate -> OPEN transition
        for (int i = 0; i < 5; i++) {
            cb.onSuccess(100, TimeUnit.MILLISECONDS);
        }
        for (int i = 0; i < 5; i++) {
            cb.onError(100, TimeUnit.MILLISECONDS, new IOException("connection refused"));
        }

        assertThat(cb.getState()).isEqualTo(CircuitBreaker.State.OPEN);
        assertThat(cb.getMetrics().getFailureRate()).isGreaterThanOrEqualTo(40f);
    }

    @Test
    @DisplayName("Transitions to HALF-OPEN after waitDuration elapses in OPEN state")
    void shouldTransitionToHalfOpenAfterWaitDuration() throws Exception {
        CircuitBreaker cb = circuitBreakerRegistry.circuitBreaker("paymentGateway");
        cb.reset();
        cb.transitionToOpenState();

        assertThat(cb.getState()).isEqualTo(CircuitBreaker.State.OPEN);

        // Simulate waitDurationInOpenState elapsed
        // (In tests, use a short waitDuration setting or transition directly)
        cb.transitionToHalfOpenState();

        assertThat(cb.getState()).isEqualTo(CircuitBreaker.State.HALF_OPEN);
    }

    @Test
    @DisplayName("Transitions to CLOSED on successful trial calls in HALF-OPEN")
    void shouldTransitionToClosedOnSuccessfulTrialCalls() {
        CircuitBreaker cb = circuitBreakerRegistry.circuitBreaker("paymentGateway");
        cb.reset();
        cb.transitionToOpenState();
        cb.transitionToHalfOpenState();

        // permittedNumberOfCallsInHalfOpenState=5 successful calls
        for (int i = 0; i < 5; i++) {
            cb.onSuccess(50, TimeUnit.MILLISECONDS);
        }

        assertThat(cb.getState()).isEqualTo(CircuitBreaker.State.CLOSED);
    }

    @Test
    @DisplayName("Fallback method is properly called when CircuitBreaker is OPEN")
    void shouldInvokeFallbackWhenCircuitIsOpen() throws Exception {
        // Force circuit to OPEN state
        CircuitBreaker cb = circuitBreakerRegistry.circuitBreaker("paymentGateway");
        cb.transitionToForcedOpenState();

        mockMvc.perform(post("/api/v1/orders")
                .contentType(MediaType.APPLICATION_JSON)
                .content("{\"productId\": \"P001\", \"quantity\": 1}"))
            .andExpect(status().isOk())
            .andExpect(jsonPath("$.payment.status").value("QUEUED"))
            .andExpect(jsonPath("$.payment.message").exists());

        // Restore state after test
        cb.transitionToClosedState();
    }
}