RE
Circuit Breaker
Resilience core v1.0.0
Circuit Breaker Pattern
Overview
The circuit breaker pattern prevents cascading failures by stopping requests to failing services. When failures exceed a threshold, the circuit “opens” and fails fast, giving the downstream service time to recover.
Key Concepts
Circuit Breaker States
┌─────────────────────────────────────────────────────────────┐
│ Circuit Breaker States │
├─────────────────────────────────────────────────────────────┤
│ │
│ ┌─────────────┐ │
│ success │ │ failure count │
│ ┌────▶│ CLOSED │◀────┐ < threshold │
│ │ │ (normal) │ │ │
│ │ └──────┬──────┘ │ │
│ │ │ │ │
│ │ failure count >= threshold │
│ │ │ │
│ │ ▼ │
│ │ ┌─────────────┐ │
│ │ │ OPEN │ all requests fail fast │
│ │ │ (blocking) │ │
│ │ └──────┬──────┘ │
│ │ │ │
│ │ timeout expires │
│ │ │ │
│ │ ▼ │
│ │ ┌─────────────┐ │
│ │ │ HALF-OPEN │ allow limited requests │
│ └─────│ (testing) │─────┘ │
│ └─────────────┘ │
│ │ │ │
│ success│ │failure │
│ ▼ ▼ │
│ CLOSED OPEN │
│ │
│ Metrics tracked: │
│ • Failure rate (percentage) │
│ • Slow call rate (calls exceeding threshold) │
│ • Number of permitted calls in half-open │
│ │
└─────────────────────────────────────────────────────────────┘
Best Practices
1. Configure Appropriate Thresholds
Start conservative, tune based on normal failure rates.
2. Use Sliding Window
Count-based or time-based window for failure rate.
3. Implement Meaningful Fallbacks
Provide degraded but useful responses.
4. Monitor Circuit State
Alert on state changes.
5. Different Circuits Per Dependency
Isolate failures to specific integrations.
Code Examples
Example 1: Resilience4j Circuit Breaker
@Configuration
public class CircuitBreakerConfig {
@Bean
public CircuitBreakerRegistry circuitBreakerRegistry() {
CircuitBreakerConfig config = CircuitBreakerConfig.custom()
// Sliding window configuration
.slidingWindowType(SlidingWindowType.COUNT_BASED)
.slidingWindowSize(100)
.minimumNumberOfCalls(10)
// Failure thresholds
.failureRateThreshold(50) // Open if 50% fail
.slowCallRateThreshold(80) // Open if 80% are slow
.slowCallDurationThreshold(Duration.ofSeconds(2))
// State transitions
.waitDurationInOpenState(Duration.ofSeconds(30))
.permittedNumberOfCallsInHalfOpenState(5)
// What counts as failure
.recordExceptions(
IOException.class,
TimeoutException.class,
ServiceUnavailableException.class
)
.ignoreExceptions(
BusinessException.class,
ValidationException.class
)
// Automatic transition from half-open
.automaticTransitionFromOpenToHalfOpenEnabled(true)
.build();
return CircuitBreakerRegistry.of(config);
}
@Bean
public CircuitBreaker paymentServiceCircuitBreaker(CircuitBreakerRegistry registry) {
CircuitBreaker cb = registry.circuitBreaker("payment-service");
// Register event handlers
cb.getEventPublisher()
.onStateTransition(event ->
log.warn("Circuit {} transitioned from {} to {}",
event.getCircuitBreakerName(),
event.getStateTransition().getFromState(),
event.getStateTransition().getToState()))
.onFailureRateExceeded(event ->
log.error("Circuit {} failure rate exceeded: {}%",
event.getCircuitBreakerName(),
event.getFailureRate()))
.onSlowCallRateExceeded(event ->
log.warn("Circuit {} slow call rate exceeded: {}%",
event.getCircuitBreakerName(),
event.getSlowCallRate()));
return cb;
}
}
@Service
public class PaymentServiceClient {
private final CircuitBreaker circuitBreaker;
private final PaymentFallbackService fallbackService;
private final RestClient restClient;
public PaymentResult processPayment(PaymentRequest request) {
Supplier<PaymentResult> decoratedSupplier = CircuitBreaker
.decorateSupplier(circuitBreaker, () -> {
return restClient.post("/payments")
.body(request)
.retrieve()
.body(PaymentResult.class);
});
return Try.ofSupplier(decoratedSupplier)
.recover(CallNotPermittedException.class, e -> {
log.warn("Circuit is open, using fallback");
return fallbackService.processFallback(request);
})
.recover(throwable -> {
log.error("Payment failed, using fallback", throwable);
return fallbackService.processFallback(request);
})
.get();
}
public CircuitBreaker.State getCircuitState() {
return circuitBreaker.getState();
}
public CircuitBreaker.Metrics getMetrics() {
return circuitBreaker.getMetrics();
}
}
Example 2: Spring Cloud Circuit Breaker
@RestController
@RequestMapping("/api/orders")
public class OrderController {
private final CircuitBreakerFactory circuitBreakerFactory;
private final InventoryService inventoryService;
@PostMapping
public ResponseEntity<Order> createOrder(@RequestBody OrderRequest request) {
CircuitBreaker circuitBreaker = circuitBreakerFactory.create("inventory");
// Check inventory with circuit breaker
InventoryStatus inventory = circuitBreaker.run(
() -> inventoryService.checkAvailability(request.getItems()),
throwable -> handleInventoryFallback(request, throwable)
);
if (!inventory.isAvailable()) {
return ResponseEntity.badRequest()
.body(Order.failed("Insufficient inventory"));
}
return ResponseEntity.ok(processOrder(request));
}
private InventoryStatus handleInventoryFallback(OrderRequest request, Throwable t) {
log.warn("Inventory check failed, using cached data", t);
// Try cached inventory data
return cacheService.getCachedInventory(request.getItems())
.orElse(InventoryStatus.unknown());
}
}
// Custom circuit breaker factory configuration
@Configuration
public class CircuitBreakerFactoryConfig {
@Bean
public Customizer<Resilience4JCircuitBreakerFactory> defaultCustomizer() {
return factory -> factory.configureDefault(id ->
new Resilience4JConfigBuilder(id)
.circuitBreakerConfig(CircuitBreakerConfig.custom()
.slidingWindowSize(10)
.failureRateThreshold(50)
.waitDurationInOpenState(Duration.ofSeconds(10))
.build())
.timeLimiterConfig(TimeLimiterConfig.custom()
.timeoutDuration(Duration.ofSeconds(3))
.build())
.build()
);
}
@Bean
public Customizer<Resilience4JCircuitBreakerFactory> criticalServiceCustomizer() {
return factory -> factory.configure(builder ->
builder.circuitBreakerConfig(CircuitBreakerConfig.custom()
.slidingWindowSize(20)
.failureRateThreshold(30) // More sensitive
.waitDurationInOpenState(Duration.ofMinutes(1))
.build()),
"critical-service"
);
}
}
Example 3: Circuit Breaker with Retry
@Service
public class ResilientServiceClient {
private final CircuitBreaker circuitBreaker;
private final Retry retry;
private final TimeLimiter timeLimiter;
public ResilientServiceClient(
CircuitBreakerRegistry cbRegistry,
RetryRegistry retryRegistry,
TimeLimiterRegistry tlRegistry) {
this.circuitBreaker = cbRegistry.circuitBreaker("service");
this.retry = retryRegistry.retry("service");
this.timeLimiter = tlRegistry.timeLimiter("service");
}
/**
* Order matters: Retry -> CircuitBreaker -> TimeLimiter
* TimeLimiter runs first (innermost), then CB, then Retry
*/
public <T> T executeWithResilience(
Callable<T> operation,
Supplier<T> fallback) {
// Wrap with time limiter
Supplier<CompletableFuture<T>> timeLimitedSupplier =
() -> CompletableFuture.supplyAsync(() -> {
try {
return operation.call();
} catch (Exception e) {
throw new RuntimeException(e);
}
});
Callable<T> timeLimitedCallable = TimeLimiter
.decorateCallable(timeLimiter, () ->
timeLimitedSupplier.get().get());
// Wrap with circuit breaker
Callable<T> circuitBreakerCallable = CircuitBreaker
.decorateCallable(circuitBreaker, timeLimitedCallable);
// Wrap with retry
Callable<T> retryingCallable = Retry
.decorateCallable(retry, circuitBreakerCallable);
try {
return retryingCallable.call();
} catch (CallNotPermittedException e) {
log.warn("Circuit breaker is open");
return fallback.get();
} catch (Exception e) {
log.error("All retries exhausted", e);
return fallback.get();
}
}
/**
* Using Decorators API for cleaner composition
*/
public <T> T executeDecorated(
Supplier<T> supplier,
Function<Throwable, T> fallbackFunction) {
return Decorators.ofSupplier(supplier)
.withCircuitBreaker(circuitBreaker)
.withRetry(retry)
.withFallback(List.of(
CallNotPermittedException.class,
TimeoutException.class
), fallbackFunction)
.get();
}
}
// Configuration
@Configuration
public class ResilienceConfig {
@Bean
public RetryRegistry retryRegistry() {
RetryConfig config = RetryConfig.custom()
.maxAttempts(3)
.waitDuration(Duration.ofMillis(500))
.retryExceptions(IOException.class, TimeoutException.class)
.ignoreExceptions(BusinessException.class)
.build();
return RetryRegistry.of(config);
}
@Bean
public TimeLimiterRegistry timeLimiterRegistry() {
TimeLimiterConfig config = TimeLimiterConfig.custom()
.timeoutDuration(Duration.ofSeconds(2))
.cancelRunningFuture(true)
.build();
return TimeLimiterRegistry.of(config);
}
}
Example 4: Circuit Breaker Metrics and Monitoring
@Component
public class CircuitBreakerMetrics {
private final CircuitBreakerRegistry registry;
private final MeterRegistry meterRegistry;
@PostConstruct
public void registerMetrics() {
registry.getAllCircuitBreakers().forEach(this::registerCircuitBreakerMetrics);
// Listen for new circuit breakers
registry.getEventPublisher().onEntryAdded(event ->
registerCircuitBreakerMetrics(event.getAddedEntry())
);
}
private void registerCircuitBreakerMetrics(CircuitBreaker cb) {
String name = cb.getName();
// State gauge
Gauge.builder("circuit.breaker.state", cb,
circuitBreaker -> stateToNumber(circuitBreaker.getState()))
.tag("name", name)
.register(meterRegistry);
// Failure rate
Gauge.builder("circuit.breaker.failure.rate", cb,
circuitBreaker -> circuitBreaker.getMetrics().getFailureRate())
.tag("name", name)
.register(meterRegistry);
// Slow call rate
Gauge.builder("circuit.breaker.slow.rate", cb,
circuitBreaker -> circuitBreaker.getMetrics().getSlowCallRate())
.tag("name", name)
.register(meterRegistry);
// Buffered calls
Gauge.builder("circuit.breaker.calls.buffered", cb,
circuitBreaker -> circuitBreaker.getMetrics().getNumberOfBufferedCalls())
.tag("name", name)
.register(meterRegistry);
// Register event counters
cb.getEventPublisher()
.onSuccess(event ->
meterRegistry.counter("circuit.breaker.calls",
"name", name, "outcome", "success").increment())
.onError(event ->
meterRegistry.counter("circuit.breaker.calls",
"name", name, "outcome", "error").increment())
.onCallNotPermitted(event ->
meterRegistry.counter("circuit.breaker.calls",
"name", name, "outcome", "not_permitted").increment());
}
private double stateToNumber(CircuitBreaker.State state) {
return switch (state) {
case CLOSED -> 0;
case HALF_OPEN -> 0.5;
case OPEN -> 1;
case DISABLED -> -1;
case FORCED_OPEN -> 2;
};
}
}
@RestController
@RequestMapping("/actuator/circuitbreakers")
public class CircuitBreakerActuator {
private final CircuitBreakerRegistry registry;
@GetMapping
public Map<String, CircuitBreakerInfo> getAllCircuitBreakers() {
return registry.getAllCircuitBreakers().stream()
.collect(Collectors.toMap(
CircuitBreaker::getName,
this::toInfo
));
}
@GetMapping("/{name}")
public CircuitBreakerInfo getCircuitBreaker(@PathVariable String name) {
return registry.find(name)
.map(this::toInfo)
.orElseThrow(() -> new NotFoundException("Circuit breaker not found: " + name));
}
@PostMapping("/{name}/reset")
public void resetCircuitBreaker(@PathVariable String name) {
registry.circuitBreaker(name).reset();
}
@PostMapping("/{name}/disable")
public void disableCircuitBreaker(@PathVariable String name) {
registry.circuitBreaker(name).transitionToDisabledState();
}
private CircuitBreakerInfo toInfo(CircuitBreaker cb) {
CircuitBreaker.Metrics metrics = cb.getMetrics();
return new CircuitBreakerInfo(
cb.getName(),
cb.getState().name(),
metrics.getFailureRate(),
metrics.getSlowCallRate(),
metrics.getNumberOfSuccessfulCalls(),
metrics.getNumberOfFailedCalls(),
metrics.getNumberOfNotPermittedCalls()
);
}
}
Example 5: Custom Circuit Breaker with Health Checks
public class HealthCheckCircuitBreaker {
private final AtomicReference<State> state = new AtomicReference<>(State.CLOSED);
private final AtomicInteger failureCount = new AtomicInteger(0);
private final AtomicLong lastFailureTime = new AtomicLong(0);
private final ScheduledExecutorService scheduler;
private final int failureThreshold;
private final Duration openDuration;
private final Duration healthCheckInterval;
private final Supplier<Boolean> healthCheck;
public HealthCheckCircuitBreaker(
int failureThreshold,
Duration openDuration,
Duration healthCheckInterval,
Supplier<Boolean> healthCheck) {
this.failureThreshold = failureThreshold;
this.openDuration = openDuration;
this.healthCheckInterval = healthCheckInterval;
this.healthCheck = healthCheck;
this.scheduler = Executors.newSingleThreadScheduledExecutor();
// Start health check scheduler
scheduler.scheduleAtFixedRate(
this::performHealthCheck,
healthCheckInterval.toMillis(),
healthCheckInterval.toMillis(),
TimeUnit.MILLISECONDS
);
}
public <T> T execute(Supplier<T> operation, Supplier<T> fallback) {
if (!allowRequest()) {
return fallback.get();
}
try {
T result = operation.get();
recordSuccess();
return result;
} catch (Exception e) {
recordFailure();
throw e;
}
}
private boolean allowRequest() {
State currentState = state.get();
if (currentState == State.CLOSED) {
return true;
}
if (currentState == State.OPEN) {
// Check if open duration has passed
if (System.currentTimeMillis() - lastFailureTime.get() > openDuration.toMillis()) {
state.compareAndSet(State.OPEN, State.HALF_OPEN);
return true;
}
return false;
}
// Half-open: allow limited requests
return true;
}
private void recordSuccess() {
State currentState = state.get();
if (currentState == State.HALF_OPEN) {
// Success in half-open closes the circuit
state.set(State.CLOSED);
failureCount.set(0);
}
}
private void recordFailure() {
int failures = failureCount.incrementAndGet();
lastFailureTime.set(System.currentTimeMillis());
State currentState = state.get();
if (currentState == State.CLOSED && failures >= failureThreshold) {
state.set(State.OPEN);
} else if (currentState == State.HALF_OPEN) {
// Failure in half-open reopens
state.set(State.OPEN);
}
}
private void performHealthCheck() {
if (state.get() != State.OPEN) {
return;
}
try {
if (healthCheck.get()) {
log.info("Health check passed, transitioning to half-open");
state.set(State.HALF_OPEN);
}
} catch (Exception e) {
log.debug("Health check failed", e);
}
}
public void forceOpen() {
state.set(State.OPEN);
}
public void reset() {
state.set(State.CLOSED);
failureCount.set(0);
}
enum State {
CLOSED, OPEN, HALF_OPEN
}
}
Anti-Patterns
❌ Same Circuit for All Services
// WRONG - one failing service affects all
CircuitBreaker globalBreaker = ...;
// ✅ CORRECT - separate circuits per dependency
CircuitBreaker paymentBreaker = registry.circuitBreaker("payment");
CircuitBreaker inventoryBreaker = registry.circuitBreaker("inventory");
❌ No Fallback
Always provide a fallback when circuit is open.