Overview
A Circuit Breaker Dashboard provides real-time monitoring and visualization of circuit breaker states across microservices. It helps track system health, failure rates, and operational metrics.
Architecture Components
- Circuit Breaker Registry: Central registry for all circuit breakers
- Metrics Collector: Gathers metrics from circuit breakers
- Dashboard API: REST API for dashboard data
- Web Dashboard: Real-time UI for visualization
- Alerting System: Notifications for state changes
Core Implementation
1. Circuit Breaker Interface and Implementation
public interface CircuitBreaker {
String getName();
State getState();
Metrics getMetrics();
boolean allowRequest();
void onSuccess();
void onFailure();
void onTimeout();
enum State {
CLOSED, OPEN, HALF_OPEN
}
class Metrics {
private final long totalRequests;
private final long failedRequests;
private final long timeoutRequests;
private final double failureRate;
private final long lastFailureTimestamp;
private final long lastSuccessTimestamp;
// Constructor, getters, builder pattern
public Metrics(long totalRequests, long failedRequests, long timeoutRequests,
double failureRate, long lastFailureTimestamp, long lastSuccessTimestamp) {
this.totalRequests = totalRequests;
this.failedRequests = failedRequests;
this.timeoutRequests = timeoutRequests;
this.failureRate = failureRate;
this.lastFailureTimestamp = lastFailureTimestamp;
this.lastSuccessTimestamp = lastSuccessTimestamp;
}
// Getters
public long getTotalRequests() { return totalRequests; }
public long getFailedRequests() { return failedRequests; }
public long getTimeoutRequests() { return timeoutRequests; }
public double getFailureRate() { return failureRate; }
public long getLastFailureTimestamp() { return lastFailureTimestamp; }
public long getLastSuccessTimestamp() { return lastSuccessTimestamp; }
public static class Builder {
private long totalRequests;
private long failedRequests;
private long timeoutRequests;
private double failureRate;
private long lastFailureTimestamp;
private long lastSuccessTimestamp;
public Builder totalRequests(long totalRequests) {
this.totalRequests = totalRequests;
return this;
}
public Builder failedRequests(long failedRequests) {
this.failedRequests = failedRequests;
return this;
}
public Builder timeoutRequests(long timeoutRequests) {
this.timeoutRequests = timeoutRequests;
return this;
}
public Builder failureRate(double failureRate) {
this.failureRate = failureRate;
return this;
}
public Builder lastFailureTimestamp(long lastFailureTimestamp) {
this.lastFailureTimestamp = lastFailureTimestamp;
return this;
}
public Builder lastSuccessTimestamp(long lastSuccessTimestamp) {
this.lastSuccessTimestamp = lastSuccessTimestamp;
return this;
}
public Metrics build() {
return new Metrics(totalRequests, failedRequests, timeoutRequests,
failureRate, lastFailureTimestamp, lastSuccessTimestamp);
}
}
}
}
2. Resilience4j-based Circuit Breaker Implementation
import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry;
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
import io.github.resilience4j.circuitbreaker.event.CircuitBreakerEvent;
import java.time.Duration;
import java.util.concurrent.atomic.AtomicLong;
public class Resilience4jCircuitBreaker implements CircuitBreaker {
private final String name;
private final CircuitBreaker resilience4jCircuitBreaker;
private final AtomicLong totalRequests = new AtomicLong(0);
private final AtomicLong failedRequests = new AtomicLong(0);
private final AtomicLong timeoutRequests = new AtomicLong(0);
private volatile long lastFailureTimestamp = 0;
private volatile long lastSuccessTimestamp = 0;
public Resilience4jCircuitBreaker(String name, CircuitBreakerConfig config) {
this.name = name;
this.resilience4jCircuitBreaker = CircuitBreakerRegistry.of(config).circuitBreaker(name);
// Register event listeners for metrics
this.resilience4jCircuitBreaker.getEventPublisher()
.onSuccess(event -> {
lastSuccessTimestamp = System.currentTimeMillis();
totalRequests.incrementAndGet();
})
.onError(event -> {
lastFailureTimestamp = System.currentTimeMillis();
totalRequests.incrementAndGet();
failedRequests.incrementAndGet();
})
.onCallNotPermitted(event -> {
totalRequests.incrementAndGet();
})
.onStateTransition(event -> {
System.out.println("CircuitBreaker " + name + " changed state from " +
event.getStateTransition().getFromState() + " to " +
event.getStateTransition().getToState());
});
}
public static CircuitBreakerConfig defaultConfig() {
return CircuitBreakerConfig.custom()
.failureRateThreshold(50) // 50% failure rate
.slowCallRateThreshold(50) // 50% slow calls
.slowCallDurationThreshold(Duration.ofSeconds(2))
.waitDurationInOpenState(Duration.ofSeconds(60))
.permittedNumberOfCallsInHalfOpenState(10)
.minimumNumberOfCalls(10)
.slidingWindowType(CircuitBreakerConfig.SlidingWindowType.COUNT_BASED)
.slidingWindowSize(100)
.build();
}
@Override
public String getName() {
return name;
}
@Override
public State getState() {
switch (resilience4jCircuitBreaker.getState()) {
case CLOSED: return State.CLOSED;
case OPEN: return State.OPEN;
case HALF_OPEN: return State.HALF_OPEN;
case FORCED_OPEN: return State.OPEN;
case DISABLED: return State.CLOSED;
default: return State.CLOSED;
}
}
@Override
public Metrics getMetrics() {
long total = totalRequests.get();
long failed = failedRequests.get();
long timeout = timeoutRequests.get();
double failureRate = total > 0 ? (double) failed / total * 100 : 0.0;
CircuitBreaker.Metrics resilienceMetrics = resilience4jCircuitBreaker.getMetrics();
return new Metrics.Builder()
.totalRequests(total)
.failedRequests(failed)
.timeoutRequests(timeout)
.failureRate(failureRate)
.lastFailureTimestamp(lastFailureTimestamp)
.lastSuccessTimestamp(lastSuccessTimestamp)
.build();
}
@Override
public boolean allowRequest() {
return resilience4jCircuitBreaker.tryAcquirePermission();
}
@Override
public void onSuccess() {
resilience4jCircuitBreaker.onSuccess(System.nanoTime(), Duration.ofNanos(0));
}
@Override
public void onFailure() {
resilience4jCircuitBreaker.onError(System.nanoTime(), Duration.ofNanos(0),
new RuntimeException("Operation failed"));
}
@Override
public void onTimeout() {
timeoutRequests.incrementAndGet();
resilience4jCircuitBreaker.onError(System.nanoTime(), Duration.ofNanos(0),
new RuntimeException("Operation timeout"));
}
// Additional methods for dashboard integration
public CircuitBreaker.State getDetailedState() {
return resilience4jCircuitBreaker.getState();
}
public io.github.resilience4j.circuitbreaker.CircuitBreaker.Metrics getResilienceMetrics() {
return resilience4jCircuitBreaker.getMetrics();
}
}
3. Circuit Breaker Registry
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
public class CircuitBreakerRegistry {
private static CircuitBreakerRegistry instance;
private final Map<String, CircuitBreaker> circuitBreakers = new ConcurrentHashMap<>();
private final List<CircuitBreakerStateListener> listeners = new CopyOnWriteArrayList<>();
private CircuitBreakerRegistry() {}
public static synchronized CircuitBreakerRegistry getInstance() {
if (instance == null) {
instance = new CircuitBreakerRegistry();
}
return instance;
}
public CircuitBreaker register(String name, CircuitBreakerConfig config) {
return circuitBreakers.computeIfAbsent(name,
k -> new Resilience4jCircuitBreaker(name, config));
}
public CircuitBreaker getCircuitBreaker(String name) {
return circuitBreakers.get(name);
}
public Collection<CircuitBreaker> getAllCircuitBreakers() {
return Collections.unmodifiableCollection(circuitBreakers.values());
}
public Map<String, CircuitBreaker> getCircuitBreakersByService(String serviceName) {
return circuitBreakers.entrySet().stream()
.filter(entry -> entry.getKey().startsWith(serviceName + "."))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
public void addListener(CircuitBreakerStateListener listener) {
listeners.add(listener);
}
public void removeListener(CircuitBreakerStateListener listener) {
listeners.remove(listener);
}
public void notifyStateChange(CircuitBreaker circuitBreaker,
CircuitBreaker.State oldState,
CircuitBreaker.State newState) {
for (CircuitBreakerStateListener listener : listeners) {
listener.onStateChange(circuitBreaker, oldState, newState);
}
}
public void notifyMetricsUpdate(CircuitBreaker circuitBreaker) {
for (CircuitBreakerStateListener listener : listeners) {
listener.onMetricsUpdate(circuitBreaker);
}
}
public interface CircuitBreakerStateListener {
void onStateChange(CircuitBreaker circuitBreaker,
CircuitBreaker.State oldState,
CircuitBreaker.State newState);
void onMetricsUpdate(CircuitBreaker circuitBreaker);
}
// Configuration class
public static class CircuitBreakerConfig {
private double failureRateThreshold = 50.0;
private int slowCallRateThreshold = 50;
private long slowCallDurationThreshold = 2000; // ms
private long waitDurationInOpenState = 60000; // ms
private int permittedNumberOfCallsInHalfOpenState = 10;
private int minimumNumberOfCalls = 10;
private int slidingWindowSize = 100;
// Getters and setters
public double getFailureRateThreshold() { return failureRateThreshold; }
public void setFailureRateThreshold(double failureRateThreshold) {
this.failureRateThreshold = failureRateThreshold;
}
public int getSlowCallRateThreshold() { return slowCallRateThreshold; }
public void setSlowCallRateThreshold(int slowCallRateThreshold) {
this.slowCallRateThreshold = slowCallRateThreshold;
}
public long getSlowCallDurationThreshold() { return slowCallDurationThreshold; }
public void setSlowCallDurationThreshold(long slowCallDurationThreshold) {
this.slowCallDurationThreshold = slowCallDurationThreshold;
}
public long getWaitDurationInOpenState() { return waitDurationInOpenState; }
public void setWaitDurationInOpenState(long waitDurationInOpenState) {
this.waitDurationInOpenState = waitDurationInOpenState;
}
public int getPermittedNumberOfCallsInHalfOpenState() {
return permittedNumberOfCallsInHalfOpenState;
}
public void setPermittedNumberOfCallsInHalfOpenState(int permittedNumberOfCallsInHalfOpenState) {
this.permittedNumberOfCallsInHalfOpenState = permittedNumberOfCallsInHalfOpenState;
}
public int getMinimumNumberOfCalls() { return minimumNumberOfCalls; }
public void setMinimumNumberOfCalls(int minimumNumberOfCalls) {
this.minimumNumberOfCalls = minimumNumberOfCalls;
}
public int getSlidingWindowSize() { return slidingWindowSize; }
public void setSlidingWindowSize(int slidingWindowSize) {
this.slidingWindowSize = slidingWindowSize;
}
}
}
Dashboard Backend API
1. Spring Boot Dashboard Controller
import org.springframework.web.bind.annotation.*;
import org.springframework.http.ResponseEntity;
import org.springframework.http.MediaType;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.*;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/api/dashboard")
@CrossOrigin(origins = "*") // For frontend development
public class CircuitBreakerDashboardController {
private final CircuitBreakerRegistry registry;
private final MetricsService metricsService;
private final AlertService alertService;
private final ObjectMapper objectMapper;
public CircuitBreakerDashboardController(CircuitBreakerRegistry registry,
MetricsService metricsService,
AlertService alertService,
ObjectMapper objectMapper) {
this.registry = registry;
this.metricsService = metricsService;
this.alertService = alertService;
this.objectMapper = objectMapper;
}
@GetMapping("/circuit-breakers")
public ResponseEntity<List<CircuitBreakerDTO>> getAllCircuitBreakers() {
List<CircuitBreakerDTO> circuitBreakers = registry.getAllCircuitBreakers().stream()
.map(this::toDTO)
.collect(Collectors.toList());
return ResponseEntity.ok(circuitBreakers);
}
@GetMapping("/circuit-breakers/{name}")
public ResponseEntity<CircuitBreakerDTO> getCircuitBreaker(@PathVariable String name) {
CircuitBreaker cb = registry.getCircuitBreaker(name);
if (cb == null) {
return ResponseEntity.notFound().build();
}
return ResponseEntity.ok(toDTO(cb));
}
@GetMapping("/circuit-breakers/service/{serviceName}")
public ResponseEntity<Map<String, CircuitBreakerDTO>> getCircuitBreakersByService(
@PathVariable String serviceName) {
Map<String, CircuitBreakerDTO> circuitBreakers =
registry.getCircuitBreakersByService(serviceName).entrySet().stream()
.collect(Collectors.toMap(
Map.Entry::getKey,
entry -> toDTO(entry.getValue())
));
return ResponseEntity.ok(circuitBreakers);
}
@GetMapping("/metrics/{circuitBreakerName}")
public ResponseEntity<CircuitBreakerMetricsDTO> getMetrics(
@PathVariable String circuitBreakerName,
@RequestParam(defaultValue = "1h") String duration) {
CircuitBreaker cb = registry.getCircuitBreaker(circuitBreakerName);
if (cb == null) {
return ResponseEntity.notFound().build();
}
CircuitBreakerMetricsDTO metrics = metricsService.getMetrics(cb, duration);
return ResponseEntity.ok(metrics);
}
@GetMapping("/alerts")
public ResponseEntity<List<AlertDTO>> getActiveAlerts() {
List<AlertDTO> alerts = alertService.getActiveAlerts();
return ResponseEntity.ok(alerts);
}
@PostMapping("/alerts/{alertId}/acknowledge")
public ResponseEntity<Void> acknowledgeAlert(@PathVariable String alertId) {
alertService.acknowledgeAlert(alertId);
return ResponseEntity.ok().build();
}
@GetMapping("/summary")
public ResponseEntity<DashboardSummaryDTO> getDashboardSummary() {
Collection<CircuitBreaker> allCircuitBreakers = registry.getAllCircuitBreakers();
long totalCircuitBreakers = allCircuitBreakers.size();
long openCircuitBreakers = allCircuitBreakers.stream()
.filter(cb -> cb.getState() == CircuitBreaker.State.OPEN)
.count();
long halfOpenCircuitBreakers = allCircuitBreakers.stream()
.filter(cb -> cb.getState() == CircuitBreaker.State.HALF_OPEN)
.count();
double overallHealthScore = calculateOverallHealthScore(allCircuitBreakers);
DashboardSummaryDTO summary = new DashboardSummaryDTO(
totalCircuitBreakers,
openCircuitBreakers,
halfOpenCircuitBreakers,
overallHealthScore,
alertService.getActiveAlerts().size()
);
return ResponseEntity.ok(summary);
}
@GetMapping(value = "/events", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
public SseEmitter streamCircuitBreakerEvents() {
SseEmitter emitter = new SseEmitter(3600000L); // 1 hour timeout
// Register for real-time events
CircuitBreakerRegistry.CircuitBreakerStateListener listener =
new CircuitBreakerRegistry.CircuitBreakerStateListener() {
@Override
public void onStateChange(CircuitBreaker circuitBreaker,
CircuitBreaker.State oldState,
CircuitBreaker.State newState) {
try {
Map<String, Object> event = new HashMap<>();
event.put("type", "STATE_CHANGE");
event.put("circuitBreaker", circuitBreaker.getName());
event.put("oldState", oldState.name());
event.put("newState", newState.name());
event.put("timestamp", System.currentTimeMillis());
emitter.send(SseEmitter.event()
.name("circuit-breaker-event")
.data(objectMapper.writeValueAsString(event)));
} catch (Exception e) {
emitter.completeWithError(e);
}
}
@Override
public void onMetricsUpdate(CircuitBreaker circuitBreaker) {
try {
Map<String, Object> event = new HashMap<>();
event.put("type", "METRICS_UPDATE");
event.put("circuitBreaker", circuitBreaker.getName());
event.put("metrics", toDTO(circuitBreaker).getMetrics());
event.put("timestamp", System.currentTimeMillis());
emitter.send(SseEmitter.event()
.name("circuit-breaker-event")
.data(objectMapper.writeValueAsString(event)));
} catch (Exception e) {
emitter.completeWithError(e);
}
}
};
registry.addListener(listener);
emitter.onCompletion(() -> registry.removeListener(listener));
emitter.onTimeout(() -> registry.removeListener(listener));
emitter.onError((e) -> registry.removeListener(listener));
return emitter;
}
private CircuitBreakerDTO toDTO(CircuitBreaker circuitBreaker) {
CircuitBreaker.Metrics metrics = circuitBreaker.getMetrics();
CircuitBreakerMetricsDTO metricsDTO = new CircuitBreakerMetricsDTO(
metrics.getTotalRequests(),
metrics.getFailedRequests(),
metrics.getTimeoutRequests(),
metrics.getFailureRate(),
metrics.getLastFailureTimestamp(),
metrics.getLastSuccessTimestamp()
);
return new CircuitBreakerDTO(
circuitBreaker.getName(),
circuitBreaker.getState(),
metricsDTO,
System.currentTimeMillis()
);
}
private double calculateOverallHealthScore(Collection<CircuitBreaker> circuitBreakers) {
if (circuitBreakers.isEmpty()) {
return 100.0;
}
double totalScore = 0.0;
int count = 0;
for (CircuitBreaker cb : circuitBreakers) {
CircuitBreaker.Metrics metrics = cb.getMetrics();
if (metrics.getTotalRequests() > 0) {
double health = 100.0 - metrics.getFailureRate();
totalScore += Math.max(0, health);
count++;
}
}
return count > 0 ? totalScore / count : 100.0;
}
// DTO Classes
public static class CircuitBreakerDTO {
private final String name;
private final CircuitBreaker.State state;
private final CircuitBreakerMetricsDTO metrics;
private final long lastUpdated;
public CircuitBreakerDTO(String name, CircuitBreaker.State state,
CircuitBreakerMetricsDTO metrics, long lastUpdated) {
this.name = name;
this.state = state;
this.metrics = metrics;
this.lastUpdated = lastUpdated;
}
// Getters
public String getName() { return name; }
public CircuitBreaker.State getState() { return state; }
public CircuitBreakerMetricsDTO getMetrics() { return metrics; }
public long getLastUpdated() { return lastUpdated; }
}
public static class CircuitBreakerMetricsDTO {
private final long totalRequests;
private final long failedRequests;
private final long timeoutRequests;
private final double failureRate;
private final long lastFailureTimestamp;
private final long lastSuccessTimestamp;
public CircuitBreakerMetricsDTO(long totalRequests, long failedRequests,
long timeoutRequests, double failureRate,
long lastFailureTimestamp, long lastSuccessTimestamp) {
this.totalRequests = totalRequests;
this.failedRequests = failedRequests;
this.timeoutRequests = timeoutRequests;
this.failureRate = failureRate;
this.lastFailureTimestamp = lastFailureTimestamp;
this.lastSuccessTimestamp = lastSuccessTimestamp;
}
// Getters
public long getTotalRequests() { return totalRequests; }
public long getFailedRequests() { return failedRequests; }
public long getTimeoutRequests() { return timeoutRequests; }
public double getFailureRate() { return failureRate; }
public long getLastFailureTimestamp() { return lastFailureTimestamp; }
public long getLastSuccessTimestamp() { return lastSuccessTimestamp; }
}
public static class DashboardSummaryDTO {
private final long totalCircuitBreakers;
private final long openCircuitBreakers;
private final long halfOpenCircuitBreakers;
private final double overallHealthScore;
private final int activeAlerts;
public DashboardSummaryDTO(long totalCircuitBreakers, long openCircuitBreakers,
long halfOpenCircuitBreakers, double overallHealthScore,
int activeAlerts) {
this.totalCircuitBreakers = totalCircuitBreakers;
this.openCircuitBreakers = openCircuitBreakers;
this.halfOpenCircuitBreakers = halfOpenCircuitBreakers;
this.overallHealthScore = overallHealthScore;
this.activeAlerts = activeAlerts;
}
// Getters
public long getTotalCircuitBreakers() { return totalCircuitBreakers; }
public long getOpenCircuitBreakers() { return openCircuitBreakers; }
public long getHalfOpenCircuitBreakers() { return halfOpenCircuitBreakers; }
public double getOverallHealthScore() { return overallHealthScore; }
public int getActiveAlerts() { return activeAlerts; }
}
public static class AlertDTO {
private final String id;
private final String circuitBreakerName;
private final String type;
private final String message;
private final String severity;
private final long timestamp;
private final boolean acknowledged;
public AlertDTO(String id, String circuitBreakerName, String type,
String message, String severity, long timestamp, boolean acknowledged) {
this.id = id;
this.circuitBreakerName = circuitBreakerName;
this.type = type;
this.message = message;
this.severity = severity;
this.timestamp = timestamp;
this.acknowledged = acknowledged;
}
// Getters
public String getId() { return id; }
public String getCircuitBreakerName() { return circuitBreakerName; }
public String getType() { return type; }
public String getMessage() { return message; }
public String getSeverity() { return severity; }
public long getTimestamp() { return timestamp; }
public boolean isAcknowledged() { return acknowledged; }
}
}
2. Metrics Service
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@Service
public class MetricsService {
private final CircuitBreakerRegistry registry;
private final Map<String, List<CircuitBreakerMetricsSnapshot>> metricsHistory =
new ConcurrentHashMap<>();
private final ScheduledExecutorService scheduler =
Executors.newScheduledThreadPool(1);
public MetricsService(CircuitBreakerRegistry registry) {
this.registry = registry;
startMetricsCollection();
}
private void startMetricsCollection() {
// Collect metrics every 10 seconds
scheduler.scheduleAtFixedRate(this::collectMetrics, 0, 10, TimeUnit.SECONDS);
// Clean up old metrics every hour
scheduler.scheduleAtFixedRate(this::cleanupOldMetrics, 1, 1, TimeUnit.HOURS);
}
private void collectMetrics() {
for (CircuitBreaker cb : registry.getAllCircuitBreakers()) {
String cbName = cb.getName();
CircuitBreaker.Metrics currentMetrics = cb.getMetrics();
CircuitBreakerMetricsSnapshot snapshot = new CircuitBreakerMetricsSnapshot(
System.currentTimeMillis(),
currentMetrics.getTotalRequests(),
currentMetrics.getFailedRequests(),
currentMetrics.getTimeoutRequests(),
currentMetrics.getFailureRate(),
cb.getState()
);
metricsHistory.computeIfAbsent(cbName, k -> new ArrayList<>())
.add(snapshot);
}
}
private void cleanupOldMetrics() {
long cutoffTime = System.currentTimeMillis() - (24 * 60 * 60 * 1000); // 24 hours
for (List<CircuitBreakerMetricsSnapshot> snapshots : metricsHistory.values()) {
snapshots.removeIf(snapshot -> snapshot.getTimestamp() < cutoffTime);
}
}
public CircuitBreakerMetricsDTO getMetrics(CircuitBreaker circuitBreaker, String duration) {
String cbName = circuitBreaker.getName();
List<CircuitBreakerMetricsSnapshot> snapshots = metricsHistory.getOrDefault(cbName,
Collections.emptyList());
long durationMillis = parseDuration(duration);
long cutoffTime = System.currentTimeMillis() - durationMillis;
// Filter snapshots for the requested duration
List<CircuitBreakerMetricsSnapshot> filteredSnapshots = snapshots.stream()
.filter(snapshot -> snapshot.getTimestamp() >= cutoffTime)
.collect(Collectors.toList());
if (filteredSnapshots.isEmpty()) {
// Return current metrics if no history
CircuitBreaker.Metrics currentMetrics = circuitBreaker.getMetrics();
return new CircuitBreakerDashboardController.CircuitBreakerMetricsDTO(
currentMetrics.getTotalRequests(),
currentMetrics.getFailedRequests(),
currentMetrics.getTimeoutRequests(),
currentMetrics.getFailureRate(),
currentMetrics.getLastFailureTimestamp(),
currentMetrics.getLastSuccessTimestamp()
);
}
// Calculate aggregated metrics
long totalRequests = filteredSnapshots.stream()
.mapToLong(CircuitBreakerMetricsSnapshot::getTotalRequests)
.max()
.orElse(0);
long failedRequests = filteredSnapshots.stream()
.mapToLong(CircuitBreakerMetricsSnapshot::getFailedRequests)
.max()
.orElse(0);
long timeoutRequests = filteredSnapshots.stream()
.mapToLong(CircuitBreakerMetricsSnapshot::getTimeoutRequests)
.max()
.orElse(0);
double averageFailureRate = filteredSnapshots.stream()
.mapToDouble(CircuitBreakerMetricsSnapshot::getFailureRate)
.average()
.orElse(0.0);
CircuitBreaker.Metrics currentMetrics = circuitBreaker.getMetrics();
return new CircuitBreakerDashboardController.CircuitBreakerMetricsDTO(
totalRequests,
failedRequests,
timeoutRequests,
averageFailureRate,
currentMetrics.getLastFailureTimestamp(),
currentMetrics.getLastSuccessTimestamp()
);
}
public List<CircuitBreakerMetricsSnapshot> getMetricsHistory(String circuitBreakerName,
String duration) {
List<CircuitBreakerMetricsSnapshot> snapshots = metricsHistory.getOrDefault(
circuitBreakerName, Collections.emptyList());
long durationMillis = parseDuration(duration);
long cutoffTime = System.currentTimeMillis() - durationMillis;
return snapshots.stream()
.filter(snapshot -> snapshot.getTimestamp() >= cutoffTime)
.collect(Collectors.toList());
}
private long parseDuration(String duration) {
if (duration.endsWith("h")) {
return Long.parseLong(duration.substring(0, duration.length() - 1)) * 60 * 60 * 1000;
} else if (duration.endsWith("m")) {
return Long.parseLong(duration.substring(0, duration.length() - 1)) * 60 * 1000;
} else if (duration.endsWith("s")) {
return Long.parseLong(duration.substring(0, duration.length() - 1)) * 1000;
} else {
return 3600000; // Default 1 hour
}
}
public static class CircuitBreakerMetricsSnapshot {
private final long timestamp;
private final long totalRequests;
private final long failedRequests;
private final long timeoutRequests;
private final double failureRate;
private final CircuitBreaker.State state;
public CircuitBreakerMetricsSnapshot(long timestamp, long totalRequests,
long failedRequests, long timeoutRequests,
double failureRate, CircuitBreaker.State state) {
this.timestamp = timestamp;
this.totalRequests = totalRequests;
this.failedRequests = failedRequests;
this.timeoutRequests = timeoutRequests;
this.failureRate = failureRate;
this.state = state;
}
// Getters
public long getTimestamp() { return timestamp; }
public long getTotalRequests() { return totalRequests; }
public long getFailedRequests() { return failedRequests; }
public long getTimeoutRequests() { return timeoutRequests; }
public double getFailureRate() { return failureRate; }
public CircuitBreaker.State getState() { return state; }
}
}
3. Alert Service
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
@Service
public class AlertService {
private final List<Alert> activeAlerts = new CopyOnWriteArrayList<>();
private final Map<String, AlertRule> alertRules = new ConcurrentHashMap<>();
private final CircuitBreakerRegistry registry;
public AlertService(CircuitBreakerRegistry registry) {
this.registry = registry;
setupDefaultAlertRules();
startAlertMonitoring();
}
private void setupDefaultAlertRules() {
// Rule for circuit breaker opening
alertRules.put("CIRCUIT_BREAKER_OPENED", new AlertRule(
"CIRCUIT_BREAKER_OPENED",
"Circuit Breaker Opened",
"Circuit breaker {circuitBreakerName} changed state to OPEN",
"HIGH",
(circuitBreaker, oldState, newState) ->
oldState != CircuitBreaker.State.OPEN && newState == CircuitBreaker.State.OPEN
));
// Rule for high failure rate
alertRules.put("HIGH_FAILURE_RATE", new AlertRule(
"HIGH_FAILURE_RATE",
"High Failure Rate",
"Circuit breaker {circuitBreakerName} has high failure rate: {failureRate}%",
"MEDIUM",
(circuitBreaker, oldState, newState) -> {
CircuitBreaker.Metrics metrics = circuitBreaker.getMetrics();
return metrics.getFailureRate() > 30.0; // 30% failure rate threshold
}
));
// Rule for circuit breaker staying open for too long
alertRules.put("CIRCUIT_BREAKER_STUCK_OPEN", new AlertRule(
"CIRCUIT_BREAKER_STUCK_OPEN",
"Circuit Breaker Stuck Open",
"Circuit breaker {circuitBreakerName} has been OPEN for more than 5 minutes",
"HIGH",
(circuitBreaker, oldState, newState) -> {
if (circuitBreaker.getState() == CircuitBreaker.State.OPEN) {
CircuitBreaker.Metrics metrics = circuitBreaker.getMetrics();
long timeSinceLastSuccess = System.currentTimeMillis() -
metrics.getLastSuccessTimestamp();
return timeSinceLastSuccess > 5 * 60 * 1000; // 5 minutes
}
return false;
}
));
}
private void startAlertMonitoring() {
// Register listener for state changes
registry.addListener(new CircuitBreakerRegistry.CircuitBreakerStateListener() {
@Override
public void onStateChange(CircuitBreaker circuitBreaker,
CircuitBreaker.State oldState,
CircuitBreaker.State newState) {
checkAlertRules(circuitBreaker, oldState, newState);
}
@Override
public void onMetricsUpdate(CircuitBreaker circuitBreaker) {
checkAlertRules(circuitBreaker, circuitBreaker.getState(),
circuitBreaker.getState());
}
});
// Periodic check for all circuit breakers
Timer timer = new Timer(true);
timer.scheduleAtFixedRate(new TimerTask() {
@Override
public void run() {
for (CircuitBreaker cb : registry.getAllCircuitBreakers()) {
checkAlertRules(cb, cb.getState(), cb.getState());
}
}
}, 0, 30000); // Check every 30 seconds
}
private void checkAlertRules(CircuitBreaker circuitBreaker,
CircuitBreaker.State oldState,
CircuitBreaker.State newState) {
for (AlertRule rule : alertRules.values()) {
if (rule.getCondition().test(circuitBreaker, oldState, newState)) {
createAlert(rule, circuitBreaker);
}
}
}
private void createAlert(AlertRule rule, CircuitBreaker circuitBreaker) {
String alertId = UUID.randomUUID().toString();
String message = rule.getMessageTemplate()
.replace("{circuitBreakerName}", circuitBreaker.getName())
.replace("{failureRate}",
String.format("%.2f", circuitBreaker.getMetrics().getFailureRate()));
// Check if similar alert already exists
boolean alertExists = activeAlerts.stream()
.anyMatch(alert -> alert.getRuleId().equals(rule.getId()) &&
alert.getCircuitBreakerName().equals(circuitBreaker.getName()) &&
!alert.isAcknowledged());
if (!alertExists) {
Alert alert = new Alert(
alertId,
circuitBreaker.getName(),
rule.getId(),
rule.getTitle(),
message,
rule.getSeverity(),
System.currentTimeMillis(),
false
);
activeAlerts.add(alert);
// TODO: Send notification (email, Slack, etc.)
System.out.println("ALERT: " + message);
}
}
public List<CircuitBreakerDashboardController.AlertDTO> getActiveAlerts() {
return activeAlerts.stream()
.filter(alert -> !alert.isAcknowledged())
.map(this::toDTO)
.collect(Collectors.toList());
}
public void acknowledgeAlert(String alertId) {
activeAlerts.stream()
.filter(alert -> alert.getId().equals(alertId))
.findFirst()
.ifPresent(alert -> alert.setAcknowledged(true));
}
public void addAlertRule(AlertRule rule) {
alertRules.put(rule.getId(), rule);
}
public void removeAlertRule(String ruleId) {
alertRules.remove(ruleId);
}
private CircuitBreakerDashboardController.AlertDTO toDTO(Alert alert) {
return new CircuitBreakerDashboardController.AlertDTO(
alert.getId(),
alert.getCircuitBreakerName(),
alert.getRuleId(),
alert.getMessage(),
alert.getSeverity(),
alert.getTimestamp(),
alert.isAcknowledged()
);
}
// Internal classes
private static class Alert {
private final String id;
private final String circuitBreakerName;
private final String ruleId;
private final String title;
private final String message;
private final String severity;
private final long timestamp;
private volatile boolean acknowledged;
public Alert(String id, String circuitBreakerName, String ruleId,
String title, String message, String severity,
long timestamp, boolean acknowledged) {
this.id = id;
this.circuitBreakerName = circuitBreakerName;
this.ruleId = ruleId;
this.title = title;
this.message = message;
this.severity = severity;
this.timestamp = timestamp;
this.acknowledged = acknowledged;
}
// Getters and setters
public String getId() { return id; }
public String getCircuitBreakerName() { return circuitBreakerName; }
public String getRuleId() { return ruleId; }
public String getTitle() { return title; }
public String getMessage() { return message; }
public String getSeverity() { return severity; }
public long getTimestamp() { return timestamp; }
public boolean isAcknowledged() { return acknowledged; }
public void setAcknowledged(boolean acknowledged) { this.acknowledged = acknowledged; }
}
public static class AlertRule {
private final String id;
private final String title;
private final String messageTemplate;
private final String severity;
private final AlertCondition condition;
public AlertRule(String id, String title, String messageTemplate,
String severity, AlertCondition condition) {
this.id = id;
this.title = title;
this.messageTemplate = messageTemplate;
this.severity = severity;
this.condition = condition;
}
// Getters
public String getId() { return id; }
public String getTitle() { return title; }
public String getMessageTemplate() { return messageTemplate; }
public String getSeverity() { return severity; }
public AlertCondition getCondition() { return condition; }
}
@FunctionalInterface
public interface AlertCondition {
boolean test(CircuitBreaker circuitBreaker,
CircuitBreaker.State oldState,
CircuitBreaker.State newState);
}
}
Frontend Dashboard (React Example)
1. Main Dashboard Component
import React, { useState, useEffect } from 'react';
import { Card, Row, Col, Statistic, Alert, Table, Tag, Spin } from 'antd';
import { LineChart, Line, XAxis, YAxis, CartesianGrid, Tooltip, Legend, ResponsiveContainer } from 'recharts';
const CircuitBreakerDashboard = () => {
const [summary, setSummary] = useState(null);
const [circuitBreakers, setCircuitBreakers] = useState([]);
const [alerts, setAlerts] = useState([]);
const [loading, setLoading] = useState(true);
useEffect(() => {
fetchDashboardData();
setupEventSource();
}, []);
const fetchDashboardData = async () => {
try {
const [summaryRes, circuitBreakersRes, alertsRes] = await Promise.all([
fetch('/api/dashboard/summary'),
fetch('/api/dashboard/circuit-breakers'),
fetch('/api/dashboard/alerts')
]);
const summaryData = await summaryRes.json();
const circuitBreakersData = await circuitBreakersRes.json();
const alertsData = await alertsRes.json();
setSummary(summaryData);
setCircuitBreakers(circuitBreakersData);
setAlerts(alertsData);
} catch (error) {
console.error('Failed to fetch dashboard data:', error);
} finally {
setLoading(false);
}
};
const setupEventSource = () => {
const eventSource = new EventSource('/api/dashboard/events');
eventSource.onmessage = (event) => {
const eventData = JSON.parse(event.data);
if (eventData.type === 'STATE_CHANGE' || eventData.type === 'METRICS_UPDATE') {
fetchDashboardData(); // Refresh data
}
};
eventSource.onerror = (error) => {
console.error('EventSource failed:', error);
eventSource.close();
};
return () => eventSource.close();
};
const getStateColor = (state) => {
switch (state) {
case 'CLOSED': return 'green';
case 'OPEN': return 'red';
case 'HALF_OPEN': return 'orange';
default: return 'gray';
}
};
const columns = [
{
title: 'Name',
dataIndex: 'name',
key: 'name',
},
{
title: 'State',
dataIndex: 'state',
key: 'state',
render: (state) => <Tag color={getStateColor(state)}>{state}</Tag>,
},
{
title: 'Failure Rate',
dataIndex: ['metrics', 'failureRate'],
key: 'failureRate',
render: (rate) => `${rate.toFixed(2)}%`,
},
{
title: 'Total Requests',
dataIndex: ['metrics', 'totalRequests'],
key: 'totalRequests',
},
{
title: 'Failed Requests',
dataIndex: ['metrics', 'failedRequests'],
key: 'failedRequests',
},
{
title: 'Last Updated',
dataIndex: 'lastUpdated',
key: 'lastUpdated',
render: (timestamp) => new Date(timestamp).toLocaleString(),
},
];
if (loading) {
return <Spin size="large" />;
}
return (
<div style={{ padding: '24px' }}>
{/* Summary Cards */}
<Row gutter={16} style={{ marginBottom: '24px' }}>
<Col span={6}>
<Card>
<Statistic
title="Total Circuit Breakers"
value={summary?.totalCircuitBreakers}
/>
</Card>
</Col>
<Col span={6}>
<Card>
<Statistic
title="Open Circuit Breakers"
value={summary?.openCircuitBreakers}
valueStyle={{ color: '#cf1322' }}
/>
</Card>
</Col>
<Col span={6}>
<Card>
<Statistic
title="Half-Open Circuit Breakers"
value={summary?.halfOpenCircuitBreakers}
valueStyle={{ color: '#fa8c16' }}
/>
</Card>
</Col>
<Col span={6}>
<Card>
<Statistic
title="Overall Health Score"
value={summary?.overallHealthScore}
precision={2}
suffix="%"
valueStyle={{ color: '#3f8600' }}
/>
</Card>
</Col>
</Row>
{/* Alerts */}
{alerts.length > 0 && (
<div style={{ marginBottom: '24px' }}>
<Alert
message={`${alerts.length} Active Alerts`}
description={
<ul>
{alerts.map(alert => (
<li key={alert.id}>
<strong>{alert.circuitBreakerName}</strong>: {alert.message}
</li>
))}
</ul>
}
type="warning"
showIcon
closable
/>
</div>
)}
{/* Circuit Breakers Table */}
<Card title="Circuit Breakers" style={{ marginBottom: '24px' }}>
<Table
dataSource={circuitBreakers}
columns={columns}
rowKey="name"
pagination={false}
/>
</Card>
{/* Metrics Chart */}
<Card title="Failure Rate Trends">
<ResponsiveContainer width="100%" height={300}>
<LineChart data={circuitBreakers}>
<CartesianGrid strokeDasharray="3 3" />
<XAxis dataKey="name" />
<YAxis />
<Tooltip />
<Legend />
<Line
type="monotone"
dataKey="metrics.failureRate"
stroke="#8884d8"
name="Failure Rate %"
/>
</LineChart>
</ResponsiveContainer>
</Card>
</div>
);
};
export default CircuitBreakerDashboard;
Configuration and Deployment
1. Spring Boot Configuration
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.CorsRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
@Configuration
public class DashboardConfig {
@Bean
public CircuitBreakerRegistry circuitBreakerRegistry() {
return CircuitBreakerRegistry.getInstance();
}
@Bean
public WebMvcConfigurer corsConfigurer() {
return new WebMvcConfigurer() {
@Override
public void addCorsMappings(CorsRegistry registry) {
registry.addMapping("/api/**")
.allowedOrigins("http://localhost:3000") // React app
.allowedMethods("GET", "POST", "PUT", "DELETE")
.allowCredentials(true);
}
};
}
@Bean
public CircuitBreakerRegistry.CircuitBreakerConfig defaultCircuitBreakerConfig() {
CircuitBreakerRegistry.CircuitBreakerConfig config =
new CircuitBreakerRegistry.CircuitBreakerConfig();
config.setFailureRateThreshold(50.0);
config.setSlowCallRateThreshold(50);
config.setSlowCallDurationThreshold(2000);
config.setWaitDurationInOpenState(60000);
config.setPermittedNumberOfCallsInHalfOpenState(10);
config.setMinimumNumberOfCalls(10);
config.setSlidingWindowSize(100);
return config;
}
}
2. Application Properties
# application.properties server.port=8080 spring.application.name=circuit-breaker-dashboard # Logging logging.level.com.yourcompany.dashboard=DEBUG # Metrics retention (24 hours) dashboard.metrics.retention-hours=24 # Alert settings dashboard.alerts.enabled=true dashboard.alerts.notification.email.enabled=false dashboard.alerts.notification.slack.enabled=false # Security (if needed) spring.security.oauth2.client.registration.google.client-id=your-client-id spring.security.oauth2.client.registration.google.client-secret=your-client-secret
Best Practices
- Real-time Updates: Use Server-Sent Events for live dashboard updates
- Historical Data: Store metrics for trend analysis and debugging
- Alerting: Implement multi-level alerting with different severities
- Security: Secure dashboard endpoints with authentication/authorization
- Scalability: Use distributed caching for large-scale deployments
- Customization: Allow custom alert rules and dashboard configurations
- Integration: Integrate with existing monitoring systems (Prometheus, Grafana)
- Testing: Test circuit breaker behavior under different failure scenarios
This Circuit Breaker Dashboard provides comprehensive monitoring, alerting, and visualization capabilities for managing circuit breakers in distributed systems.