Project Overview
A comprehensive resilience testing framework for Java applications that helps identify weaknesses, validate fault tolerance, and ensure system reliability under various failure conditions. Inspired by Chaos Engineering principles and tools like Chaos Monkey, Gremlin, and Resilience4j.
Technology Stack
- Core: Java 17+, JUnit 5, AssertJ
- Fault Injection: ByteBuddy for runtime instrumentation
- Async Testing: Reactor, CompletableFuture
- Monitoring: Micrometer, Micrometer Tracing
- Reporting: Allure, Custom HTML reports
- Configuration: Typesafe Config, YAML
- Build: Maven/Gradle plugins
Architecture
Test Scenarios → Fault Injectors → System Under Test → Observability → Analysis & Reports ↓ ↓ ↓ ↓ ↓ Chaos API Network Latency Instrumented Metrics & HTML Dashboard DSL Service Faults Proxies Traces JUnit Reports
Project Structure
resilience-test-framework/ ├── core/ │ ├── src/main/java/com/resilience/ │ │ ├── fault/ │ │ ├── scenario/ │ │ ├── injector/ │ │ ├── monitor/ │ │ └── report/ │ └── src/test/java/ ├── agent/ │ └── src/main/java/com/resilience/agent/ ├── maven-plugin/ ├── gradle-plugin/ └── examples/
Core Implementation
1. Fault Models
package com.resilience.fault;
import java.time.Duration;
import java.util.concurrent.Callable;
import java.util.function.Supplier;
public interface Fault {
String getName();
FaultType getType();
void apply() throws FaultException;
<T> T apply(Callable<T> callable) throws FaultException;
boolean isEnabled();
enum FaultType {
LATENCY, EXCEPTION, MEMORY, CPU, NETWORK, DISK, SERVICE
}
}
public abstract class AbstractFault implements Fault {
protected final String name;
protected final FaultType type;
protected final boolean enabled;
protected final double probability;
protected AbstractFault(String name, FaultType type, boolean enabled, double probability) {
this.name = name;
this.type = type;
this.enabled = enabled;
this.probability = Math.max(0, Math.min(1, probability));
}
@Override
public String getName() { return name; }
@Override
public FaultType getType() { return type; }
@Override
public boolean isEnabled() { return enabled; }
protected boolean shouldApply() {
return enabled && Math.random() <= probability;
}
@Override
public <T> T apply(Callable<T> callable) throws FaultException {
if (!shouldApply()) {
try {
return callable.call();
} catch (Exception e) {
throw new FaultException("Operation failed without fault injection", e);
}
}
try {
apply();
return callable.call();
} catch (Exception e) {
throw new FaultException("Fault injection caused failure", e);
}
}
}
public class LatencyFault extends AbstractFault {
private final Duration latency;
private final Duration jitter;
private final TimeUnit timeUnit;
public LatencyFault(String name, Duration latency, Duration jitter, double probability) {
super(name, FaultType.LATENCY, true, probability);
this.latency = latency;
this.jitter = jitter != null ? jitter : Duration.ZERO;
}
@Override
public void apply() throws FaultException {
if (!shouldApply()) return;
try {
long actualLatency = calculateActualLatency();
Thread.sleep(actualLatency);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FaultException("Latency fault interrupted", e);
}
}
private long calculateActualLatency() {
long baseMs = latency.toMillis();
long jitterMs = jitter.toMillis();
if (jitterMs > 0) {
long randomJitter = (long) (Math.random() * jitterMs * 2) - jitterMs;
return Math.max(0, baseMs + randomJitter);
}
return baseMs;
}
public static class Builder {
private String name = "latency-fault";
private Duration latency;
private Duration jitter;
private double probability = 1.0;
public Builder name(String name) {
this.name = name;
return this;
}
public Builder latency(Duration latency) {
this.latency = latency;
return this;
}
public Builder latency(long amount, TimeUnit unit) {
this.latency = Duration.of(amount, unit.toChronoUnit());
return this;
}
public Builder jitter(Duration jitter) {
this.jitter = jitter;
return this;
}
public Builder probability(double probability) {
this.probability = probability;
return this;
}
public LatencyFault build() {
if (latency == null) {
throw new IllegalStateException("Latency must be specified");
}
return new LatencyFault(name, latency, jitter, probability);
}
}
}
public class ExceptionFault extends AbstractFault {
private final Supplier<Exception> exceptionSupplier;
private final Class<? extends Exception> exceptionType;
private final String message;
public ExceptionFault(String name, Class<? extends Exception> exceptionType,
String message, double probability) {
super(name, FaultType.EXCEPTION, true, probability);
this.exceptionType = exceptionType;
this.message = message;
this.exceptionSupplier = createExceptionSupplier();
}
public ExceptionFault(String name, Supplier<Exception> exceptionSupplier,
double probability) {
super(name, FaultType.EXCEPTION, true, probability);
this.exceptionSupplier = exceptionSupplier;
this.exceptionType = null;
this.message = null;
}
@Override
public void apply() throws FaultException {
if (!shouldApply()) return;
Exception exception = exceptionSupplier.get();
if (exception instanceof RuntimeException) {
throw (RuntimeException) exception;
} else {
throw new FaultException("Injected exception", exception);
}
}
private Supplier<Exception> createExceptionSupplier() {
return () -> {
try {
if (message != null) {
return exceptionType.getConstructor(String.class).newInstance(message);
} else {
return exceptionType.getConstructor().newInstance();
}
} catch (Exception e) {
return new RuntimeException("Failed to create exception: " + message);
}
};
}
public static class Builder {
private String name = "exception-fault";
private Class<? extends Exception> exceptionType = RuntimeException.class;
private String message;
private double probability = 1.0;
private Supplier<Exception> exceptionSupplier;
public Builder name(String name) {
this.name = name;
return this;
}
public Builder exceptionType(Class<? extends Exception> exceptionType) {
this.exceptionType = exceptionType;
return this;
}
public Builder message(String message) {
this.message = message;
return this;
}
public Builder exceptionSupplier(Supplier<Exception> supplier) {
this.exceptionSupplier = supplier;
return this;
}
public Builder probability(double probability) {
this.probability = probability;
return this;
}
public ExceptionFault build() {
if (exceptionSupplier != null) {
return new ExceptionFault(name, exceptionSupplier, probability);
} else {
return new ExceptionFault(name, exceptionType, message, probability);
}
}
}
}
public class MemoryPressureFault extends AbstractFault {
private final long memoryBytes;
private final Duration duration;
private byte[] memoryBuffer;
public MemoryPressureFault(String name, long memoryBytes, Duration duration, double probability) {
super(name, FaultType.MEMORY, true, probability);
this.memoryBytes = memoryBytes;
this.duration = duration;
}
@Override
public void apply() throws FaultException {
if (!shouldApply()) return;
try {
// Allocate memory to create pressure
memoryBuffer = new byte[(int) Math.min(memoryBytes, Integer.MAX_VALUE)];
// Fill with data to ensure actual memory usage
for (int i = 0; i < memoryBuffer.length; i++) {
memoryBuffer[i] = (byte) (i % 256);
}
// Hold memory for specified duration
if (!duration.isZero()) {
Thread.sleep(duration.toMillis());
}
} catch (OutOfMemoryError e) {
throw new FaultException("Memory allocation failed", e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FaultException("Memory pressure fault interrupted", e);
} finally {
// Release memory
memoryBuffer = null;
System.gc();
}
}
public static class Builder {
private String name = "memory-pressure-fault";
private long memoryBytes = 100 * 1024 * 1024; // 100MB
private Duration duration = Duration.ZERO;
private double probability = 1.0;
public Builder name(String name) {
this.name = name;
return this;
}
public Builder memoryBytes(long memoryBytes) {
this.memoryBytes = memoryBytes;
return this;
}
public Builder memoryMB(long megabytes) {
this.memoryBytes = megabytes * 1024 * 1024;
return this;
}
public Builder duration(Duration duration) {
this.duration = duration;
return this;
}
public Builder probability(double probability) {
this.probability = probability;
return this;
}
public MemoryPressureFault build() {
return new MemoryPressureFault(name, memoryBytes, duration, probability);
}
}
}
public class CPULoadFault extends AbstractFault {
private final Duration duration;
private final int threadCount;
public CPULoadFault(String name, Duration duration, int threadCount, double probability) {
super(name, FaultType.CPU, true, probability);
this.duration = duration;
this.threadCount = Math.max(1, threadCount);
}
@Override
public void apply() throws FaultException {
if (!shouldApply()) return;
var executor = java.util.concurrent.Executors.newFixedThreadPool(threadCount);
var startTime = System.currentTimeMillis();
var endTime = startTime + duration.toMillis();
try {
var tasks = new ArrayList<Callable<Void>>();
for (int i = 0; i < threadCount; i++) {
tasks.add(() -> {
while (System.currentTimeMillis() < endTime &&
!Thread.currentThread().isInterrupted()) {
// Burn CPU cycles
Math.pow(Math.random(), Math.random());
}
return null;
});
}
executor.invokeAll(tasks);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FaultException("CPU load fault interrupted", e);
} finally {
executor.shutdownNow();
}
}
public static class Builder {
private String name = "cpu-load-fault";
private Duration duration = Duration.ofSeconds(10);
private int threadCount = Runtime.getRuntime().availableProcessors();
private double probability = 1.0;
public Builder name(String name) {
this.name = name;
return this;
}
public Builder duration(Duration duration) {
this.duration = duration;
return this;
}
public Builder threadCount(int threadCount) {
this.threadCount = threadCount;
return this;
}
public Builder probability(double probability) {
this.probability = probability;
return this;
}
public CPULoadFault build() {
return new CPULoadFault(name, duration, threadCount, probability);
}
}
}
public class ServiceDegradationFault extends AbstractFault {
private final double errorRate;
private final Duration increasedLatency;
private final Random random = new Random();
public ServiceDegradationFault(String name, double errorRate, Duration increasedLatency,
double probability) {
super(name, FaultType.SERVICE, true, probability);
this.errorRate = Math.max(0, Math.min(1, errorRate));
this.increasedLatency = increasedLatency != null ? increasedLatency : Duration.ZERO;
}
@Override
public void apply() throws FaultException {
if (!shouldApply()) return;
// Apply increased latency
if (!increasedLatency.isZero()) {
try {
Thread.sleep(increasedLatency.toMillis());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FaultException("Service degradation latency interrupted", e);
}
}
// Inject errors based on error rate
if (random.nextDouble() < errorRate) {
throw new FaultException("Service degradation error injected");
}
}
public static class Builder {
private String name = "service-degradation-fault";
private double errorRate = 0.5;
private Duration increasedLatency = Duration.ofSeconds(1);
private double probability = 1.0;
public Builder name(String name) {
this.name = name;
return this;
}
public Builder errorRate(double errorRate) {
this.errorRate = errorRate;
return this;
}
public Builder increasedLatency(Duration latency) {
this.increasedLatency = latency;
return this;
}
public Builder probability(double probability) {
this.probability = probability;
return this;
}
public ServiceDegradationFault build() {
return new ServiceDegradationFault(name, errorRate, increasedLatency, probability);
}
}
}
public class FaultException extends RuntimeException {
public FaultException(String message) {
super(message);
}
public FaultException(String message, Throwable cause) {
super(message, cause);
}
}
2. Fault Injection Engine
package com.resilience.injector;
import com.resilience.fault.Fault;
import net.bytebuddy.ByteBuddy;
import net.bytebuddy.agent.ByteBuddyAgent;
import net.bytebuddy.dynamic.loading.ClassReloadingStrategy;
import net.bytebuddy.implementation.MethodDelegation;
import net.bytebuddy.matcher.ElementMatchers;
import java.lang.instrument.Instrumentation;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
public class FaultInjector {
private static final FaultInjector INSTANCE = new FaultInjector();
private final Map<String, List<Fault>> classFaults;
private final Map<String, List<Fault>> methodFaults;
private final Instrumentation instrumentation;
private boolean initialized = false;
private FaultInjector() {
this.classFaults = new ConcurrentHashMap<>();
this.methodFaults = new ConcurrentHashMap<>();
this.instrumentation = ByteBuddyAgent.install();
}
public static FaultInjector getInstance() {
return INSTANCE;
}
public synchronized void initialize() {
if (!initialized) {
// Initialize ByteBuddy agent
ByteBuddyAgent.install();
initialized = true;
}
}
public void injectClassFault(Class<?> targetClass, Fault fault) {
String className = targetClass.getName();
classFaults.computeIfAbsent(className, k -> new CopyOnWriteArrayList<>()).add(fault);
// Re-instrument the class
instrumentClass(targetClass);
}
public void injectMethodFault(Class<?> targetClass, String methodName, Fault fault) {
String key = targetClass.getName() + "#" + methodName;
methodFaults.computeIfAbsent(key, k -> new CopyOnWriteArrayList<>()).add(fault);
// Re-instrument the class
instrumentClass(targetClass);
}
public void removeClassFaults(Class<?> targetClass) {
classFaults.remove(targetClass.getName());
reinstrumentClass(targetClass);
}
public void removeMethodFaults(Class<?> targetClass, String methodName) {
String key = targetClass.getName() + "#" + methodName;
methodFaults.remove(key);
reinstrumentClass(targetClass);
}
public void clearAllFaults() {
classFaults.clear();
methodFaults.clear();
// Note: Cannot easily undo instrumentation, typically need JVM restart
}
private void instrumentClass(Class<?> targetClass) {
try {
new ByteBuddy()
.redefine(targetClass)
.method(ElementMatchers.any())
.intercept(MethodDelegation.to(FaultInterceptor.class))
.make()
.load(targetClass.getClassLoader(),
ClassReloadingStrategy.fromInstalledAgent());
} catch (Exception e) {
throw new RuntimeException("Failed to instrument class: " + targetClass.getName(), e);
}
}
private void reinstrumentClass(Class<?> targetClass) {
try {
new ByteBuddy()
.redefine(targetClass)
.make()
.load(targetClass.getClassLoader(),
ClassReloadingStrategy.fromInstalledAgent());
} catch (Exception e) {
throw new RuntimeException("Failed to reinstrument class: " + targetClass.getName(), e);
}
}
public static class FaultInterceptor {
public static Object intercept(@net.bytebuddy.implementation.bind.annotation.Origin
java.lang.reflect.Method method,
@net.bytebuddy.implementation.bind.annotation.AllArguments
Object[] args,
@net.bytebuddy.implementation.bind.annotation.SuperCall
java.util.concurrent.Callable<?> callable) throws Exception {
String className = method.getDeclaringClass().getName();
String methodName = method.getName();
String methodKey = className + "#" + methodName;
// Apply class-level faults
List<Fault> classFaults = INSTANCE.classFaults.getOrDefault(className,
Collections.emptyList());
for (Fault fault : classFaults) {
if (fault.isEnabled()) {
fault.apply();
}
}
// Apply method-level faults
List<Fault> methodFaults = INSTANCE.methodFaults.getOrDefault(methodKey,
Collections.emptyList());
for (Fault fault : methodFaults) {
if (fault.isEnabled()) {
fault.apply();
}
}
// Execute original method
return callable.call();
}
}
public FaultInjectorStats getStats() {
int totalClassFaults = classFaults.values().stream()
.mapToInt(List::size)
.sum();
int totalMethodFaults = methodFaults.values().stream()
.mapToInt(List::size)
.sum();
return new FaultInjectorStats(totalClassFaults, totalMethodFaults,
classFaults.size(), methodFaults.size());
}
public static class FaultInjectorStats {
private final int totalClassFaults;
private final int totalMethodFaults;
private final int instrumentedClasses;
private final int instrumentedMethods;
public FaultInjectorStats(int totalClassFaults, int totalMethodFaults,
int instrumentedClasses, int instrumentedMethods) {
this.totalClassFaults = totalClassFaults;
this.totalMethodFaults = totalMethodFaults;
this.instrumentedClasses = instrumentedClasses;
this.instrumentedMethods = instrumentedMethods;
}
// Getters
public int getTotalClassFaults() { return totalClassFaults; }
public int getTotalMethodFaults() { return totalMethodFaults; }
public int getInstrumentedClasses() { return instrumentedClasses; }
public int getInstrumentedMethods() { return instrumentedMethods; }
@Override
public String toString() {
return String.format(
"FaultInjectorStats[classes=%d, methods=%d, classFaults=%d, methodFaults=%d]",
instrumentedClasses, instrumentedMethods, totalClassFaults, totalMethodFaults
);
}
}
}
3. Resilience Scenarios
package com.resilience.scenario;
import com.resilience.fault.*;
import com.resilience.monitor.SystemMetrics;
import com.resilience.monitor.ScenarioMetrics;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicInteger;
public abstract class ResilienceScenario {
protected final String name;
protected final List<Fault> faults;
protected final ScenarioMetrics metrics;
protected final SystemMetrics systemMetrics;
protected volatile boolean running = false;
protected volatile ScenarioStatus status = ScenarioStatus.CREATED;
public ResilienceScenario(String name) {
this.name = name;
this.faults = new CopyOnWriteArrayList<>();
this.metrics = new ScenarioMetrics(name);
this.systemMetrics = new SystemMetrics();
}
public abstract void execute() throws ScenarioException;
public CompletableFuture<Void> executeAsync() {
return CompletableFuture.runAsync(() -> {
try {
execute();
} catch (ScenarioException e) {
throw new RuntimeException("Scenario execution failed", e);
}
});
}
public void addFault(Fault fault) {
faults.add(fault);
}
public void addFaults(List<Fault> faults) {
this.faults.addAll(faults);
}
public void clearFaults() {
faults.clear();
}
public ScenarioResult run() {
long startTime = System.currentTimeMillis();
status = ScenarioStatus.RUNNING;
running = true;
try {
systemMetrics.captureBaseline();
execute();
status = ScenarioStatus.COMPLETED;
return new ScenarioResult(name, true, "Scenario completed successfully",
System.currentTimeMillis() - startTime,
metrics, systemMetrics);
} catch (ScenarioException e) {
status = ScenarioStatus.FAILED;
return new ScenarioResult(name, false, e.getMessage(),
System.currentTimeMillis() - startTime,
metrics, systemMetrics);
} catch (Exception e) {
status = ScenarioStatus.ERROR;
return new ScenarioResult(name, false, "Unexpected error: " + e.getMessage(),
System.currentTimeMillis() - startTime,
metrics, systemMetrics);
} finally {
running = false;
systemMetrics.captureAfterScenario();
}
}
public void stop() {
running = false;
status = ScenarioStatus.STOPPED;
}
// Getters
public String getName() { return name; }
public List<Fault> getFaults() { return Collections.unmodifiableList(faults); }
public ScenarioMetrics getMetrics() { return metrics; }
public SystemMetrics getSystemMetrics() { return systemMetrics; }
public boolean isRunning() { return running; }
public ScenarioStatus getStatus() { return status; }
protected void validatePreconditions() throws ScenarioException {
// Check system resources
if (systemMetrics.getFreeMemory() < 100 * 1024 * 1024) { // 100MB
throw new ScenarioException("Insufficient memory to run scenario");
}
if (systemMetrics.getSystemLoad() > 0.8) {
throw new ScenarioException("System load too high to run scenario");
}
}
protected void applyFaults() {
for (Fault fault : faults) {
if (fault.isEnabled()) {
try {
fault.apply();
metrics.recordFaultApplied(fault.getName(), true);
} catch (FaultException e) {
metrics.recordFaultApplied(fault.getName(), false);
// Continue with other faults even if one fails
}
}
}
}
public enum ScenarioStatus {
CREATED, RUNNING, COMPLETED, FAILED, ERROR, STOPPED
}
}
public class LatencyScenario extends ResilienceScenario {
private final Duration testDuration;
private final int operationsPerSecond;
private final AtomicInteger successfulOperations = new AtomicInteger();
private final AtomicInteger failedOperations = new AtomicInteger();
public LatencyScenario(String name, Duration testDuration, int operationsPerSecond) {
super(name);
this.testDuration = testDuration;
this.operationsPerSecond = operationsPerSecond;
}
@Override
public void execute() throws ScenarioException {
validatePreconditions();
long startTime = System.currentTimeMillis();
long endTime = startTime + testDuration.toMillis();
long operationInterval = 1000 / operationsPerSecond;
System.out.printf("Starting latency scenario: %s for %s at %d ops/sec%n",
name, testDuration, operationsPerSecond);
while (System.currentTimeMillis() < endTime && running) {
long operationStart = System.currentTimeMillis();
try {
// Simulate operation with potential faults
applyFaults();
performOperation();
successfulOperations.incrementAndGet();
} catch (Exception e) {
failedOperations.incrementAndGet();
metrics.recordError(e);
}
// Maintain operation rate
long operationTime = System.currentTimeMillis() - operationStart;
long sleepTime = Math.max(0, operationInterval - operationTime);
if (sleepTime > 0) {
try {
Thread.sleep(sleepTime);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}
metrics.recordMetric("successful_operations", successfulOperations.get());
metrics.recordMetric("failed_operations", failedOperations.get());
metrics.recordMetric("total_operations", successfulOperations.get() + failedOperations.get());
double successRate = (double) successfulOperations.get() /
(successfulOperations.get() + failedOperations.get()) * 100;
metrics.recordMetric("success_rate", successRate);
}
protected void performOperation() {
// Simulate a typical operation - can be overridden by subclasses
try {
// Simulate some work
Thread.sleep(10);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
public class ChaosMonkeyScenario extends ResilienceScenario {
private final Random random = new Random();
private final Map<String, Double> faultProbabilities;
private final Duration scenarioDuration;
public ChaosMonkeyScenario(String name, Duration scenarioDuration) {
super(name);
this.scenarioDuration = scenarioDuration;
this.faultProbabilities = new HashMap<>();
// Default fault probabilities
faultProbabilities.put("latency", 0.3);
faultProbabilities.put("exception", 0.2);
faultProbabilities.put("memory", 0.1);
faultProbabilities.put("cpu", 0.1);
faultProbabilities.put("degradation", 0.3);
}
@Override
public void execute() throws ScenarioException {
validatePreconditions();
long startTime = System.currentTimeMillis();
long endTime = startTime + scenarioDuration.toMillis();
System.out.printf("Starting Chaos Monkey scenario: %s for %s%n", name, scenarioDuration);
while (System.currentTimeMillis() < endTime && running) {
// Randomly select and apply faults based on probabilities
applyRandomFaults();
// Wait between fault injections
try {
Thread.sleep(1000 + random.nextInt(2000)); // 1-3 seconds between faults
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}
private void applyRandomFaults() {
for (Map.Entry<String, Double> entry : faultProbabilities.entrySet()) {
if (random.nextDouble() < entry.getValue()) {
Fault fault = createRandomFault(entry.getKey());
if (fault != null) {
try {
fault.apply();
metrics.recordFaultApplied(fault.getName(), true);
} catch (FaultException e) {
metrics.recordFaultApplied(fault.getName(), false);
}
}
}
}
}
private Fault createRandomFault(String faultType) {
switch (faultType) {
case "latency":
return new LatencyFault.Builder()
.name("random-latency")
.latency(Duration.ofMillis(100 + random.nextInt(900))) // 100-1000ms
.probability(1.0)
.build();
case "exception":
return new ExceptionFault.Builder()
.name("random-exception")
.exceptionType(RuntimeException.class)
.message("Chaos Monkey injected exception")
.probability(1.0)
.build();
case "memory":
return new MemoryPressureFault.Builder()
.name("random-memory-pressure")
.memoryMB(50 + random.nextInt(150)) // 50-200MB
.duration(Duration.ofSeconds(1 + random.nextInt(4))) // 1-5 seconds
.probability(1.0)
.build();
case "cpu":
return new CPULoadFault.Builder()
.name("random-cpu-load")
.duration(Duration.ofSeconds(2 + random.nextInt(3))) // 2-5 seconds
.threadCount(1 + random.nextInt(3)) // 1-4 threads
.probability(1.0)
.build();
case "degradation":
return new ServiceDegradationFault.Builder()
.name("random-service-degradation")
.errorRate(0.1 + random.nextDouble() * 0.4) // 10-50% error rate
.increasedLatency(Duration.ofMillis(200 + random.nextInt(800))) // 200-1000ms
.probability(1.0)
.build();
default:
return null;
}
}
public void setFaultProbability(String faultType, double probability) {
faultProbabilities.put(faultType, Math.max(0, Math.min(1, probability)));
}
}
public class CircuitBreakerScenario extends ResilienceScenario {
private final int requestCount;
private final double failureThreshold;
private final Duration timeout;
public CircuitBreakerScenario(String name, int requestCount,
double failureThreshold, Duration timeout) {
super(name);
this.requestCount = requestCount;
this.failureThreshold = failureThreshold;
this.timeout = timeout;
}
@Override
public void execute() throws ScenarioException {
validatePreconditions();
int failures = 0;
int successes = 0;
long totalLatency = 0;
System.out.printf("Starting Circuit Breaker scenario: %s with %d requests%n",
name, requestCount);
for (int i = 0; i < requestCount && running; i++) {
long startTime = System.currentTimeMillis();
try {
boolean success = executeWithTimeout();
if (success) {
successes++;
} else {
failures++;
}
long latency = System.currentTimeMillis() - startTime;
totalLatency += latency;
metrics.recordLatency(latency);
} catch (Exception e) {
failures++;
metrics.recordError(e);
}
// Check if circuit should break
double failureRate = (double) failures / (failures + successes);
if (failureRate >= failureThreshold) {
metrics.recordEvent("circuit_broken",
String.format("Failure rate %.2f exceeded threshold %.2f",
failureRate, failureThreshold));
break;
}
}
metrics.recordMetric("total_requests", requestCount);
metrics.recordMetric("successful_requests", successes);
metrics.recordMetric("failed_requests", failures);
metrics.recordMetric("average_latency", totalLatency / (double) (successes + failures));
metrics.recordMetric("failure_rate", (double) failures / requestCount * 100);
}
private boolean executeWithTimeout() {
CompletableFuture<Boolean> future = CompletableFuture.supplyAsync(() -> {
try {
applyFaults();
return true;
} catch (Exception e) {
return false;
}
});
try {
return future.get(timeout.toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS);
} catch (java.util.concurrent.TimeoutException e) {
future.cancel(true);
return false;
} catch (Exception e) {
return false;
}
}
}
public class ScenarioResult {
private final String scenarioName;
private final boolean successful;
private final String message;
private final long durationMs;
private final ScenarioMetrics scenarioMetrics;
private final SystemMetrics systemMetrics;
private final Date timestamp;
public ScenarioResult(String scenarioName, boolean successful, String message,
long durationMs, ScenarioMetrics scenarioMetrics,
SystemMetrics systemMetrics) {
this.scenarioName = scenarioName;
this.successful = successful;
this.message = message;
this.durationMs = durationMs;
this.scenarioMetrics = scenarioMetrics;
this.systemMetrics = systemMetrics;
this.timestamp = new Date();
}
// Getters
public String getScenarioName() { return scenarioName; }
public boolean isSuccessful() { return successful; }
public String getMessage() { return message; }
public long getDurationMs() { return durationMs; }
public ScenarioMetrics getScenarioMetrics() { return scenarioMetrics; }
public SystemMetrics getSystemMetrics() { return systemMetrics; }
public Date getTimestamp() { return timestamp; }
public Map<String, Object> toMap() {
Map<String, Object> result = new HashMap<>();
result.put("scenarioName", scenarioName);
result.put("successful", successful);
result.put("message", message);
result.put("durationMs", durationMs);
result.put("timestamp", timestamp);
result.put("scenarioMetrics", scenarioMetrics.toMap());
result.put("systemMetrics", systemMetrics.toMap());
return result;
}
@Override
public String toString() {
return String.format("ScenarioResult[name=%s, success=%s, duration=%dms, message=%s]",
scenarioName, successful, durationMs, message);
}
}
public class ScenarioException extends Exception {
public ScenarioException(String message) {
super(message);
}
public ScenarioException(String message, Throwable cause) {
super(message, cause);
}
}
4. Monitoring & Metrics
package com.resilience.monitor;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.DistributionSummary;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.management.OperatingSystemMXBean;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
public class ScenarioMetrics {
private final String scenarioName;
private final Map<String, Object> metrics;
private final List<String> events;
private final List<Throwable> errors;
private final Map<String, AtomicLong> counters;
private final Map<String, List<Long>> latencySamples;
private final MeterRegistry meterRegistry;
public ScenarioMetrics(String scenarioName) {
this.scenarioName = scenarioName;
this.metrics = new ConcurrentHashMap<>();
this.events = new CopyOnWriteArrayList<>();
this.errors = new CopyOnWriteArrayList<>();
this.counters = new ConcurrentHashMap<>();
this.latencySamples = new ConcurrentHashMap<>();
this.meterRegistry = createMeterRegistry();
// Initialize default counters
counters.put("faults_applied", new AtomicLong());
counters.put("faults_failed", new AtomicLong());
counters.put("operations_completed", new AtomicLong());
counters.put("operations_failed", new AtomicLong());
}
public void recordMetric(String name, Object value) {
metrics.put(name, value);
// Also record to Micrometer if it's a numeric value
if (value instanceof Number) {
Gauge.builder("resilience.scenario." + name, () -> (Number) value)
.tag("scenario", scenarioName)
.register(meterRegistry);
}
}
public void recordEvent(String eventType, String description) {
String event = String.format("[%s] %s: %s",
new Date(), eventType, description);
events.add(event);
Counter.builder("resilience.scenario.events")
.tag("scenario", scenarioName)
.tag("type", eventType)
.register(meterRegistry)
.increment();
}
public void recordError(Throwable error) {
errors.add(error);
Counter.builder("resilience.scenario.errors")
.tag("scenario", scenarioName)
.tag("errorType", error.getClass().getSimpleName())
.register(meterRegistry)
.increment();
}
public void recordFaultApplied(String faultName, boolean successful) {
String counterName = successful ? "faults_applied" : "faults_failed";
counters.get(counterName).incrementAndGet();
Counter.builder("resilience.scenario.faults")
.tag("scenario", scenarioName)
.tag("fault", faultName)
.tag("successful", String.valueOf(successful))
.register(meterRegistry)
.increment();
}
public void recordLatency(long latencyMs) {
latencySamples.computeIfAbsent("operation_latency", k -> new CopyOnWriteArrayList<>())
.add(latencyMs);
Timer.builder("resilience.scenario.latency")
.tag("scenario", scenarioName)
.register(meterRegistry)
.record(latencyMs, TimeUnit.MILLISECONDS);
}
public void recordOperation(boolean successful) {
String counterName = successful ? "operations_completed" : "operations_failed";
counters.get(counterName).incrementAndGet();
Counter.builder("resilience.scenario.operations")
.tag("scenario", scenarioName)
.tag("successful", String.valueOf(successful))
.register(meterRegistry)
.increment();
}
public Map<String, Object> toMap() {
Map<String, Object> result = new HashMap<>();
result.putAll(metrics);
result.put("events", new ArrayList<>(events));
result.put("errorCount", errors.size());
// Add counter values
counters.forEach((name, counter) -> {
result.put(name, counter.get());
});
// Calculate latency statistics
latencySamples.forEach((name, samples) -> {
if (!samples.isEmpty()) {
result.put(name + "_count", samples.size());
result.put(name + "_min", samples.stream().min(Long::compare).orElse(0L));
result.put(name + "_max", samples.stream().max(Long::compare).orElse(0L));
result.put(name + "_avg", samples.stream().mapToLong(Long::longValue).average().orElse(0.0));
// Calculate percentiles
List<Long> sorted = new ArrayList<>(samples);
Collections.sort(sorted);
result.put(name + "_p50", calculatePercentile(sorted, 50));
result.put(name + "_p95", calculatePercentile(sorted, 95));
result.put(name + "_p99", calculatePercentile(sorted, 99));
}
});
return result;
}
private long calculatePercentile(List<Long> sortedValues, double percentile) {
if (sortedValues.isEmpty()) return 0L;
int index = (int) Math.ceil(percentile / 100.0 * sortedValues.size()) - 1;
index = Math.max(0, Math.min(index, sortedValues.size() - 1));
return sortedValues.get(index);
}
private MeterRegistry createMeterRegistry() {
// In a real implementation, this would be configured based on the monitoring system
return new io.micrometer.core.instrument.simple.SimpleMeterRegistry();
}
// Getters
public String getScenarioName() { return scenarioName; }
public Map<String, Object> getMetrics() { return new HashMap<>(metrics); }
public List<String> getEvents() { return new ArrayList<>(events); }
public List<Throwable> getErrors() { return new ArrayList<>(errors); }
public Map<String, AtomicLong> getCounters() { return new HashMap<>(counters); }
}
public class SystemMetrics {
private final OperatingSystemMXBean osBean;
private final MemoryMXBean memoryBean;
private final Runtime runtime;
private final Map<String, Object> baselineMetrics;
private final Map<String, Object> currentMetrics;
public SystemMetrics() {
this.osBean = ManagementFactory.getOperatingSystemMXBean();
this.memoryBean = ManagementFactory.getMemoryMXBean();
this.runtime = Runtime.getRuntime();
this.baselineMetrics = new HashMap<>();
this.currentMetrics = new HashMap<>();
}
public void captureBaseline() {
baselineMetrics.clear();
baselineMetrics.putAll(captureCurrentMetrics());
}
public void captureAfterScenario() {
currentMetrics.clear();
currentMetrics.putAll(captureCurrentMetrics());
}
private Map<String, Object> captureCurrentMetrics() {
Map<String, Object> metrics = new HashMap<>();
// Memory metrics
long freeMemory = runtime.freeMemory();
long totalMemory = runtime.totalMemory();
long maxMemory = runtime.maxMemory();
long usedMemory = totalMemory - freeMemory;
metrics.put("memory.free", freeMemory);
metrics.put("memory.total", totalMemory);
metrics.put("memory.max", maxMemory);
metrics.put("memory.used", usedMemory);
metrics.put("memory.usedPercentage", (double) usedMemory / totalMemory * 100);
// System load
if (osBean instanceof com.sun.management.OperatingSystemMXBean) {
com.sun.management.OperatingSystemMXBean sunOsBean =
(com.sun.management.OperatingSystemMXBean) osBean;
metrics.put("system.cpuLoad", sunOsBean.getSystemCpuLoad() * 100);
metrics.put("process.cpuLoad", sunOsBean.getProcessCpuLoad() * 100);
}
metrics.put("system.loadAverage", osBean.getSystemLoadAverage());
metrics.put("system.availableProcessors", osBean.getAvailableProcessors());
// Thread metrics
metrics.put("thread.count", Thread.activeCount());
// GC metrics
metrics.put("gc.collectionCount", memoryBean.getGarbageCollectorMXBeans().stream()
.mapToLong(gc -> gc.getCollectionCount())
.sum());
metrics.put("gc.collectionTime", memoryBean.getGarbageCollectorMXBeans().stream()
.mapToLong(gc -> gc.getCollectionTime())
.sum());
return metrics;
}
public Map<String, Object> getBaselineMetrics() {
return new HashMap<>(baselineMetrics);
}
public Map<String, Object> getCurrentMetrics() {
return new HashMap<>(currentMetrics);
}
public Map<String, Object> getMetricDeltas() {
Map<String, Object> deltas = new HashMap<>();
for (Map.Entry<String, Object> entry : currentMetrics.entrySet()) {
String key = entry.getKey();
Object current = entry.getValue();
Object baseline = baselineMetrics.get(key);
if (current instanceof Number && baseline instanceof Number) {
double delta = ((Number) current).doubleValue() - ((Number) baseline).doubleValue();
deltas.put(key + ".delta", delta);
if (((Number) baseline).doubleValue() != 0) {
double deltaPercent = (delta / ((Number) baseline).doubleValue()) * 100;
deltas.put(key + ".deltaPercent", deltaPercent);
}
}
}
return deltas;
}
// Convenience getters
public long getFreeMemory() {
return runtime.freeMemory();
}
public double getSystemLoad() {
return osBean.getSystemLoadAverage();
}
public int getAvailableProcessors() {
return osBean.getAvailableProcessors();
}
public Map<String, Object> toMap() {
Map<String, Object> result = new HashMap<>();
result.put("baseline", baselineMetrics);
result.put("current", currentMetrics);
result.put("deltas", getMetricDeltas());
return result;
}
}
5. JUnit 5 Integration
package com.resilience.junit;
import com.resilience.fault.*;
import com.resilience.scenario.*;
import org.junit.jupiter.api.extension.*;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.TestInstance;
import java.lang.annotation.*;
import java.util.*;
import java.util.concurrent.TimeUnit;
@ExtendWith(ResilienceTestExtension.class)
@Target(ElementType.TYPE)
@Retention(RetentionPolicy.RUNTIME)
public @interface ResilienceTest {
String config() default "";
boolean enableMonitoring() default true;
String reportFormat() default "html";
}
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
public @interface InjectFault {
String type();
String target() default "";
long latency() default 0;
String exception() default "java.lang.RuntimeException";
String message() default "";
double probability() default 1.0;
}
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
public @interface ChaosTest {
int duration() default 60;
TimeUnit timeUnit() default TimeUnit.SECONDS;
double latencyProbability() default 0.3;
double exceptionProbability() default 0.2;
}
public class ResilienceTestExtension implements
BeforeAllCallback, AfterAllCallback, BeforeEachCallback, AfterEachCallback {
private final FaultInjector faultInjector;
private final Map<String, ScenarioResult> testResults;
private SystemMetrics globalMetrics;
public ResilienceTestExtension() {
this.faultInjector = FaultInjector.getInstance();
this.testResults = new HashMap<>();
this.globalMetrics = new SystemMetrics();
}
@Override
public void beforeAll(ExtensionContext context) {
faultInjector.initialize();
globalMetrics.captureBaseline();
// Apply class-level fault injections
context.getTestClass().ifPresent(testClass -> {
ResilienceTest classAnnotation = testClass.getAnnotation(ResilienceTest.class);
if (classAnnotation != null) {
loadConfiguration(classAnnotation.config());
}
});
}
@Override
public void afterAll(ExtensionContext context) {
globalMetrics.captureAfterScenario();
generateReport(context);
faultInjector.clearAllFaults();
}
@Override
public void beforeEach(ExtensionContext context) {
context.getTestMethod().ifPresent(method -> {
// Apply method-level fault injections
InjectFault[] faultAnnotations = method.getAnnotationsByType(InjectFault.class);
for (InjectFault faultAnnotation : faultAnnotations) {
Fault fault = createFaultFromAnnotation(faultAnnotation);
if (faultAnnotation.target().isEmpty()) {
faultInjector.injectClassFault(method.getDeclaringClass(), fault);
} else {
faultInjector.injectMethodFault(method.getDeclaringClass(),
faultAnnotation.target(), fault);
}
}
});
}
@Override
public void afterEach(ExtensionContext context) {
// Clean up method-level injections
context.getTestMethod().ifPresent(method -> {
InjectFault[] faultAnnotations = method.getAnnotationsByType(InjectFault.class);
for (InjectFault faultAnnotation : faultAnnotations) {
if (faultAnnotation.target().isEmpty()) {
faultInjector.removeClassFaults(method.getDeclaringClass());
} else {
faultInjector.removeMethodFaults(method.getDeclaringClass(),
faultAnnotation.target());
}
}
});
}
private Fault createFaultFromAnnotation(InjectFault annotation) {
switch (annotation.type().toLowerCase()) {
case "latency":
return new LatencyFault.Builder()
.name("test-latency-" + System.currentTimeMillis())
.latency(annotation.latency(), TimeUnit.MILLISECONDS)
.probability(annotation.probability())
.build();
case "exception":
try {
Class<? extends Exception> exceptionClass =
(Class<? extends Exception>) Class.forName(annotation.exception());
return new ExceptionFault.Builder()
.name("test-exception-" + System.currentTimeMillis())
.exceptionType(exceptionClass)
.message(annotation.message())
.probability(annotation.probability())
.build();
} catch (ClassNotFoundException e) {
throw new RuntimeException("Exception class not found: " + annotation.exception(), e);
}
default:
throw new IllegalArgumentException("Unknown fault type: " + annotation.type());
}
}
private void loadConfiguration(String configPath) {
// Load configuration from file
// Implementation would use Typesafe Config or similar
}
private void generateReport(ExtensionContext context) {
ResilienceTestReport report = new ResilienceTestReport(
context.getDisplayName(),
testResults,
globalMetrics,
faultInjector.getStats()
);
report.generate();
}
public void recordTestResult(String testName, ScenarioResult result) {
testResults.put(testName, result);
}
}
// Example test class
@ResilienceTest(config = "resilience.conf", enableMonitoring = true)
public class ServiceResilienceTest {
@Test
@InjectFault(type = "latency", latency = 1000, probability = 0.5)
public void testServiceWithLatency() {
// This test will have 50% chance of 1 second latency injected
MyService service = new MyService();
long startTime = System.currentTimeMillis();
String result = service.processRequest("test");
long duration = System.currentTimeMillis() - startTime;
assertNotNull(result);
System.out.println("Request completed in " + duration + "ms");
}
@Test
@InjectFault(type = "exception", exception = "java.io.IOException",
message = "Simulated IO error", probability = 0.3)
public void testServiceWithExceptions() {
MyService service = new MyService();
try {
service.processRequest("test");
// If we get here, no exception was injected (70% chance)
System.out.println("No exception injected");
} catch (IOException e) {
// Exception was injected (30% chance)
assertEquals("Simulated IO error", e.getMessage());
System.out.println("Exception correctly injected");
}
}
@Test
@ChaosTest(duration = 30, timeUnit = TimeUnit.SECONDS,
latencyProbability = 0.4, exceptionProbability = 0.3)
public void testServiceUnderChaos() {
ChaosMonkeyScenario scenario = new ChaosMonkeyScenario(
"service-chaos-test", Duration.ofSeconds(30));
// Configure custom fault probabilities
scenario.setFaultProbability("latency", 0.4);
scenario.setFaultProbability("exception", 0.3);
scenario.setFaultProbability("memory", 0.1);
ScenarioResult result = scenario.run();
assertTrue(result.isSuccessful(), "Chaos test should complete successfully");
assertTrue(result.getScenarioMetrics().getCounters()
.get("faults_applied").get() > 0, "Should have applied some faults");
}
@Test
public void testCircuitBreakerPattern() {
CircuitBreakerScenario scenario = new CircuitBreakerScenario(
"circuit-breaker-test", 100, 0.5, Duration.ofSeconds(2));
// Add some faults to trigger circuit breaker
scenario.addFault(new ExceptionFault.Builder()
.name("circuit-breaker-exception")
.exceptionType(RuntimeException.class)
.probability(0.6)
.build());
ScenarioResult result = scenario.run();
assertTrue(result.isSuccessful());
double failureRate = (Double) result.getScenarioMetrics()
.getMetrics().get("failure_rate");
assertTrue(failureRate >= 50, "Failure rate should be at least 50%");
}
}
6. HTML Report Generator
package com.resilience.report;
import com.resilience.scenario.ScenarioResult;
import com.resilience.monitor.SystemMetrics;
import com.resilience.injector.FaultInjector;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.*;
public class ResilienceTestReport {
private final String testSuiteName;
private final Map<String, ScenarioResult> testResults;
private final SystemMetrics systemMetrics;
private final FaultInjector.FaultInjectorStats faultStats;
private final Date reportDate;
public ResilienceTestReport(String testSuiteName,
Map<String, ScenarioResult> testResults,
SystemMetrics systemMetrics,
FaultInjector.FaultInjectorStats faultStats) {
this.testSuiteName = testSuiteName;
this.testResults = new HashMap<>(testResults);
this.systemMetrics = systemMetrics;
this.faultStats = faultStats;
this.reportDate = new Date();
}
public void generate() {
generateHTMLReport();
generateJSONReport();
}
private void generateHTMLReport() {
String htmlContent = buildHTMLContent();
Path reportDir = Paths.get("reports", "resilience");
try {
Files.createDirectories(reportDir);
Path reportFile = reportDir.resolve("resilience-report-" +
new SimpleDateFormat("yyyyMMdd-HHmmss").format(reportDate) + ".html");
Files.writeString(reportFile, htmlContent);
System.out.println("HTML report generated: " + reportFile.toAbsolutePath());
} catch (IOException e) {
System.err.println("Failed to generate HTML report: " + e.getMessage());
}
}
private String buildHTMLContent() {
StringBuilder html = new StringBuilder();
html.append("""
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Resilience Test Report</title>
<style>
body { font-family: Arial, sans-serif; margin: 20px; background-color: #f5f5f5; }
.container { max-width: 1200px; margin: 0 auto; background: white; padding: 20px; border-radius: 8px; box-shadow: 0 2px 10px rgba(0,0,0,0.1); }
.header { background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; padding: 30px; border-radius: 8px; margin-bottom: 20px; }
.summary-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 20px; margin: 20px 0; }
.summary-card { background: #f8f9fa; padding: 20px; border-radius: 8px; text-align: center; border-left: 4px solid #3498db; }
.summary-value { font-size: 24px; font-weight: bold; color: #2c3e50; margin: 10px 0; }
.summary-label { color: #7f8c8d; font-size: 14px; }
.test-results { margin: 30px 0; }
.test-card { border: 1px solid #e0e0e0; border-radius: 8px; margin: 10px 0; padding: 15px; }
.test-success { border-left: 4px solid #2ecc71; }
.test-failure { border-left: 4px solid #e74c3c; }
.metrics-table { width: 100%; border-collapse: collapse; margin: 20px 0; }
.metrics-table th, .metrics-table td { padding: 12px; text-align: left; border-bottom: 1px solid #e0e0e0; }
.metrics-table th { background-color: #34495e; color: white; }
.chart-container { height: 300px; margin: 20px 0; background: white; padding: 15px; border-radius: 8px; border: 1px solid #e0e0e0; }
.success-rate { font-size: 18px; font-weight: bold; padding: 5px 10px; border-radius: 4px; }
.rate-high { background: #d4edda; color: #155724; }
.rate-medium { background: #fff3cd; color: #856404; }
.rate-low { background: #f8d7da; color: #721c24; }
</style>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
</head>
<body>
<div class="container">
""");
// Header
html.append("""
<div class="header">
<h1>Resilience Test Report</h1>
<p>Test Suite: """ + escapeHtml(testSuiteName) + """</p>
<p>Generated: """ + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(reportDate) + """</p>
</div>
""");
// Summary Statistics
html.append(buildSummarySection());
// Test Results
html.append(buildTestResultsSection());
// System Metrics
html.append(buildSystemMetricsSection());
// Fault Injection Statistics
html.append(buildFaultInjectionSection());
html.append("""
</div>
<script>
""");
// JavaScript for charts
html.append(buildChartsJavaScript());
html.append("""
</script>
</body>
</html>
""");
return html.toString();
}
private String buildSummarySection() {
long totalTests = testResults.size();
long successfulTests = testResults.values().stream()
.filter(ScenarioResult::isSuccessful)
.count();
long failedTests = totalTests - successfulTests;
double successRate = totalTests > 0 ? (double) successfulTests / totalTests * 100 : 0;
long totalDuration = testResults.values().stream()
.mapToLong(ScenarioResult::getDurationMs)
.sum();
long totalFaults = testResults.values().stream()
.mapToLong(result -> result.getScenarioMetrics().getCounters()
.getOrDefault("faults_applied", new java.util.concurrent.atomic.AtomicLong()).get())
.sum();
return """
<div class="summary-section">
<h2>Summary</h2>
<div class="summary-grid">
<div class="summary-card">
<div class="summary-value">""" + totalTests + """</div>
<div class="summary-label">Total Tests</div>
</div>
<div class="summary-card">
<div class="summary-value">""" + successfulTests + """</div>
<div class="summary-label">Successful Tests</div>
</div>
<div class="summary-card">
<div class="summary-value">""" + failedTests + """</div>
<div class="summary-label">Failed Tests</div>
</div>
<div class="summary-card">
<div class="summary-value">
<span class="success-rate """ + getSuccessRateClass(successRate) + """">
""" + String.format("%.1f%%", successRate) + """
</span>
</div>
<div class="summary-label">Success Rate</div>
</div>
<div class="summary-card">
<div class="summary-value">""" + totalFaults + """</div>
<div class="summary-label">Faults Injected</div>
</div>
<div class="summary-card">
<div class="summary-value">""" + String.format("%.1fs", totalDuration / 1000.0) + """</div>
<div class="summary-label">Total Duration</div>
</div>
</div>
</div>
""";
}
private String buildTestResultsSection() {
StringBuilder html = new StringBuilder();
html.append("""
<div class="test-results">
<h2>Test Results</h2>
""");
for (Map.Entry<String, ScenarioResult> entry : testResults.entrySet()) {
ScenarioResult result = entry.getValue();
String testClass = result.isSuccessful() ? "test-success" : "test-failure";
html.append("""
<div class="test-card """ + testClass + """">
<h3>""" + escapeHtml(entry.getKey()) + """</h3>
<div style="display: flex; justify-content: space-between; align-items: center;">
<div>
<strong>Status:</strong> """ + (result.isSuccessful() ? "✅ PASSED" : "❌ FAILED") + """<br>
<strong>Duration:</strong> """ + String.format("%.2fs", result.getDurationMs() / 1000.0) + """<br>
<strong>Message:</strong> """ + escapeHtml(result.getMessage()) + """
</div>
<div style="text-align: right;">
<strong>Faults Applied:</strong> """ +
result.getScenarioMetrics().getCounters()
.getOrDefault("faults_applied", new java.util.concurrent.atomic.AtomicLong()).get() + """<br>
<strong>Operations:</strong> """ +
result.getScenarioMetrics().getMetrics().getOrDefault("total_operations", 0) + """
</div>
</div>
</div>
""");
}
html.append("</div>");
return html.toString();
}
private String buildSystemMetricsSection() {
return """
<div class="system-metrics">
<h2>System Metrics</h2>
<div class="chart-container">
<canvas id="systemMetricsChart"></canvas>
</div>
<table class="metrics-table">
<thead>
<tr>
<th>Metric</th>
<th>Baseline</th>
<th>After Tests</th>
<th>Delta</th>
</tr>
</thead>
<tbody>
""" + buildSystemMetricsRows() + """
</tbody>
</table>
</div>
""";
}
private String buildSystemMetricsRows() {
StringBuilder rows = new StringBuilder();
Map<String, Object> baseline = systemMetrics.getBaselineMetrics();
Map<String, Object> current = systemMetrics.getCurrentMetrics();
Map<String, Object> deltas = systemMetrics.getMetricDeltas();
String[] metrics = {"memory.used", "system.cpuLoad", "system.loadAverage", "thread.count"};
for (String metric : metrics) {
Object baselineValue = baseline.get(metric);
Object currentValue = current.get(metric);
Object delta = deltas.get(metric + ".delta");
if (baselineValue != null && currentValue != null) {
rows.append("""
<tr>
<td>""" + metric + """</td>
<td>""" + formatMetricValue(metric, baselineValue) + """</td>
<td>""" + formatMetricValue(metric, currentValue) + """</td>
<td>""" + formatMetricValue(metric, delta) + """</td>
</tr>
""");
}
}
return rows.toString();
}
private String buildFaultInjectionSection() {
return """
<div class="fault-injection">
<h2>Fault Injection Statistics</h2>
<div class="summary-grid">
<div class="summary-card">
<div class="summary-value">""" + faultStats.getInstrumentedClasses() + """</div>
<div class="summary-label">Instrumented Classes</div>
</div>
<div class="summary-card">
<div class="summary-value">""" + faultStats.getInstrumentedMethods() + """</div>
<div class="summary-label">Instrumented Methods</div>
</div>
<div class="summary-card">
<div class="summary-value">""" + faultStats.getTotalClassFaults() + """</div>
<div class="summary-label">Class-level Faults</div>
</div>
<div class="summary-card">
<div class="summary-value">""" + faultStats.getTotalMethodFaults() + """</div>
<div class="summary-label">Method-level Faults</div>
</div>
</div>
</div>
""";
}
private String buildChartsJavaScript() {
return """
// System metrics chart
var ctx = document.getElementById('systemMetricsChart').getContext('2d');
var chart = new Chart(ctx, {
type: 'bar',
data: {
labels: ['Memory Used', 'CPU Load', 'System Load', 'Thread Count'],
datasets: [{
label: 'Baseline',
data: [""" + getMetricValueForChart("memory.used") + ", " +
getMetricValueForChart("system.cpuLoad") + ", " +
getMetricValueForChart("system.loadAverage") + ", " +
getMetricValueForChart("thread.count") + """],
backgroundColor: 'rgba(54, 162, 235, 0.5)'
}, {
label: 'After Tests',
data: [""" + getMetricValueForChart("memory.used", systemMetrics.getCurrentMetrics()) + ", " +
getMetricValueForChart("system.cpuLoad", systemMetrics.getCurrentMetrics()) + ", " +
getMetricValueForChart("system.loadAverage", systemMetrics.getCurrentMetrics()) + ", " +
getMetricValueForChart("thread.count", systemMetrics.getCurrentMetrics()) + """],
backgroundColor: 'rgba(255, 99, 132, 0.5)'
}]
},
options: {
responsive: true,
scales: {
y: {
beginAtZero: true
}
}
}
});
""";
}
// Helper methods
private String escapeHtml(String text) {
if (text == null) return "";
return text.replace("&", "&")
.replace("<", "<")
.replace(">", ">")
.replace("\"", """)
.replace("'", "'");
}
private String getSuccessRateClass(double rate) {
if (rate >= 80) return "rate-high";
if (rate >= 60) return "rate-medium";
return "rate-low";
}
private String formatMetricValue(String metric, Object value) {
if (value == null) return "N/A";
if (value instanceof Number) {
double num = ((Number) value).doubleValue();
if (metric.contains("memory")) {
return String.format("%.1f MB", num / (1024 * 1024));
} else if (metric.contains("cpuLoad") || metric.contains("Percentage")) {
return String.format("%.1f%%", num);
} else if (metric.contains("loadAverage")) {
return String.format("%.2f", num);
} else {
return String.format("%.0f", num);
}
}
return value.toString();
}
private String getMetricValueForChart(String metric) {
return getMetricValueForChart(metric, systemMetrics.getBaselineMetrics());
}
private String getMetricValueForChart(String metric, Map<String, Object> metrics) {
Object value = metrics.get(metric);
if (value instanceof Number) {
return String.valueOf(((Number) value).doubleValue());
}
return "0";
}
private void generateJSONReport() {
// Similar implementation for JSON report
// Useful for integration with CI/CD systems
}
}
Usage Examples
// Example 1: Basic fault injection
public class BasicResilienceTest {
@Test
public void testDatabaseConnectionResilience() {
// Create latency fault for database calls
LatencyFault dbLatency = new LatencyFault.Builder()
.name("database-latency")
.latency(Duration.ofSeconds(2))
.jitter(Duration.ofMillis(500))
.probability(0.7)
.build();
// Create exception fault for database errors
ExceptionFault dbException = new ExceptionFault.Builder()
.name("database-exception")
.exceptionType(SQLException.class)
.message("Database connection timeout")
.probability(0.3)
.build();
// Create scenario
LatencyScenario scenario = new LatencyScenario(
"database-resilience-test", Duration.ofMinutes(2), 10);
scenario.addFault(dbLatency);
scenario.addFault(dbException);
ScenarioResult result = scenario.run();
assertTrue(result.isSuccessful());
assertTrue(result.getScenarioMetrics().getCounters()
.get("faults_applied").get() > 0);
}
}
// Example 2: Service degradation testing
public class ServiceResilienceTest {
@Test
public void testPaymentServiceUnderLoad() {
ServiceDegradationFault paymentFault = new ServiceDegradationFault.Builder()
.name("payment-service-degradation")
.errorRate(0.4)
.increasedLatency(Duration.ofSeconds(3))
.probability(0.8)
.build();
CircuitBreakerScenario scenario = new CircuitBreakerScenario(
"payment-service-test", 200, 0.5, Duration.ofSeconds(5));
scenario.addFault(paymentFault);
ScenarioResult result = scenario.run();
// Verify circuit breaker behavior
double failureRate = (Double) result.getScenarioMetrics()
.getMetrics().get("failure_rate");
assertTrue(failureRate > 30, "Should have significant failure rate");
// Check that circuit breaker would have triggered
assertTrue(result.getScenarioMetrics().getEvents().stream()
.anyMatch(event -> event.contains("circuit_broken")));
}
}
// Example 3: Comprehensive chaos testing
public class ChaosEngineeringTest {
@Test
public void testFullSystemChaos() {
ChaosMonkeyScenario chaos = new ChaosMonkeyScenario(
"full-system-chaos", Duration.ofMinutes(5));
// Configure chaos probabilities
chaos.setFaultProbability("latency", 0.4);
chaos.setFaultProbability("exception", 0.3);
chaos.setFaultProbability("memory", 0.2);
chaos.setFaultProbability("cpu", 0.1);
chaos.setFaultProbability("degradation", 0.5);
ScenarioResult result = chaos.run();
// The test passes as long as the system doesn't crash completely
assertTrue(result.isSuccessful());
// Verify that various faults were applied
long totalFaults = result.getScenarioMetrics().getCounters()
.get("faults_applied").get();
assertTrue(totalFaults > 10, "Should have applied multiple fault types");
// Check system recovered
double memoryDelta = (Double) result.getSystemMetrics()
.getMetricDeltas()
.get("memory.used.deltaPercent");
assertTrue(memoryDelta < 50, "Memory usage should not increase excessively");
}
}
Features
✅ Comprehensive Fault Injection
- Latency injection with jitter
- Exception throwing
- Memory pressure simulation
- CPU load generation
- Service degradation patterns
- Network fault simulation