Overview
Alertmanager webhooks allow you to receive and process alerts from Prometheus Alertmanager. This enables custom alert handling, integration with other systems, and advanced alert processing logic.
Core Implementation
1. Alertmanager Webhook Data Model
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.*;
import java.time.Instant;
public class AlertmanagerWebhook {
public static class WebhookMessage {
private String version;
private String groupKey;
private int truncatedAlerts;
private String status;
private String receiver;
private GroupLabels groupLabels;
private CommonLabels commonLabels;
private CommonAnnotations commonAnnotations;
private String externalURL;
private List<Alert> alerts;
// Constructors
public WebhookMessage() {}
public WebhookMessage(String version, String status, List<Alert> alerts) {
this.version = version;
this.status = status;
this.alerts = alerts;
}
// Getters and setters
public String getVersion() { return version; }
public void setVersion(String version) { this.version = version; }
public String getGroupKey() { return groupKey; }
public void setGroupKey(String groupKey) { this.groupKey = groupKey; }
public int getTruncatedAlerts() { return truncatedAlerts; }
public void setTruncatedAlerts(int truncatedAlerts) { this.truncatedAlerts = truncatedAlerts; }
public String getStatus() { return status; }
public void setStatus(String status) { this.status = status; }
public String getReceiver() { return receiver; }
public void setReceiver(String receiver) { this.receiver = receiver; }
public GroupLabels getGroupLabels() { return groupLabels; }
public void setGroupLabels(GroupLabels groupLabels) { this.groupLabels = groupLabels; }
public CommonLabels getCommonLabels() { return commonLabels; }
public void setCommonLabels(CommonLabels commonLabels) { this.commonLabels = commonLabels; }
public CommonAnnotations getCommonAnnotations() { return commonAnnotations; }
public void setCommonAnnotations(CommonAnnotations commonAnnotations) { this.commonAnnotations = commonAnnotations; }
public String getExternalURL() { return externalURL; }
public void setExternalURL(String externalURL) { this.externalURL = externalURL; }
public List<Alert> getAlerts() { return alerts; }
public void setAlerts(List<Alert> alerts) { this.alerts = alerts; }
}
public static class GroupLabels {
private Map<String, String> labels = new HashMap<>();
public Map<String, String> getLabels() { return labels; }
public void setLabels(Map<String, String> labels) { this.labels = labels; }
public void addLabel(String key, String value) {
this.labels.put(key, value);
}
}
public static class CommonLabels {
private Map<String, String> labels = new HashMap<>();
public Map<String, String> getLabels() { return labels; }
public void setLabels(Map<String, String> labels) { this.labels = labels; }
public void addLabel(String key, String value) {
this.labels.put(key, value);
}
}
public static class CommonAnnotations {
private Map<String, String> annotations = new HashMap<>();
public Map<String, String> getAnnotations() { return annotations; }
public void setAnnotations(Map<String, String> annotations) { this.annotations = annotations; }
public void addAnnotation(String key, String value) {
this.annotations.put(key, value);
}
}
public static class Alert {
private String status;
private Map<String, String> labels;
private Map<String, String> annotations;
private Instant startsAt;
private Instant endsAt;
private String generatorURL;
private String fingerprint;
public Alert() {
this.labels = new HashMap<>();
this.annotations = new HashMap<>();
}
public Alert(String status, Map<String, String> labels, Map<String, String> annotations) {
this.status = status;
this.labels = labels != null ? labels : new HashMap<>();
this.annotations = annotations != null ? annotations : new HashMap<>();
}
// Getters and setters
public String getStatus() { return status; }
public void setStatus(String status) { this.status = status; }
public Map<String, String> getLabels() { return labels; }
public void setLabels(Map<String, String> labels) { this.labels = labels; }
public Map<String, String> getAnnotations() { return annotations; }
public void setAnnotations(Map<String, String> annotations) { this.annotations = annotations; }
public Instant getStartsAt() { return startsAt; }
public void setStartsAt(Instant startsAt) { this.startsAt = startsAt; }
public Instant getEndsAt() { return endsAt; }
public void setEndsAt(Instant endsAt) { this.endsAt = endsAt; }
public String getGeneratorURL() { return generatorURL; }
public void setGeneratorURL(String generatorURL) { this.generatorURL = generatorURL; }
public String getFingerprint() { return fingerprint; }
public void setFingerprint(String fingerprint) { this.fingerprint = fingerprint; }
// Helper methods
public void addLabel(String key, String value) {
this.labels.put(key, value);
}
public void addAnnotation(String key, String value) {
this.annotations.put(key, value);
}
public String getLabel(String key) {
return labels.get(key);
}
public String getAnnotation(String key) {
return annotations.get(key);
}
public boolean isFiring() {
return "firing".equals(status);
}
public boolean isResolved() {
return "resolved".equals(status);
}
public String getAlertName() {
return getLabel("alertname");
}
public String getSeverity() {
return getLabel("severity");
}
public String getInstance() {
return getLabel("instance");
}
public String getJob() {
return getLabel("job");
}
public String getSummary() {
return getAnnotation("summary");
}
public String getDescription() {
return getAnnotation("description");
}
}
}
2. Spring Boot Webhook Controller
import org.springframework.web.bind.annotation.*;
import org.springframework.http.ResponseEntity;
import org.springframework.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.validation.Valid;
@RestController
@RequestMapping("/webhook")
public class AlertmanagerWebhookController {
private static final Logger logger = LoggerFactory.getLogger(AlertmanagerWebhookController.class);
private final AlertProcessor alertProcessor;
private final NotificationService notificationService;
private final AlertStore alertStore;
public AlertmanagerWebhookController(AlertProcessor alertProcessor,
NotificationService notificationService,
AlertStore alertStore) {
this.alertProcessor = alertProcessor;
this.notificationService = notificationService;
this.alertStore = alertStore;
}
@PostMapping("/alerts")
public ResponseEntity<WebhookResponse> handleAlerts(
@Valid @RequestBody AlertmanagerWebhook.WebhookMessage webhookMessage) {
try {
logger.info("Received Alertmanager webhook. Status: {}, Alerts: {}",
webhookMessage.getStatus(), webhookMessage.getAlerts().size());
// Process the webhook message
ProcessingResult result = alertProcessor.processWebhook(webhookMessage);
// Store alerts for historical tracking
alertStore.storeAlerts(webhookMessage.getAlerts());
// Send notifications if needed
if (result.shouldNotify()) {
notificationService.sendNotifications(webhookMessage);
}
// Log processing results
logProcessingResults(webhookMessage, result);
return ResponseEntity.ok(new WebhookResponse(
"success",
String.format("Processed %d alerts", webhookMessage.getAlerts().size())
));
} catch (Exception e) {
logger.error("Failed to process Alertmanager webhook", e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
.body(new WebhookResponse("error", e.getMessage()));
}
}
@GetMapping("/health")
public ResponseEntity<Map<String, String>> healthCheck() {
return ResponseEntity.ok(Map.of("status", "healthy", "timestamp",
java.time.Instant.now().toString()));
}
@GetMapping("/alerts/active")
public ResponseEntity<List<StoredAlert>> getActiveAlerts() {
List<StoredAlert> activeAlerts = alertStore.getActiveAlerts();
return ResponseEntity.ok(activeAlerts);
}
@GetMapping("/alerts/history")
public ResponseEntity<List<StoredAlert>> getAlertHistory(
@RequestParam(defaultValue = "24h") String duration) {
List<StoredAlert> history = alertStore.getAlertHistory(duration);
return ResponseEntity.ok(history);
}
private void logProcessingResults(AlertmanagerWebhook.WebhookMessage message,
ProcessingResult result) {
Map<String, Long> alertCounts = message.getAlerts().stream()
.collect(Collectors.groupingBy(AlertmanagerWebhook.Alert::getStatus,
Collectors.counting()));
logger.info("Webhook processing completed - Firing: {}, Resolved: {}",
alertCounts.getOrDefault("firing", 0L),
alertCounts.getOrDefault("resolved", 0L));
// Log individual alerts for debugging
if (logger.isDebugEnabled()) {
message.getAlerts().forEach(alert -> {
logger.debug("Alert: {} - {} - {}",
alert.getAlertName(), alert.getStatus(), alert.getSummary());
});
}
}
public static class WebhookResponse {
private String status;
private String message;
private Instant timestamp;
public WebhookResponse(String status, String message) {
this.status = status;
this.message = message;
this.timestamp = Instant.now();
}
// Getters and setters
public String getStatus() { return status; }
public void setStatus(String status) { this.status = status; }
public String getMessage() { return message; }
public void setMessage(String message) { this.message = message; }
public Instant getTimestamp() { return timestamp; }
public void setTimestamp(Instant timestamp) { this.timestamp = timestamp; }
}
}
3. Alert Processing Service
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@Service
public class AlertProcessor {
private static final Logger logger = LoggerFactory.getLogger(AlertProcessor.class);
private final Map<String, AlertHandler> alertHandlers = new ConcurrentHashMap<>();
private final DeduplicationService deduplicationService;
private final AlertCorrelationService correlationService;
public AlertProcessor(DeduplicationService deduplicationService,
AlertCorrelationService correlationService) {
this.deduplicationService = deduplicationService;
this.correlationService = correlationService;
registerDefaultHandlers();
}
public ProcessingResult processWebhook(AlertmanagerWebhook.WebhookMessage webhookMessage) {
List<AlertmanagerWebhook.Alert> alerts = webhookMessage.getAlerts();
ProcessingResult result = new ProcessingResult();
for (AlertmanagerWebhook.Alert alert : alerts) {
try {
processSingleAlert(alert, result);
} catch (Exception e) {
logger.error("Failed to process alert: {}", alert.getFingerprint(), e);
result.addFailedAlert(alert, e.getMessage());
}
}
// Perform correlation analysis
correlationService.analyzeCorrelations(alerts);
return result;
}
private void processSingleAlert(AlertmanagerWebhook.Alert alert, ProcessingResult result) {
// Check for duplicates
if (deduplicationService.isDuplicate(alert)) {
logger.debug("Skipping duplicate alert: {}", alert.getFingerprint());
result.addSkippedAlert(alert, "duplicate");
return;
}
// Route to appropriate handler based on alert name or labels
AlertHandler handler = findHandler(alert);
if (handler != null) {
handler.handleAlert(alert);
result.addProcessedAlert(alert, handler.getName());
} else {
// Use default handler
getDefaultHandler().handleAlert(alert);
result.addProcessedAlert(alert, "default");
}
// Record for deduplication
deduplicationService.recordAlert(alert);
}
private AlertHandler findHandler(AlertmanagerWebhook.Alert alert) {
String alertName = alert.getAlertName();
if (alertName != null) {
return alertHandlers.get(alertName.toLowerCase());
}
return null;
}
private void registerDefaultHandlers() {
registerHandler("highcpu", new HighCpuAlertHandler());
registerHandler("highmemory", new HighMemoryAlertHandler());
registerHandler("servicedown", new ServiceDownAlertHandler());
registerHandler("latencyhigh", new LatencyAlertHandler());
registerHandler("errorratehigh", new ErrorRateAlertHandler());
registerHandler("diskspace", new DiskSpaceAlertHandler());
}
public void registerHandler(String alertName, AlertHandler handler) {
alertHandlers.put(alertName.toLowerCase(), handler);
logger.info("Registered alert handler for: {}", alertName);
}
public void unregisterHandler(String alertName) {
alertHandlers.remove(alertName.toLowerCase());
}
private AlertHandler getDefaultHandler() {
return new DefaultAlertHandler();
}
public static class ProcessingResult {
private final List<ProcessedAlert> processedAlerts = new ArrayList<>();
private final List<FailedAlert> failedAlerts = new ArrayList<>();
public void addProcessedAlert(AlertmanagerWebhook.Alert alert, String handler) {
processedAlerts.add(new ProcessedAlert(alert, handler, Instant.now()));
}
public void addSkippedAlert(AlertmanagerWebhook.Alert alert, String reason) {
processedAlerts.add(new ProcessedAlert(alert, "skipped", Instant.now(), reason));
}
public void addFailedAlert(AlertmanagerWebhook.Alert alert, String error) {
failedAlerts.add(new FailedAlert(alert, error, Instant.now()));
}
public boolean shouldNotify() {
return !processedAlerts.isEmpty() &&
processedAlerts.stream().anyMatch(pa -> !"skipped".equals(pa.getHandler()));
}
// Getters
public List<ProcessedAlert> getProcessedAlerts() { return processedAlerts; }
public List<FailedAlert> getFailedAlerts() { return failedAlerts; }
}
public static class ProcessedAlert {
private final AlertmanagerWebhook.Alert alert;
private final String handler;
private final Instant processedAt;
private final String reason;
public ProcessedAlert(AlertmanagerWebhook.Alert alert, String handler, Instant processedAt) {
this(alert, handler, processedAt, null);
}
public ProcessedAlert(AlertmanagerWebhook.Alert alert, String handler,
Instant processedAt, String reason) {
this.alert = alert;
this.handler = handler;
this.processedAt = processedAt;
this.reason = reason;
}
// Getters
public AlertmanagerWebhook.Alert getAlert() { return alert; }
public String getHandler() { return handler; }
public Instant getProcessedAt() { return processedAt; }
public String getReason() { return reason; }
}
public static class FailedAlert {
private final AlertmanagerWebhook.Alert alert;
private final String error;
private final Instant failedAt;
public FailedAlert(AlertmanagerWebhook.Alert alert, String error, Instant failedAt) {
this.alert = alert;
this.error = error;
this.failedAt = failedAt;
}
// Getters
public AlertmanagerWebhook.Alert getAlert() { return alert; }
public String getError() { return error; }
public Instant getFailedAt() { return failedAt; }
}
}
4. Alert Handlers
public interface AlertHandler {
String getName();
void handleAlert(AlertmanagerWebhook.Alert alert);
}
// Default handler for unhandled alerts
@Service
class DefaultAlertHandler implements AlertHandler {
private static final Logger logger = LoggerFactory.getLogger(DefaultAlertHandler.class);
@Override
public String getName() {
return "default";
}
@Override
public void handleAlert(AlertmanagerWebhook.Alert alert) {
logger.info("Default handler processing alert: {} - {}",
alert.getAlertName(), alert.getStatus());
// Log the alert but don't take any specific action
if (alert.isFiring()) {
logger.warn("FIRING ALERT - {}: {} - {}",
alert.getSeverity(), alert.getAlertName(), alert.getSummary());
} else if (alert.isResolved()) {
logger.info("RESOLVED ALERT - {}: {}", alert.getAlertName(), alert.getSummary());
}
}
}
// High CPU alert handler
@Service
class HighCpuAlertHandler implements AlertHandler {
private static final Logger logger = LoggerFactory.getLogger(HighCpuAlertHandler.class);
@Override
public String getName() {
return "high-cpu";
}
@Override
public void handleAlert(AlertmanagerWebhook.Alert alert) {
String instance = alert.getInstance();
String severity = alert.getSeverity();
if (alert.isFiring()) {
logger.warn("High CPU detected on {} (severity: {})", instance, severity);
// Add custom logic for high CPU alerts
if ("critical".equals(severity)) {
// Trigger automatic scaling or other mitigation
triggerMitigation(instance);
}
} else if (alert.isResolved()) {
logger.info("High CPU resolved on {}", instance);
// Clean up any mitigation actions
}
}
private void triggerMitigation(String instance) {
logger.info("Triggering mitigation for high CPU on {}", instance);
// Implementation: scale up, restart service, etc.
}
}
// Service down alert handler
@Service
class ServiceDownAlertHandler implements AlertHandler {
private static final Logger logger = LoggerFactory.getLogger(ServiceDownAlertHandler.class);
@Override
public String getName() {
return "service-down";
}
@Override
public void handleAlert(AlertmanagerWebhook.Alert alert) {
String service = alert.getLabel("service");
String instance = alert.getInstance();
if (alert.isFiring()) {
logger.error("Service down: {} on instance {}", service, instance);
// Trigger service restart or failover
if (shouldAutoRestart(service)) {
restartService(service, instance);
}
} else if (alert.isResolved()) {
logger.info("Service restored: {} on instance {}", service, instance);
}
}
private boolean shouldAutoRestart(String service) {
// Check if this service should be auto-restarted
return !service.contains("database"); // Don't auto-restart databases
}
private void restartService(String service, String instance) {
logger.info("Attempting to restart service {} on instance {}", service, instance);
// Implementation: call deployment system, Kubernetes API, etc.
}
}
// Latency alert handler
@Service
class LatencyAlertHandler implements AlertHandler {
private static final Logger logger = LoggerFactory.getLogger(LatencyAlertHandler.class);
@Override
public String getName() {
return "latency";
}
@Override
public void handleAlert(AlertmanagerWebhook.Alert alert) {
String service = alert.getLabel("service");
String endpoint = alert.getLabel("endpoint");
String latency = alert.getAnnotation("latency");
if (alert.isFiring()) {
logger.warn("High latency detected for {} endpoint {}: {}",
service, endpoint, latency);
// Could trigger circuit breaker or load shedding
} else if (alert.isResolved()) {
logger.info("Latency normalized for {} endpoint {}", service, endpoint);
}
}
}
// Error rate alert handler
@Service
class ErrorRateAlertHandler implements AlertHandler {
private static final Logger logger = LoggerFactory.getLogger(ErrorRateAlertHandler.class);
@Override
public String getName() {
return "error-rate";
}
@Override
public void handleAlert(AlertmanagerWebhook.Alert alert) {
String service = alert.getLabel("service");
String errorRate = alert.getAnnotation("error_rate");
if (alert.isFiring()) {
logger.error("High error rate detected for {}: {}", service, errorRate);
// Could trigger rollback or traffic shifting
} else if (alert.isResolved()) {
logger.info("Error rate normalized for {}", service);
}
}
}
// Disk space alert handler
@Service
class DiskSpaceAlertHandler implements AlertHandler {
private static final Logger logger = LoggerFactory.getLogger(DiskSpaceAlertHandler.class);
@Override
public String getName() {
return "disk-space";
}
@Override
public void handleAlert(AlertmanagerWebhook.Alert alert) {
String instance = alert.getInstance();
String device = alert.getLabel("device");
String usage = alert.getAnnotation("usage");
if (alert.isFiring()) {
logger.warn("Low disk space on {} device {}: {}", instance, device, usage);
// Trigger cleanup or expansion
triggerCleanup(instance, device);
} else if (alert.isResolved()) {
logger.info("Disk space recovered on {} device {}", instance, device);
}
}
private void triggerCleanup(String instance, String device) {
logger.info("Triggering disk cleanup on {} device {}", instance, device);
// Implementation: call cleanup scripts, expand volume, etc.
}
}
5. Deduplication Service
import org.springframework.stereotype.Service;
import java.util.concurrent.*;
@Service
public class DeduplicationService {
private static final Logger logger = LoggerFactory.getLogger(DeduplicationService.class);
// Cache to track recent alerts (expire after 1 hour)
private final Cache<String, AlertRecord> alertCache;
private final long deduplicationWindowMinutes = 60; // 1 hour
public DeduplicationService() {
this.alertCache = CacheBuilder.newBuilder()
.expireAfterWrite(deduplicationWindowMinutes, TimeUnit.MINUTES)
.maximumSize(10000)
.build();
}
public boolean isDuplicate(AlertmanagerWebhook.Alert alert) {
String fingerprint = alert.getFingerprint();
if (fingerprint == null) {
return false; // Can't deduplicate without fingerprint
}
AlertRecord existing = alertCache.getIfPresent(fingerprint);
if (existing == null) {
return false;
}
// Check if this is the same alert state
return existing.getStatus().equals(alert.getStatus()) &&
existing.getStartsAt().equals(alert.getStartsAt());
}
public void recordAlert(AlertmanagerWebhook.Alert alert) {
String fingerprint = alert.getFingerprint();
if (fingerprint != null) {
AlertRecord record = new AlertRecord(
alert.getStatus(),
alert.getStartsAt(),
Instant.now()
);
alertCache.put(fingerprint, record);
}
}
public void clearCache() {
alertCache.invalidateAll();
}
public long getCacheSize() {
return alertCache.size();
}
private static class AlertRecord {
private final String status;
private final Instant startsAt;
private final Instant recordedAt;
public AlertRecord(String status, Instant startsAt, Instant recordedAt) {
this.status = status;
this.startsAt = startsAt;
this.recordedAt = recordedAt;
}
// Getters
public String getStatus() { return status; }
public Instant getStartsAt() { return startsAt; }
public Instant getRecordedAt() { return recordedAt; }
}
}
6. Notification Service
import org.springframework.stereotype.Service;
import java.util.*;
@Service
public class NotificationService {
private static final Logger logger = LoggerFactory.getLogger(NotificationService.class);
private final List<NotificationChannel> channels = new ArrayList<>();
private final NotificationTemplateEngine templateEngine;
public NotificationService(NotificationTemplateEngine templateEngine) {
this.templateEngine = templateEngine;
initializeChannels();
}
private void initializeChannels() {
// Add different notification channels
channels.add(new SlackNotificationChannel());
channels.add(new EmailNotificationChannel());
channels.add(new PagerDutyNotificationChannel());
channels.add(new WebhookNotificationChannel());
}
public void sendNotifications(AlertmanagerWebhook.WebhookMessage webhookMessage) {
NotificationContext context = createNotificationContext(webhookMessage);
for (NotificationChannel channel : channels) {
if (channel.shouldSend(context)) {
try {
channel.sendNotification(context);
logger.info("Sent notification via {}", channel.getName());
} catch (Exception e) {
logger.error("Failed to send notification via {}", channel.getName(), e);
}
}
}
}
public void sendCriticalNotifications(AlertmanagerWebhook.WebhookMessage webhookMessage) {
// Filter for critical alerts only
List<AlertmanagerWebhook.Alert> criticalAlerts = webhookMessage.getAlerts().stream()
.filter(alert -> "critical".equals(alert.getSeverity()))
.collect(Collectors.toList());
if (!criticalAlerts.isEmpty()) {
AlertmanagerWebhook.WebhookMessage criticalMessage =
createFilteredMessage(webhookMessage, criticalAlerts);
sendNotifications(criticalMessage);
}
}
private NotificationContext createNotificationContext(AlertmanagerWebhook.WebhookMessage message) {
return new NotificationContext(message, templateEngine);
}
private AlertmanagerWebhook.WebhookMessage createFilteredMessage(
AlertmanagerWebhook.WebhookMessage original,
List<AlertmanagerWebhook.Alert> filteredAlerts) {
AlertmanagerWebhook.WebhookMessage filtered = new AlertmanagerWebhook.WebhookMessage();
filtered.setVersion(original.getVersion());
filtered.setStatus(original.getStatus());
filtered.setAlerts(filteredAlerts);
filtered.setGroupKey(original.getGroupKey());
filtered.setReceiver(original.getReceiver());
return filtered;
}
public void addChannel(NotificationChannel channel) {
channels.add(channel);
}
}
// Notification context
class NotificationContext {
private final AlertmanagerWebhook.WebhookMessage webhookMessage;
private final NotificationTemplateEngine templateEngine;
private final Map<String, Object> additionalData = new HashMap<>();
public NotificationContext(AlertmanagerWebhook.WebhookMessage webhookMessage,
NotificationTemplateEngine templateEngine) {
this.webhookMessage = webhookMessage;
this.templateEngine = templateEngine;
}
public AlertmanagerWebhook.WebhookMessage getWebhookMessage() { return webhookMessage; }
public NotificationTemplateEngine getTemplateEngine() { return templateEngine; }
public Map<String, Object> getAdditionalData() { return additionalData; }
public void addData(String key, Object value) {
additionalData.put(key, value);
}
}
// Notification channel interface
interface NotificationChannel {
String getName();
boolean shouldSend(NotificationContext context);
void sendNotification(NotificationContext context) throws Exception;
}
// Slack notification channel
class SlackNotificationChannel implements NotificationChannel {
private static final Logger logger = LoggerFactory.getLogger(SlackNotificationChannel.class);
@Override
public String getName() {
return "slack";
}
@Override
public boolean shouldSend(NotificationContext context) {
// Only send for firing alerts, not resolved ones
return context.getWebhookMessage().getAlerts().stream()
.anyMatch(AlertmanagerWebhook.Alert::isFiring);
}
@Override
public void sendNotification(NotificationContext context) throws Exception {
AlertmanagerWebhook.WebhookMessage message = context.getWebhookMessage();
// Build Slack message
SlackMessage slackMessage = buildSlackMessage(message);
// Send to Slack webhook
sendToSlack(slackMessage);
}
private SlackMessage buildSlackMessage(AlertmanagerWebhook.WebhookMessage message) {
SlackMessage slackMessage = new SlackMessage();
// Customize based on alert severity
String status = message.getStatus();
int alertCount = message.getAlerts().size();
if ("firing".equals(status)) {
slackMessage.setText("🚨 " + alertCount + " alerts are firing!");
slackMessage.setColor("danger");
} else {
slackMessage.setText("✅ " + alertCount + " alerts have been resolved");
slackMessage.setColor("good");
}
// Add alert details as attachments
message.getAlerts().forEach(alert -> {
slackMessage.addAttachment(buildAlertAttachment(alert));
});
return slackMessage;
}
private SlackMessage.Attachment buildAlertAttachment(AlertmanagerWebhook.Alert alert) {
SlackMessage.Attachment attachment = new SlackMessage.Attachment();
String status = alert.isFiring() ? "FIRING" : "RESOLVED";
String color = alert.isFiring() ? "danger" : "good";
attachment.setColor(color);
attachment.setTitle(alert.getAlertName() + " - " + status);
attachment.setText(alert.getDescription());
// Add fields
attachment.addField("Severity", alert.getSeverity(), true);
attachment.addField("Instance", alert.getInstance(), true);
attachment.addField("Job", alert.getJob(), true);
if (alert.getGeneratorURL() != null) {
attachment.addField("Details", alert.getGeneratorURL(), false);
}
return attachment;
}
private void sendToSlack(SlackMessage message) {
// Implementation for sending to Slack webhook
logger.info("Sending Slack notification: {}", message.getText());
// Use RestTemplate or WebClient to POST to Slack webhook URL
}
}
// Email notification channel
class EmailNotificationChannel implements NotificationChannel {
@Override
public String getName() {
return "email";
}
@Override
public boolean shouldSend(NotificationContext context) {
// Only send critical alerts via email
return context.getWebhookMessage().getAlerts().stream()
.anyMatch(alert -> "critical".equals(alert.getSeverity()));
}
@Override
public void sendNotification(NotificationContext context) throws Exception {
// Implementation for sending email
logger.info("Sending email notification");
}
}
// Slack message model
class SlackMessage {
private String text;
private String color;
private List<Attachment> attachments = new ArrayList<>();
// Getters and setters
public String getText() { return text; }
public void setText(String text) { this.text = text; }
public String getColor() { return color; }
public void setColor(String color) { this.color = color; }
public List<Attachment> getAttachments() { return attachments; }
public void setAttachments(List<Attachment> attachments) { this.attachments = attachments; }
public void addAttachment(Attachment attachment) {
this.attachments.add(attachment);
}
public static class Attachment {
private String title;
private String text;
private String color;
private List<Field> fields = new ArrayList<>();
// Getters and setters
public String getTitle() { return title; }
public void setTitle(String title) { this.title = title; }
public String getText() { return text; }
public void setText(String text) { this.text = text; }
public String getColor() { return color; }
public void setColor(String color) { this.color = color; }
public List<Field> getFields() { return fields; }
public void setFields(List<Field> fields) { this.fields = fields; }
public void addField(String title, String value, boolean shortField) {
fields.add(new Field(title, value, shortField));
}
}
public static class Field {
private String title;
private String value;
private boolean shortField;
public Field(String title, String value, boolean shortField) {
this.title = title;
this.value = value;
this.shortField = shortField;
}
// Getters and setters
public String getTitle() { return title; }
public void setTitle(String title) { this.title = title; }
public String getValue() { return value; }
public void setValue(String value) { this.value = value; }
public boolean isShortField() { return shortField; }
public void setShortField(boolean shortField) { this.shortField = shortField; }
}
}
7. Alert Storage and History
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@Service
public class AlertStore {
private static final Logger logger = LoggerFactory.getLogger(AlertStore.class);
// In-memory storage (in production, use a database)
private final Map<String, StoredAlert> activeAlerts = new ConcurrentHashMap<>();
private final List<StoredAlert> alertHistory = Collections.synchronizedList(new ArrayList<>());
private final int maxHistorySize = 10000;
public void storeAlerts(List<AlertmanagerWebhook.Alert> alerts) {
for (AlertmanagerWebhook.Alert alert : alerts) {
storeAlert(alert);
}
// Clean up old history if needed
cleanupHistory();
}
private void storeAlert(AlertmanagerWebhook.Alert alert) {
StoredAlert storedAlert = new StoredAlert(alert);
if (alert.isFiring()) {
activeAlerts.put(alert.getFingerprint(), storedAlert);
} else if (alert.isResolved()) {
activeAlerts.remove(alert.getFingerprint());
}
// Add to history
alertHistory.add(storedAlert);
}
public List<StoredAlert> getActiveAlerts() {
return new ArrayList<>(activeAlerts.values());
}
public List<StoredAlert> getAlertHistory(String duration) {
Instant cutoff = calculateCutoffTime(duration);
return alertHistory.stream()
.filter(alert -> alert.getReceivedAt().isAfter(cutoff))
.sorted(Comparator.comparing(StoredAlert::getReceivedAt).reversed())
.collect(Collectors.toList());
}
public List<StoredAlert> getAlertsBySeverity(String severity) {
return alertHistory.stream()
.filter(alert -> severity.equals(alert.getSeverity()))
.collect(Collectors.toList());
}
public List<StoredAlert> getAlertsByName(String alertName) {
return alertHistory.stream()
.filter(alert -> alertName.equals(alert.getAlertName()))
.collect(Collectors.toList());
}
private Instant calculateCutoffTime(String duration) {
// Parse duration like "24h", "7d", "30d"
if (duration.endsWith("h")) {
long hours = Long.parseLong(duration.substring(0, duration.length() - 1));
return Instant.now().minusSeconds(hours * 3600);
} else if (duration.endsWith("d")) {
long days = Long.parseLong(duration.substring(0, duration.length() - 1));
return Instant.now().minusSeconds(days * 86400);
} else {
// Default to 24 hours
return Instant.now().minusSeconds(86400);
}
}
private void cleanupHistory() {
if (alertHistory.size() > maxHistorySize) {
// Remove oldest entries
alertHistory.sort(Comparator.comparing(StoredAlert::getReceivedAt));
int removeCount = alertHistory.size() - maxHistorySize;
alertHistory.subList(0, removeCount).clear();
logger.info("Cleaned up {} old alert records", removeCount);
}
}
public void clearHistory() {
alertHistory.clear();
activeAlerts.clear();
}
}
class StoredAlert {
private final String fingerprint;
private final String alertName;
private final String status;
private final String severity;
private final String instance;
private final String job;
private final String summary;
private final String description;
private final Map<String, String> labels;
private final Map<String, String> annotations;
private final Instant startsAt;
private final Instant endsAt;
private final Instant receivedAt;
public StoredAlert(AlertmanagerWebhook.Alert alert) {
this.fingerprint = alert.getFingerprint();
this.alertName = alert.getAlertName();
this.status = alert.getStatus();
this.severity = alert.getSeverity();
this.instance = alert.getInstance();
this.job = alert.getJob();
this.summary = alert.getSummary();
this.description = alert.getDescription();
this.labels = new HashMap<>(alert.getLabels());
this.annotations = new HashMap<>(alert.getAnnotations());
this.startsAt = alert.getStartsAt();
this.endsAt = alert.getEndsAt();
this.receivedAt = Instant.now();
}
// Getters
public String getFingerprint() { return fingerprint; }
public String getAlertName() { return alertName; }
public String getStatus() { return status; }
public String getSeverity() { return severity; }
public String getInstance() { return instance; }
public String getJob() { return job; }
public String getSummary() { return summary; }
public String getDescription() { return description; }
public Map<String, String> getLabels() { return labels; }
public Map<String, String> getAnnotations() { return annotations; }
public Instant getStartsAt() { return startsAt; }
public Instant getEndsAt() { return endsAt; }
public Instant getReceivedAt() { return receivedAt; }
public boolean isFiring() {
return "firing".equals(status);
}
public boolean isResolved() {
return "resolved".equals(status);
}
}
8. Configuration
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.client.RestTemplate;
@Configuration
public class WebhookConfig {
@Bean
public RestTemplate restTemplate() {
return new RestTemplate();
}
@Bean
public AlertProcessor alertProcessor(DeduplicationService deduplicationService,
AlertCorrelationService correlationService) {
return new AlertProcessor(deduplicationService, correlationService);
}
@Bean
public DeduplicationService deduplicationService() {
return new DeduplicationService();
}
@Bean
public AlertStore alertStore() {
return new AlertStore();
}
}
// application.properties
/*
# Webhook server configuration
server.port=8080
server.servlet.context-path=/webhook
# Alert processing
webhook.alert.deduplication-window-minutes=60
webhook.alert.max-history-size=10000
# Notification channels
webhook.notification.slack.enabled=true
webhook.notification.slack.webhook-url=${SLACK_WEBHOOK_URL}
webhook.notification.email.enabled=false
# Security (optional)
webhook.security.enabled=true
webhook.security.api-key=${WEBHOOK_API_KEY}
*/
9. Security Configuration
import org.springframework.context.annotation.Configuration;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
import org.springframework.security.config.http.SessionCreationPolicy;
@Configuration
public class SecurityConfig extends WebSecurityConfigurerAdapter {
@Override
protected void configure(HttpSecurity http) throws Exception {
http
.csrf().disable()
.sessionManagement().sessionCreationPolicy(SessionCreationPolicy.STATELESS)
.and()
.authorizeRequests()
.antMatchers("/webhook/health").permitAll()
.antMatchers("/webhook/alerts").hasRole("WEBHOOK")
.anyRequest().authenticated()
.and()
.httpBasic(); // Or use API key authentication
}
}
Usage Example
@SpringBootApplication
public class AlertmanagerWebhookApplication {
public static void main(String[] args) {
SpringApplication.run(AlertmanagerWebhookApplication.class, args);
}
@Bean
public CommandLineRunner demo(AlertProcessor alertProcessor) {
return args -> {
// Register custom alert handlers
alertProcessor.registerHandler("customalert", new CustomAlertHandler());
};
}
}
// Custom alert handler example
class CustomAlertHandler implements AlertHandler {
@Override
public String getName() {
return "custom-alert";
}
@Override
public void handleAlert(AlertmanagerWebhook.Alert alert) {
// Custom logic for handling specific alerts
System.out.println("Custom handler processing: " + alert.getAlertName());
}
}
This implementation provides a complete Alertmanager webhook receiver in Java with features for alert processing, deduplication, notification routing, and historical tracking.