This comprehensive guide covers implementing AI-powered log analysis in Java applications, including log parsing, anomaly detection, pattern recognition, and intelligent alerting using machine learning and natural language processing.

Architecture Overview

Log Sources → Log Ingestion → Preprocessing → AI Analysis
→ Application Logs → Parsing/Normalization → Anomaly Detection
→ System Logs → Feature Extraction → Pattern Recognition
→ Network Logs → Vectorization → Predictive Analytics
→ Storage → Elasticsearch/DB → Visualization/Dashboards

Prerequisites and Setup

Maven Dependencies

<properties>
<dl4j.version>1.0.0-M2.1</dl4j.version>
<opennlp.version>2.2.0</opennlp.version>
<tensorflow.version>0.4.1</tensorflow.version>
<spring-boot.version>3.1.0</spring-boot.version>
<elasticsearch.version>8.8.0</elasticsearch.version>
</properties>
<dependencies>
<!-- Deep Learning -->
<dependency>
<groupId>org.deeplearning4j</groupId>
<artifactId>deeplearning4j-core</artifactId>
<version>${dl4j.version}</version>
</dependency>
<dependency>
<groupId>org.nd4j</groupId>
<artifactId>nd4j-native-platform</artifactId>
<version>${dl4j.version}</version>
</dependency>
<!-- NLP -->
<dependency>
<groupId>org.apache.opennlp</groupId>
<artifactId>opennlp-tools</artifactId>
<version>${opennlp.version}</version>
</dependency>
<!-- TensorFlow -->
<dependency>
<groupId>org.tensorflow</groupId>
<artifactId>tensorflow-core-api</artifactId>
<version>${tensorflow.version}</version>
</dependency>
<!-- Spring Boot -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<!-- Utilities -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.15.2</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
<version>3.6.1</version>
</dependency>
</dependencies>

Configuration Properties

# AI Model Configuration
ai.log-analysis.model-path=./models/
ai.log-analysis.anomaly-threshold=0.8
ai.log-analysis.similarity-threshold=0.7
ai.log-analysis.training-enabled=true
# Log Processing
log.ingestion.batch-size=1000
log.ingestion.buffer-timeout=5000
log.ingestion.max-queue-size=10000
# Elasticsearch
spring.elasticsearch.uris=http://localhost:9200
spring.elasticsearch.username=elastic
spring.elasticsearch.password=password
# Feature Extraction
log.features.vector-size=100
log.features.window-size=10
log.features.min-word-frequency=5
# Anomaly Detection
anomaly.detection.sensitivity=0.75
anomaly.detection.training-interval=3600000

Core Data Models

1. Log Entry Models

package com.yourapp.loganalysis.model;
import java.time.LocalDateTime;
import java.util.Map;
public class LogEntry {
private String id;
private LocalDateTime timestamp;
private String level;
private String logger;
private String message;
private String thread;
private Map<String, String> mdc;
private String exception;
private String stackTrace;
private String application;
private String environment;
private String host;
private String source;
// AI Analysis Fields
private Double anomalyScore;
private String patternId;
private String category;
private Map<String, Object> features;
private Boolean isAnomaly;
// Constructors
public LogEntry() {}
public LogEntry(String message, String level, LocalDateTime timestamp) {
this.message = message;
this.level = level;
this.timestamp = timestamp;
}
// Getters and Setters
public String getId() { return id; }
public void setId(String id) { this.id = id; }
public LocalDateTime getTimestamp() { return timestamp; }
public void setTimestamp(LocalDateTime timestamp) { this.timestamp = timestamp; }
public String getLevel() { return level; }
public void setLevel(String level) { this.level = level; }
public String getLogger() { return logger; }
public void setLogger(String logger) { this.logger = logger; }
public String getMessage() { return message; }
public void setMessage(String message) { this.message = message; }
public String getThread() { return thread; }
public void setThread(String thread) { this.thread = thread; }
public Map<String, String> getMdc() { return mdc; }
public void setMdc(Map<String, String> mdc) { this.mdc = mdc; }
public String getException() { return exception; }
public void setException(String exception) { this.exception = exception; }
public String getStackTrace() { return stackTrace; }
public void setStackTrace(String stackTrace) { this.stackTrace = stackTrace; }
public String getApplication() { return application; }
public void setApplication(String application) { this.application = application; }
public String getEnvironment() { return environment; }
public void setEnvironment(String environment) { this.environment = environment; }
public String getHost() { return host; }
public void setHost(String host) { this.host = host; }
public String getSource() { return source; }
public void setSource(String source) { this.source = source; }
public Double getAnomalyScore() { return anomalyScore; }
public void setAnomalyScore(Double anomalyScore) { this.anomalyScore = anomalyScore; }
public String getPatternId() { return patternId; }
public void setPatternId(String patternId) { this.patternId = patternId; }
public String getCategory() { return category; }
public void setCategory(String category) { this.category = category; }
public Map<String, Object> getFeatures() { return features; }
public void setFeatures(Map<String, Object> features) { this.features = features; }
public Boolean getIsAnomaly() { return isAnomaly; }
public void setIsAnomaly(Boolean isAnomaly) { this.isAnomaly = isAnomaly; }
}
public class LogPattern {
private String patternId;
private String pattern;
private String regex;
private String category;
private Integer frequency;
private Double confidence;
private LocalDateTime firstSeen;
private LocalDateTime lastSeen;
private Map<String, Object> metadata;
// Getters and Setters
public String getPatternId() { return patternId; }
public void setPatternId(String patternId) { this.patternId = patternId; }
public String getPattern() { return pattern; }
public void setPattern(String pattern) { this.pattern = pattern; }
public String getRegex() { return regex; }
public void setRegex(String regex) { this.regex = regex; }
public String getCategory() { return category; }
public void setCategory(String category) { this.category = category; }
public Integer getFrequency() { return frequency; }
public void setFrequency(Integer frequency) { this.frequency = frequency; }
public Double getConfidence() { return confidence; }
public void setConfidence(Double confidence) { this.confidence = confidence; }
public LocalDateTime getFirstSeen() { return firstSeen; }
public void setFirstSeen(LocalDateTime firstSeen) { this.firstSeen = firstSeen; }
public LocalDateTime getLastSeen() { return lastSeen; }
public void setLastSeen(LocalDateTime lastSeen) { this.lastSeen = lastSeen; }
public Map<String, Object> getMetadata() { return metadata; }
public void setMetadata(Map<String, Object> metadata) { this.metadata = metadata; }
}
public class AnomalyDetectionResult {
private String logEntryId;
private Double anomalyScore;
private String anomalyType;
private String explanation;
private LocalDateTime detectedAt;
private Map<String, Object> context;
private Boolean isFalsePositive;
private String suggestedAction;
// Getters and Setters
public String getLogEntryId() { return logEntryId; }
public void setLogEntryId(String logEntryId) { this.logEntryId = logEntryId; }
public Double getAnomalyScore() { return anomalyScore; }
public void setAnomalyScore(Double anomalyScore) { this.anomalyScore = anomalyScore; }
public String getAnomalyType() { return anomalyType; }
public void setAnomalyType(String anomalyType) { this.anomalyType = anomalyType; }
public String getExplanation() { return explanation; }
public void setExplanation(String explanation) { this.explanation = explanation; }
public LocalDateTime getDetectedAt() { return detectedAt; }
public void setDetectedAt(LocalDateTime detectedAt) { this.detectedAt = detectedAt; }
public Map<String, Object> getContext() { return context; }
public void setContext(Map<String, Object> context) { this.context = context; }
public Boolean getIsFalsePositive() { return isFalsePositive; }
public void setIsFalsePositive(Boolean isFalsePositive) { this.isFalsePositive = isFalsePositive; }
public String getSuggestedAction() { return suggestedAction; }
public void setSuggestedAction(String suggestedAction) { this.suggestedAction = suggestedAction; }
}

Log Preprocessing & Feature Extraction

2. Log Parser and Normalizer

package com.yourapp.loganalysis.preprocessing;
import com.yourapp.loganalysis.model.LogEntry;
import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import org.springframework.stereotype.Component;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.regex.Pattern;
@Component
public class LogParser {
private final Tokenizer tokenizer;
private final List<LogPattern> knownPatterns;
private final Map<String, Pattern> compiledPatterns;
public LogParser() throws IOException {
// Initialize NLP tokenizer
try (InputStream modelIn = new FileInputStream("models/en-token.bin")) {
TokenizerModel model = new TokenizerModel(modelIn);
this.tokenizer = new TokenizerME(model);
}
this.knownPatterns = new ArrayList<>();
this.compiledPatterns = new HashMap<>();
initializeKnownPatterns();
}
/**
* Parse raw log line into structured LogEntry
*/
public LogEntry parseLogLine(String rawLog, String source) {
LogEntry entry = new LogEntry();
entry.setSource(source);
entry.setTimestamp(extractTimestamp(rawLog));
entry.setLevel(extractLogLevel(rawLog));
entry.setMessage(extractMessage(rawLog));
entry.setLogger(extractLogger(rawLog));
entry.setThread(extractThread(rawLog));
entry.setMdc(extractMdc(rawLog));
// Apply pattern matching
applyPatternMatching(entry);
return entry;
}
/**
* Extract timestamp from log line
*/
private LocalDateTime extractTimestamp(String logLine) {
// Common timestamp patterns
String[] timestampPatterns = {
"\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}",
"\\d{2}/\\d{2}/\\d{4} \\d{2}:\\d{2}:\\d{2}",
"\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z"
};
for (String pattern : timestampPatterns) {
java.util.regex.Pattern p = java.util.regex.Pattern.compile(pattern);
java.util.regex.Matcher m = p.matcher(logLine);
if (m.find()) {
String timestampStr = m.group();
try {
return LocalDateTime.parse(timestampStr, 
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"));
} catch (Exception e) {
// Try other formats...
}
}
}
return LocalDateTime.now();
}
/**
* Extract log level from log line
*/
private String extractLogLevel(String logLine) {
String[] levels = {"ERROR", "WARN", "INFO", "DEBUG", "TRACE"};
for (String level : levels) {
if (logLine.contains(level)) {
return level;
}
}
return "INFO";
}
/**
* Extract the actual log message
*/
private String extractMessage(String logLine) {
// Remove timestamp, level, logger, etc.
String message = logLine;
// Remove timestamp pattern
message = message.replaceAll("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}", "");
// Remove log level
message = message.replaceAll("\\b(ERROR|WARN|INFO|DEBUG|TRACE)\\b", "");
// Remove logger name in brackets
message = message.replaceAll("\\[.*?\\]", "");
return message.trim();
}
/**
* Extract logger name
*/
private String extractLogger(String logLine) {
java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("\\[(.*?)\\]");
java.util.regex.Matcher matcher = pattern.matcher(logLine);
if (matcher.find()) {
return matcher.group(1);
}
return "unknown";
}
/**
* Extract thread name
*/
private String extractThread(String logLine) {
java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("\\[([^\\]]*?)\\]");
java.util.regex.Matcher matcher = pattern.matcher(logLine);
List<String> brackets = new ArrayList<>();
while (matcher.find()) {
brackets.add(matcher.group(1));
}
return brackets.size() > 1 ? brackets.get(1) : "main";
}
/**
* Extract MDC context
*/
private Map<String, String> extractMdc(String logLine) {
Map<String, String> mdc = new HashMap<>();
// Extract key-value pairs from MDC
java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("\\{([^}]*)\\}");
java.util.regex.Matcher matcher = pattern.matcher(logLine);
while (matcher.find()) {
String mdcContent = matcher.group(1);
String[] pairs = mdcContent.split(",");
for (String pair : pairs) {
String[] keyValue = pair.split("=");
if (keyValue.length == 2) {
mdc.put(keyValue[0].trim(), keyValue[1].trim());
}
}
}
return mdc;
}
/**
* Apply pattern matching to categorize log entry
*/
private void applyPatternMatching(LogEntry entry) {
for (LogPattern pattern : knownPatterns) {
if (entry.getMessage().matches(pattern.getRegex())) {
entry.setPatternId(pattern.getPatternId());
entry.setCategory(pattern.getCategory());
break;
}
}
}
/**
* Tokenize log message for NLP processing
*/
public String[] tokenizeMessage(String message) {
return tokenizer.tokenize(message);
}
/**
* Normalize log message by removing variables
*/
public String normalizeMessage(String message) {
// Replace numbers with <NUM>
message = message.replaceAll("\\b\\d+\\b", "<NUM>");
// Replace UUIDs with <UUID>
message = message.replaceAll("[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}", "<UUID>");
// Replace file paths with <PATH>
message = message.replaceAll("/[\\w./-]+", "<PATH>");
// Replace IP addresses with <IP>
message = message.replaceAll("\\b(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\b", "<IP>");
// Replace email addresses with <EMAIL>
message = message.replaceAll("\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b", "<EMAIL>");
return message;
}
private void initializeKnownPatterns() {
// Database connection patterns
knownPatterns.add(createPattern("DB_CONNECTION", 
".*(connection|connect|database).*(failed|error|timeout).*", "DATABASE"));
// Authentication patterns
knownPatterns.add(createPattern("AUTH_FAILED", 
".*(authentication|auth|login).*(failed|invalid|denied).*", "SECURITY"));
// Memory patterns
knownPatterns.add(createPattern("MEMORY_ERROR", 
".*(memory|heap|outofmemory).*(error|full|exhausted).*", "RESOURCE"));
// Network patterns
knownPatterns.add(createPattern("NETWORK_ERROR", 
".*(connection|socket|network).*(timeout|refused|reset).*", "NETWORK"));
}
private LogPattern createPattern(String id, String regex, String category) {
LogPattern pattern = new LogPattern();
pattern.setPatternId(id);
pattern.setRegex(regex);
pattern.setCategory(category);
pattern.setConfidence(0.9);
return pattern;
}
}

3. Feature Extractor

package com.yourapp.loganalysis.preprocessing;
import com.yourapp.loganalysis.model.LogEntry;
import org.apache.commons.math3.stat.StatUtils;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.stream.Collectors;
@Component
public class FeatureExtractor {
private final LogParser logParser;
public FeatureExtractor(LogParser logParser) {
this.logParser = logParser;
}
/**
* Extract features from log entry for ML models
*/
public Map<String, Object> extractFeatures(LogEntry entry) {
Map<String, Object> features = new HashMap<>();
// Text-based features
features.putAll(extractTextFeatures(entry.getMessage()));
// Temporal features
features.putAll(extractTemporalFeatures(entry));
// Structural features
features.putAll(extractStructuralFeatures(entry));
// Contextual features
features.putAll(extractContextualFeatures(entry));
return features;
}
/**
* Extract text-based features from log message
*/
private Map<String, Object> extractTextFeatures(String message) {
Map<String, Object> features = new HashMap<>();
String[] tokens = logParser.tokenizeMessage(message);
String normalizedMessage = logParser.normalizeMessage(message);
features.put("message_length", message.length());
features.put("token_count", tokens.length);
features.put("word_count", countWords(message));
features.put("has_exception", containsException(message));
features.put("has_stack_trace", containsStackTrace(message));
features.put("has_url", containsUrl(message));
features.put("has_number", containsNumber(message));
features.put("capital_ratio", calculateCapitalRatio(message));
features.put("special_char_ratio", calculateSpecialCharRatio(message));
features.put("unique_token_ratio", calculateUniqueTokenRatio(tokens));
// NLP features
features.put("sentiment_score", calculateSentimentScore(message));
features.put("readability_score", calculateReadabilityScore(message));
return features;
}
/**
* Extract temporal features
*/
private Map<String, Object> extractTemporalFeatures(LogEntry entry) {
Map<String, Object> features = new HashMap<>();
LocalDateTime timestamp = entry.getTimestamp();
features.put("hour_of_day", timestamp.getHour());
features.put("day_of_week", timestamp.getDayOfWeek().getValue());
features.put("is_weekend", isWeekend(timestamp));
features.put("minute_of_hour", timestamp.getMinute());
return features;
}
/**
* Extract structural features
*/
private Map<String, Object> extractStructuralFeatures(LogEntry entry) {
Map<String, Object> features = new HashMap<>();
features.put("log_level", encodeLogLevel(entry.getLevel()));
features.put("logger_depth", calculateLoggerDepth(entry.getLogger()));
features.put("has_mdc", !entry.getMdc().isEmpty());
features.put("mdc_key_count", entry.getMdc().size());
features.put("thread_type", classifyThread(entry.getThread()));
return features;
}
/**
* Extract contextual features
*/
private Map<String, Object> extractContextualFeatures(LogEntry entry) {
Map<String, Object> features = new HashMap<>();
features.put("application", entry.getApplication() != null ? entry.getApplication() : "unknown");
features.put("environment", entry.getEnvironment() != null ? entry.getEnvironment() : "unknown");
features.put("host_type", classifyHost(entry.getHost()));
return features;
}
/**
* Extract sequence features from log batch
*/
public Map<String, Object> extractSequenceFeatures(List<LogEntry> logSequence) {
Map<String, Object> sequenceFeatures = new HashMap<>();
if (logSequence.isEmpty()) {
return sequenceFeatures;
}
// Frequency features
Map<String, Integer> levelCounts = new HashMap<>();
Map<String, Integer> loggerCounts = new HashMap<>();
for (LogEntry entry : logSequence) {
levelCounts.merge(entry.getLevel(), 1, Integer::sum);
loggerCounts.merge(entry.getLogger(), 1, Integer::sum);
}
sequenceFeatures.put("total_entries", logSequence.size());
sequenceFeatures.put("error_rate", (double) levelCounts.getOrDefault("ERROR", 0) / logSequence.size());
sequenceFeatures.put("warn_rate", (double) levelCounts.getOrDefault("WARN", 0) / logSequence.size());
sequenceFeatures.put("unique_loggers", loggerCounts.size());
sequenceFeatures.put("entropy", calculateEntropy(levelCounts, logSequence.size()));
// Temporal sequence features
double[] timeDiffs = calculateTimeDifferences(logSequence);
if (timeDiffs.length > 0) {
sequenceFeatures.put("avg_time_diff", StatUtils.mean(timeDiffs));
sequenceFeatures.put("time_diff_std", Math.sqrt(StatUtils.variance(timeDiffs)));
sequenceFeatures.put("time_diff_max", StatUtils.max(timeDiffs));
sequenceFeatures.put("time_diff_min", StatUtils.min(timeDiffs));
}
return sequenceFeatures;
}
// Helper methods
private int countWords(String text) {
return text.split("\\s+").length;
}
private boolean containsException(String message) {
return message.toLowerCase().contains("exception") || 
message.contains("Exception") ||
message.contains("Error");
}
private boolean containsStackTrace(String message) {
return message.contains("at ") && 
(message.contains(".java:") || message.contains("(Native Method)"));
}
private boolean containsUrl(String message) {
return message.matches(".*https?://[^\\s]+.*");
}
private boolean containsNumber(String message) {
return message.matches(".*\\b\\d+\\b.*");
}
private double calculateCapitalRatio(String text) {
long capitalCount = text.chars().filter(Character::isUpperCase).count();
return text.isEmpty() ? 0 : (double) capitalCount / text.length();
}
private double calculateSpecialCharRatio(String text) {
long specialCount = text.chars().filter(ch -> !Character.isLetterOrDigit(ch) && !Character.isWhitespace(ch)).count();
return text.isEmpty() ? 0 : (double) specialCount / text.length();
}
private double calculateUniqueTokenRatio(String[] tokens) {
if (tokens.length == 0) return 0;
Set<String> uniqueTokens = Arrays.stream(tokens).collect(Collectors.toSet());
return (double) uniqueTokens.size() / tokens.length;
}
private double calculateSentimentScore(String message) {
// Simple sentiment analysis based on keyword matching
String[] positiveWords = {"success", "completed", "ok", "ready", "started"};
String[] negativeWords = {"error", "failed", "exception", "timeout", "rejected"};
int positiveCount = 0;
int negativeCount = 0;
String lowerMessage = message.toLowerCase();
for (String word : positiveWords) {
if (lowerMessage.contains(word)) positiveCount++;
}
for (String word : negativeWords) {
if (lowerMessage.contains(word)) negativeCount++;
}
int total = positiveCount + negativeCount;
return total == 0 ? 0.5 : (double) positiveCount / total;
}
private double calculateReadabilityScore(String message) {
// Simple readability measure (words per sentence)
String[] sentences = message.split("[.!?]+");
if (sentences.length == 0) return 0;
int totalWords = 0;
for (String sentence : sentences) {
totalWords += countWords(sentence);
}
return (double) totalWords / sentences.length;
}
private boolean isWeekend(LocalDateTime timestamp) {
int dayOfWeek = timestamp.getDayOfWeek().getValue();
return dayOfWeek == 6 || dayOfWeek == 7; // Saturday or Sunday
}
private int encodeLogLevel(String level) {
return switch (level.toUpperCase()) {
case "TRACE" -> 1;
case "DEBUG" -> 2;
case "INFO" -> 3;
case "WARN" -> 4;
case "ERROR" -> 5;
default -> 3;
};
}
private int calculateLoggerDepth(String logger) {
return logger.split("\\.").length;
}
private String classifyThread(String thread) {
if (thread == null) return "unknown";
if (thread.contains("pool")) return "thread_pool";
if (thread.contains("http")) return "http";
if (thread.equals("main")) return "main";
return "other";
}
private String classifyHost(String host) {
if (host == null) return "unknown";
if (host.contains("prod")) return "production";
if (host.contains("staging")) return "staging";
if (host.contains("test")) return "test";
if (host.contains("dev")) return "development";
return "other";
}
private double calculateEntropy(Map<String, Integer> counts, int total) {
double entropy = 0.0;
for (int count : counts.values()) {
if (count > 0) {
double probability = (double) count / total;
entropy -= probability * (Math.log(probability) / Math.log(2));
}
}
return entropy;
}
private double[] calculateTimeDifferences(List<LogEntry> logSequence) {
if (logSequence.size() < 2) {
return new double[0];
}
double[] diffs = new double[logSequence.size() - 1];
for (int i = 1; i < logSequence.size(); i++) {
long diff = java.time.Duration.between(
logSequence.get(i - 1).getTimestamp(), 
logSequence.get(i).getTimestamp()
).toMillis();
diffs[i - 1] = diff;
}
return diffs;
}
}

AI-Powered Analysis

4. Anomaly Detection Service

package com.yourapp.loganalysis.ai;
import com.yourapp.loganalysis.model.LogEntry;
import com.yourapp.loganalysis.model.AnomalyDetectionResult;
import com.yourapp.loganalysis.preprocessing.FeatureExtractor;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.util.ModelSerializer;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.factory.Nd4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import java.time.LocalDateTime;
import java.util.*;
@Service
public class AnomalyDetectionService {
private final FeatureExtractor featureExtractor;
private MultiLayerNetwork autoencoder;
private boolean modelLoaded = false;
@Value("${ai.log-analysis.model-path:./models/}")
private String modelPath;
@Value("${ai.log-analysis.anomaly-threshold:0.8}")
private double anomalyThreshold;
public AnomalyDetectionService(FeatureExtractor featureExtractor) {
this.featureExtractor = featureExtractor;
loadModel();
}
/**
* Detect anomalies in a single log entry
*/
public AnomalyDetectionResult detectAnomaly(LogEntry logEntry) {
AnomalyDetectionResult result = new AnomalyDetectionResult();
result.setLogEntryId(logEntry.getId());
result.setDetectedAt(LocalDateTime.now());
try {
// Extract features
Map<String, Object> features = featureExtractor.extractFeatures(logEntry);
logEntry.setFeatures(features);
// Convert features to vector
INDArray input = convertFeaturesToVector(features);
// Get reconstruction error
double reconstructionError = calculateReconstructionError(input);
result.setAnomalyScore(reconstructionError);
// Determine if anomaly
boolean isAnomaly = reconstructionError > anomalyThreshold;
result.setAnomalyType(determineAnomalyType(logEntry, reconstructionError));
result.setExplanation(generateExplanation(logEntry, reconstructionError, isAnomaly));
result.setSuggestedAction(generateSuggestedAction(logEntry, isAnomaly));
logEntry.setAnomalyScore(reconstructionError);
logEntry.setIsAnomaly(isAnomaly);
} catch (Exception e) {
result.setAnomalyScore(0.0);
result.setAnomalyType("PROCESSING_ERROR");
result.setExplanation("Error processing log entry: " + e.getMessage());
result.setSuggestedAction("Check log processing configuration");
}
return result;
}
/**
* Detect anomalies in a batch of log entries
*/
public List<AnomalyDetectionResult> detectBatchAnomalies(List<LogEntry> logEntries) {
return logEntries.stream()
.map(this::detectAnomaly)
.filter(result -> result.getAnomalyScore() > anomalyThreshold)
.collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
}
/**
* Detect sequence anomalies in time-ordered log entries
*/
public List<AnomalyDetectionResult> detectSequenceAnomalies(List<LogEntry> logSequence) {
List<AnomalyDetectionResult> anomalies = new ArrayList<>();
if (logSequence.size() < 10) {
return anomalies; // Need minimum sequence length
}
// Extract sequence features
Map<String, Object> sequenceFeatures = featureExtractor.extractSequenceFeatures(logSequence);
// Detect frequency anomalies
detectFrequencyAnomalies(logSequence, anomalies);
// Detect temporal anomalies
detectTemporalAnomalies(logSequence, anomalies);
// Detect pattern anomalies
detectPatternAnomalies(logSequence, anomalies);
return anomalies;
}
/**
* Train anomaly detection model
*/
public void trainModel(List<LogEntry> trainingData) {
try {
// Convert training data to features
List<INDArray> trainingFeatures = new ArrayList<>();
for (LogEntry entry : trainingData) {
Map<String, Object> features = featureExtractor.extractFeatures(entry);
trainingFeatures.add(convertFeaturesToVector(features));
}
// Create dataset
INDArray featureMatrix = Nd4j.vstack(trainingFeatures.toArray(new INDArray[0]));
DataSet dataSet = new DataSet(featureMatrix, featureMatrix); // Autoencoder: input = output
// Train model (simplified - in practice, use proper training loop)
if (autoencoder == null) {
initializeModel(featureMatrix.columns());
}
// Training would happen here in a real implementation
// autoencoder.fit(dataSet);
// Save model
saveModel();
} catch (Exception e) {
throw new RuntimeException("Failed to train anomaly detection model", e);
}
}
private INDArray convertFeaturesToVector(Map<String, Object> features) {
// Convert features to numerical vector
// This is a simplified implementation
double[] vector = new double[50]; // Fixed size vector
int index = 0;
for (Object value : features.values()) {
if (index >= vector.length) break;
if (value instanceof Number) {
vector[index++] = ((Number) value).doubleValue();
} else if (value instanceof Boolean) {
vector[index++] = (Boolean) value ? 1.0 : 0.0;
} else if (value instanceof String) {
vector[index++] = ((String) value).hashCode() % 1000 / 1000.0;
}
}
// Pad with zeros if needed
while (index < vector.length) {
vector[index++] = 0.0;
}
return Nd4j.create(vector);
}
private double calculateReconstructionError(INDArray input) {
if (!modelLoaded || autoencoder == null) {
// Fallback: simple rule-based anomaly detection
return calculateRuleBasedAnomalyScore(input);
}
try {
// Autoencoder reconstruction
INDArray reconstructed = autoencoder.output(input);
// Calculate reconstruction error (MSE)
return reconstructed.sub(input).norm2Number().doubleValue();
} catch (Exception e) {
return calculateRuleBasedAnomalyScore(input);
}
}
private double calculateRuleBasedAnomalyScore(INDArray input) {
// Simple rule-based anomaly scoring
double score = 0.0;
// High error level
if (input.getDouble(3) > 0.8) { // Assuming error level is at index 3
score += 0.3;
}
// Long message length
if (input.getDouble(0) > 1000) { // Message length
score += 0.2;
}
// Contains exception
if (input.getDouble(5) > 0.5) { // Has exception
score += 0.3;
}
// Unusual time
if (input.getDouble(10) < 6 || input.getDouble(10) > 22) { // Hour of day
score += 0.2;
}
return Math.min(score, 1.0);
}
private String determineAnomalyType(LogEntry entry, double score) {
if (score > 0.9) return "CRITICAL";
if (score > 0.7) return "HIGH";
if (score > 0.5) return "MEDIUM";
return "LOW";
}
private String generateExplanation(LogEntry entry, double score, boolean isAnomaly) {
if (!isAnomaly) {
return "Log entry appears normal";
}
List<String> reasons = new ArrayList<>();
if (entry.getLevel().equals("ERROR")) {
reasons.add("Error level log entry");
}
if (entry.getMessage().length() > 500) {
reasons.add("Unusually long log message");
}
if (entry.getFeatures() != null) {
Object hasException = entry.getFeatures().get("has_exception");
if (hasException instanceof Boolean && (Boolean) hasException) {
reasons.add("Contains exception information");
}
}
if (reasons.isEmpty()) {
return "Unusual pattern detected with confidence: " + String.format("%.2f", score);
}
return "Anomaly detected due to: " + String.join(", ", reasons);
}
private String generateSuggestedAction(LogEntry entry, boolean isAnomaly) {
if (!isAnomaly) {
return "No action required";
}
if (entry.getLevel().equals("ERROR")) {
return "Investigate the root cause of this error";
}
if (entry.getMessage().contains("memory") || entry.getMessage().contains("heap")) {
return "Check system memory usage and garbage collection";
}
if (entry.getMessage().contains("timeout")) {
return "Investigate system performance and network connectivity";
}
return "Review log entry and monitor for similar patterns";
}
private void detectFrequencyAnomalies(List<LogEntry> sequence, List<AnomalyDetectionResult> anomalies) {
// Detect unusual frequency of error logs
long errorCount = sequence.stream()
.filter(entry -> "ERROR".equals(entry.getLevel()))
.count();
double errorRate = (double) errorCount / sequence.size();
if (errorRate > 0.1) { // More than 10% errors
AnomalyDetectionResult anomaly = new AnomalyDetectionResult();
anomaly.setAnomalyType("FREQUENCY_ANOMALY");
anomaly.setAnomalyScore(errorRate);
anomaly.setExplanation("High error rate detected: " + String.format("%.1f", errorRate * 100) + "%");
anomaly.setSuggestedAction("Investigate system stability and error sources");
anomalies.add(anomaly);
}
}
private void detectTemporalAnomalies(List<LogEntry> sequence, List<AnomalyDetectionResult> anomalies) {
// Detect bursts of log activity
if (sequence.size() < 2) return;
double[] timeDiffs = new double[sequence.size() - 1];
for (int i = 1; i < sequence.size(); i++) {
timeDiffs[i - 1] = java.time.Duration.between(
sequence.get(i - 1).getTimestamp(),
sequence.get(i).getTimestamp()
).toMillis();
}
double avgDiff = Arrays.stream(timeDiffs).average().orElse(0);
double stdDev = Math.sqrt(Arrays.stream(timeDiffs)
.map(d -> Math.pow(d - avgDiff, 2))
.average().orElse(0));
// Check for bursts (very small time differences)
long burstCount = Arrays.stream(timeDiffs).filter(d -> d < avgDiff - 2 * stdDev).count();
if (burstCount > sequence.size() * 0.1) { // More than 10% are bursts
AnomalyDetectionResult anomaly = new AnomalyDetectionResult();
anomaly.setAnomalyType("TEMPORAL_ANOMALY");
anomaly.setAnomalyScore(1.0 - (avgDiff / 1000)); // Normalize score
anomaly.setExplanation("Log burst detected with " + burstCount + " rapid successive entries");
anomaly.setSuggestedAction("Investigate system load and potential issues");
anomalies.add(anomaly);
}
}
private void detectPatternAnomalies(List<LogEntry> sequence, List<AnomalyDetectionResult> anomalies) {
// Detect unusual sequences of log patterns
Map<String, Integer> patternTransitions = new HashMap<>();
for (int i = 1; i < sequence.size(); i++) {
String from = sequence.get(i - 1).getPatternId() != null ? 
sequence.get(i - 1).getPatternId() : "unknown";
String to = sequence.get(i).getPatternId() != null ? 
sequence.get(i).getPatternId() : "unknown";
String transition = from + "->" + to;
patternTransitions.merge(transition, 1, Integer::sum);
}
// Look for rare transitions (appearing only once)
long rareTransitions = patternTransitions.values().stream()
.filter(count -> count == 1)
.count();
if (rareTransitions > patternTransitions.size() * 0.3) { // More than 30% are rare
AnomalyDetectionResult anomaly = new AnomalyDetectionResult();
anomaly.setAnomalyType("PATTERN_ANOMALY");
anomaly.setAnomalyScore((double) rareTransitions / patternTransitions.size());
anomaly.setExplanation("Unusual log sequence pattern with " + rareTransitions + " rare transitions");
anomaly.setSuggestedAction("Review system behavior for unexpected state changes");
anomalies.add(anomaly);
}
}
private void loadModel() {
try {
File modelFile = new File(modelPath + "anomaly_detector.zip");
if (modelFile.exists()) {
this.autoencoder = ModelSerializer.restoreMultiLayerNetwork(modelFile);
this.modelLoaded = true;
}
} catch (IOException e) {
System.err.println("Could not load anomaly detection model: " + e.getMessage());
}
}
private void saveModel() throws IOException {
File modelDir = new File(modelPath);
if (!modelDir.exists()) {
modelDir.mkdirs();
}
if (autoencoder != null) {
File modelFile = new File(modelPath + "anomaly_detector.zip");
ModelSerializer.writeModel(autoencoder, modelFile, true);
}
}
private void initializeModel(int inputSize) {
// This would initialize a proper autoencoder architecture
// Simplified for example purposes
System.out.println("Initializing anomaly detection model with input size: " + inputSize);
}
}

5. Log Pattern Miner

package com.yourapp.loganalysis.ai;
import com.yourapp.loganalysis.model.LogEntry;
import com.yourapp.loganalysis.model.LogPattern;
import com.yourapp.loganalysis.preprocessing.LogParser;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Service
public class LogPatternMiner {
private final LogParser logParser;
private final Map<String, LogPattern> discoveredPatterns;
public LogPatternMiner(LogParser logParser) {
this.logParser = logParser;
this.discoveredPatterns = new HashMap<>();
}
/**
* Mine patterns from log entries
*/
public List<LogPattern> minePatterns(List<LogEntry> logEntries, int minSupport) {
// Group by normalized message
Map<String, List<LogEntry>> groupedLogs = logEntries.stream()
.collect(Collectors.groupingBy(entry -> 
logParser.normalizeMessage(entry.getMessage())));
// Filter by minimum support and create patterns
return groupedLogs.entrySet().stream()
.filter(entry -> entry.getValue().size() >= minSupport)
.map(entry -> createPatternFromGroup(entry.getKey(), entry.getValue()))
.collect(Collectors.toList());
}
/**
* Incremental pattern mining
*/
public void incrementalMine(List<LogEntry> newEntries) {
for (LogEntry entry : newEntries) {
String normalized = logParser.normalizeMessage(entry.getMessage());
LogPattern pattern = discoveredPatterns.get(normalized);
if (pattern == null) {
pattern = createNewPattern(normalized, entry);
discoveredPatterns.put(normalized, pattern);
} else {
updatePattern(pattern, entry);
}
}
// Clean up infrequent patterns
cleanupPatterns();
}
/**
* Find similar patterns for a log entry
*/
public List<LogPattern> findSimilarPatterns(LogEntry entry, double similarityThreshold) {
String normalized = logParser.normalizeMessage(entry.getMessage());
return discoveredPatterns.values().stream()
.filter(pattern -> calculateSimilarity(normalized, pattern.getPattern()) >= similarityThreshold)
.sorted((p1, p2) -> Double.compare(
calculateSimilarity(normalized, p2.getPattern()),
calculateSimilarity(normalized, p1.getPattern())))
.collect(Collectors.toList());
}
/**
* Cluster log entries by pattern similarity
*/
public Map<String, List<LogEntry>> clusterLogs(List<LogEntry> entries, double similarityThreshold) {
Map<String, List<LogEntry>> clusters = new HashMap<>();
for (LogEntry entry : entries) {
List<LogPattern> similarPatterns = findSimilarPatterns(entry, similarityThreshold);
if (!similarPatterns.isEmpty()) {
String clusterId = similarPatterns.get(0).getPatternId();
clusters.computeIfAbsent(clusterId, k -> new ArrayList<>()).add(entry);
} else {
// Create new cluster
String newClusterId = "cluster_" + System.currentTimeMillis();
clusters.put(newClusterId, new ArrayList<>(List.of(entry)));
}
}
return clusters;
}
/**
* Extract emerging patterns (patterns that are becoming more frequent)
*/
public List<LogPattern> findEmergingPatterns(int timeWindowHours) {
LocalDateTime cutoff = LocalDateTime.now().minusHours(timeWindowHours);
return discoveredPatterns.values().stream()
.filter(pattern -> {
// Check if pattern frequency has increased recently
long recentCount = pattern.getMetadata() != null ? 
((Number) pattern.getMetadata().getOrDefault("recent_count", 0)).longValue() : 0;
long historicalCount = pattern.getFrequency() - recentCount;
return recentCount > historicalCount * 0.5; // 50% increase
})
.sorted((p1, p2) -> {
double trend1 = calculateTrend(p1);
double trend2 = calculateTrend(p2);
return Double.compare(trend2, trend1);
})
.collect(Collectors.toList());
}
private LogPattern createPatternFromGroup(String normalizedMessage, List<LogEntry> entries) {
LogPattern pattern = new LogPattern();
pattern.setPatternId(generatePatternId(normalizedMessage));
pattern.setPattern(normalizedMessage);
pattern.setRegex(convertToRegex(normalizedMessage));
pattern.setFrequency(entries.size());
pattern.setConfidence(calculateConfidence(entries));
pattern.setFirstSeen(entries.stream()
.map(LogEntry::getTimestamp)
.min(LocalDateTime::compareTo)
.orElse(LocalDateTime.now()));
pattern.setLastSeen(entries.stream()
.map(LogEntry::getTimestamp)
.max(LocalDateTime::compareTo)
.orElse(LocalDateTime.now()));
// Extract metadata
Map<String, Object> metadata = new HashMap<>();
metadata.put("avg_message_length", 
entries.stream().mapToInt(e -> e.getMessage().length()).average().orElse(0));
metadata.put("level_distribution", 
entries.stream().collect(Collectors.groupingBy(LogEntry::getLevel, Collectors.counting())));
metadata.put("common_loggers",
entries.stream().collect(Collectors.groupingBy(LogEntry::getLogger, Collectors.counting())));
pattern.setMetadata(metadata);
// Categorize pattern
pattern.setCategory(categorizePattern(normalizedMessage, entries));
return pattern;
}
private LogPattern createNewPattern(String normalizedMessage, LogEntry firstEntry) {
LogPattern pattern = new LogPattern();
pattern.setPatternId(generatePatternId(normalizedMessage));
pattern.setPattern(normalizedMessage);
pattern.setRegex(convertToRegex(normalizedMessage));
pattern.setFrequency(1);
pattern.setConfidence(0.5);
pattern.setFirstSeen(firstEntry.getTimestamp());
pattern.setLastSeen(firstEntry.getTimestamp());
pattern.setCategory(categorizePattern(normalizedMessage, List.of(firstEntry)));
Map<String, Object> metadata = new HashMap<>();
metadata.put("recent_count", 1);
pattern.setMetadata(metadata);
return pattern;
}
private void updatePattern(LogPattern pattern, LogEntry newEntry) {
pattern.setFrequency(pattern.getFrequency() + 1);
pattern.setLastSeen(newEntry.getTimestamp());
pattern.setConfidence(calculateUpdatedConfidence(pattern, newEntry));
// Update metadata
Map<String, Object> metadata = pattern.getMetadata();
if (metadata == null) {
metadata = new HashMap<>();
pattern.setMetadata(metadata);
}
// Update recent count if within last hour
if (newEntry.getTimestamp().isAfter(LocalDateTime.now().minusHours(1))) {
int recentCount = (int) metadata.getOrDefault("recent_count", 0);
metadata.put("recent_count", recentCount + 1);
}
}
private void cleanupPatterns() {
// Remove patterns that haven't been seen recently and have low frequency
LocalDateTime cleanupThreshold = LocalDateTime.now().minusDays(7);
discoveredPatterns.entrySet().removeIf(entry -> {
LogPattern pattern = entry.getValue();
return pattern.getFrequency() < 5 && 
pattern.getLastSeen().isBefore(cleanupThreshold);
});
}
private String generatePatternId(String normalizedMessage) {
return "pattern_" + Math.abs(normalizedMessage.hashCode());
}
private String convertToRegex(String normalizedMessage) {
// Convert normalized message to regex pattern
String regex = Pattern.quote(normalizedMessage);
// Replace placeholders with appropriate regex patterns
regex = regex.replace("<NUM>", "\\\\d+");
regex = regex.replace("<UUID>", "[0-9a-fA-F-]{36}");
regex = regex.replace("<PATH>", "[\\\\w./-]+");
regex = regex.replace("<IP>", "(?:[0-9]{1,3}\\\\.){3}[0-9]{1,3}");
regex = regex.replace("<EMAIL>", "[\\\\w.%+-]+@[\\\\w.-]+\\\\.[A-Za-z]{2,}");
return regex;
}
private double calculateConfidence(List<LogEntry> entries) {
if (entries.size() < 10) {
return 0.5; // Low confidence for small samples
}
// Calculate consistency of log levels
Map<String, Long> levelCounts = entries.stream()
.collect(Collectors.groupingBy(LogEntry::getLevel, Collectors.counting()));
long maxCount = levelCounts.values().stream().max(Long::compareTo).orElse(0L);
double levelConsistency = (double) maxCount / entries.size();
// Calculate message length consistency
double avgLength = entries.stream().mapToInt(e -> e.getMessage().length()).average().orElse(0);
double lengthVariance = entries.stream()
.mapToDouble(e -> Math.pow(e.getMessage().length() - avgLength, 2))
.average().orElse(0);
double lengthConsistency = 1.0 / (1.0 + Math.sqrt(lengthVariance) / 100.0);
return (levelConsistency + lengthConsistency) / 2.0;
}
private double calculateUpdatedConfidence(LogPattern pattern, LogEntry newEntry) {
double currentConfidence = pattern.getConfidence();
double newLevelMatch = pattern.getMetadata().get("level_distribution") instanceof Map ? 
((Map<?, ?>) pattern.getMetadata().get("level_distribution")).containsKey(newEntry.getLevel()) ? 1.0 : 0.0 : 0.5;
// Update confidence with new evidence
return (currentConfidence * pattern.getFrequency() + newLevelMatch) / (pattern.getFrequency() + 1);
}
private String categorizePattern(String normalizedMessage, List<LogEntry> entries) {
// Categorize based on content and context
String message = normalizedMessage.toLowerCase();
if (message.contains("exception") || message.contains("error")) {
return "ERROR";
} else if (message.contains("warning") || message.contains("warn")) {
return "WARNING";
} else if (message.contains("database") || message.contains("sql")) {
return "DATABASE";
} else if (message.contains("http") || message.contains("request") || message.contains("response")) {
return "NETWORK";
} else if (message.contains("memory") || message.contains("heap") || message.contains("gc")) {
return "RESOURCE";
} else if (message.contains("start") || message.contains("stop") || message.contains("shutdown")) {
return "LIFECYCLE";
} else if (message.contains("authentication") || message.contains("authorization")) {
return "SECURITY";
} else {
return "APPLICATION";
}
}
private double calculateSimilarity(String str1, String str2) {
// Calculate Jaccard similarity between token sets
Set<String> tokens1 = Set.of(str1.split("\\s+"));
Set<String> tokens2 = Set.of(str2.split("\\s+"));
Set<String> intersection = new HashSet<>(tokens1);
intersection.retainAll(tokens2);
Set<String> union = new HashSet<>(tokens1);
union.addAll(tokens2);
return union.isEmpty() ? 0.0 : (double) intersection.size() / union.size();
}
private double calculateTrend(LogPattern pattern) {
Map<String, Object> metadata = pattern.getMetadata();
if (metadata == null) return 0.0;
int recentCount = (int) metadata.getOrDefault("recent_count", 0);
int historicalCount = pattern.getFrequency() - recentCount;
if (historicalCount == 0) return 1.0; // New pattern
return (double) recentCount / historicalCount;
}
public Collection<LogPattern> getDiscoveredPatterns() {
return discoveredPatterns.values();
}
}

Integration and REST API

6. Log Analysis Controller

package com.yourapp.loganalysis.controller;
import com.yourapp.loganalysis.model.LogEntry;
import com.yourapp.loganalysis.model.AnomalyDetectionResult;
import com.yourapp.loganalysis.model.LogPattern;
import com.yourapp.loganalysis.service.LogAnalysisService;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.util.List;
import java.util.Map;
@RestController
@RequestMapping("/api/log-analysis")
public class LogAnalysisController {
private final LogAnalysisService logAnalysisService;
public LogAnalysisController(LogAnalysisService logAnalysisService) {
this.logAnalysisService = logAnalysisService;
}
@PostMapping("/analyze")
public ResponseEntity<Map<String, Object>> analyzeLogEntry(@RequestBody LogEntry logEntry) {
Map<String, Object> analysisResult = logAnalysisService.analyzeLogEntry(logEntry);
return ResponseEntity.ok(analysisResult);
}
@PostMapping("/analyze-batch")
public ResponseEntity<List<AnomalyDetectionResult>> analyzeLogBatch(@RequestBody List<LogEntry> logEntries) {
List<AnomalyDetectionResult> results = logAnalysisService.analyzeLogBatch(logEntries);
return ResponseEntity.ok(results);
}
@PostMapping("/detect-anomalies")
public ResponseEntity<List<AnomalyDetectionResult>> detectAnomalies(@RequestBody List<LogEntry> logEntries) {
List<AnomalyDetectionResult> anomalies = logAnalysisService.detectAnomalies(logEntries);
return ResponseEntity.ok(anomalies);
}
@GetMapping("/patterns")
public ResponseEntity<List<LogPattern>> getDiscoveredPatterns() {
List<LogPattern> patterns = logAnalysisService.getDiscoveredPatterns();
return ResponseEntity.ok(patterns);
}
@PostMapping("/patterns/mine")
public ResponseEntity<List<LogPattern>> minePatterns(@RequestBody List<LogEntry> logEntries,
@RequestParam(defaultValue = "5") int minSupport) {
List<LogPattern> patterns = logAnalysisService.minePatterns(logEntries, minSupport);
return ResponseEntity.ok(patterns);
}
@GetMapping("/patterns/emerging")
public ResponseEntity<List<LogPattern>> getEmergingPatterns(@RequestParam(defaultValue = "24") int hours) {
List<LogPattern> emergingPatterns = logAnalysisService.findEmergingPatterns(hours);
return ResponseEntity.ok(emergingPatterns);
}
@GetMapping("/health")
public ResponseEntity<Map<String, Object>> healthCheck() {
Map<String, Object> health = logAnalysisService.getHealthStatus();
return ResponseEntity.ok(health);
}
@PostMapping("/train")
public ResponseEntity<Map<String, Object>> trainModel(@RequestBody List<LogEntry> trainingData) {
Map<String, Object> trainingResult = logAnalysisService.trainModels(trainingData);
return ResponseEntity.ok(trainingResult);
}
}

7. Main Analysis Service

package com.yourapp.loganalysis.service;
import com.yourapp.loganalysis.ai.AnomalyDetectionService;
import com.yourapp.loganalysis.ai.LogPatternMiner;
import com.yourapp.loganalysis.model.LogEntry;
import com.yourapp.loganalysis.model.AnomalyDetectionResult;
import com.yourapp.loganalysis.model.LogPattern;
import com.yourapp.loganalysis.preprocessing.LogParser;
import org.springframework.stereotype.Service;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
public class LogAnalysisService {
private final LogParser logParser;
private final AnomalyDetectionService anomalyDetectionService;
private final LogPatternMiner logPatternMiner;
public LogAnalysisService(LogParser logParser, 
AnomalyDetectionService anomalyDetectionService,
LogPatternMiner logPatternMiner) {
this.logParser = logParser;
this.anomalyDetectionService = anomalyDetectionService;
this.logPatternMiner = logPatternMiner;
}
/**
* Comprehensive analysis of a single log entry
*/
public Map<String, Object> analyzeLogEntry(LogEntry logEntry) {
Map<String, Object> analysis = new HashMap<>();
// Parse and preprocess
LogEntry parsedEntry = logParser.parseLogLine(logEntry.getMessage(), logEntry.getSource());
analysis.put("parsed_entry", parsedEntry);
// Detect anomalies
AnomalyDetectionResult anomalyResult = anomalyDetectionService.detectAnomaly(parsedEntry);
analysis.put("anomaly_detection", anomalyResult);
// Find similar patterns
List<LogPattern> similarPatterns = logPatternMiner.findSimilarPatterns(parsedEntry, 0.7);
analysis.put("similar_patterns", similarPatterns);
// Update pattern mining
logPatternMiner.incrementalMine(List.of(parsedEntry));
return analysis;
}
/**
* Analyze batch of log entries
*/
public List<AnomalyDetectionResult> analyzeLogBatch(List<LogEntry> logEntries) {
// Parse all entries
List<LogEntry> parsedEntries = logEntries.stream()
.map(entry -> logParser.parseLogLine(entry.getMessage(), entry.getSource()))
.toList();
// Detect anomalies
List<AnomalyDetectionResult> anomalyResults = anomalyDetectionService.detectBatchAnomalies(parsedEntries);
// Update pattern mining
logPatternMiner.incrementalMine(parsedEntries);
return anomalyResults;
}
/**
* Detect anomalies in log entries
*/
public List<AnomalyDetectionResult> detectAnomalies(List<LogEntry> logEntries) {
List<LogEntry> parsedEntries = logEntries.stream()
.map(entry -> logParser.parseLogLine(entry.getMessage(), entry.getSource()))
.toList();
// Individual anomalies
List<AnomalyDetectionResult> individualAnomalies = 
anomalyDetectionService.detectBatchAnomalies(parsedEntries);
// Sequence anomalies
List<AnomalyDetectionResult> sequenceAnomalies = 
anomalyDetectionService.detectSequenceAnomalies(parsedEntries);
// Combine results
individualAnomalies.addAll(sequenceAnomalies);
return individualAnomalies;
}
/**
* Mine patterns from log entries
*/
public List<LogPattern> minePatterns(List<LogEntry> logEntries, int minSupport) {
List<LogEntry> parsedEntries = logEntries.stream()
.map(entry -> logParser.parseLogLine(entry.getMessage(), entry.getSource()))
.toList();
return logPatternMiner.minePatterns(parsedEntries, minSupport);
}
/**
* Find emerging patterns
*/
public List<LogPattern> findEmergingPatterns(int hours) {
return logPatternMiner.findEmergingPatterns(hours);
}
/**
* Get all discovered patterns
*/
public List<LogPattern> getDiscoveredPatterns() {
return List.copyOf(logPatternMiner.getDiscoveredPatterns());
}
/**
* Train AI models
*/
public Map<String, Object> trainModels(List<LogEntry> trainingData) {
Map<String, Object> trainingResult = new HashMap<>();
try {
// Parse training data
List<LogEntry> parsedTrainingData = trainingData.stream()
.map(entry -> logParser.parseLogLine(entry.getMessage(), entry.getSource()))
.toList();
// Train anomaly detection model
anomalyDetectionService.trainModel(parsedTrainingData);
trainingResult.put("anomaly_detection_training", "SUCCESS");
// Mine patterns for initial knowledge base
List<LogPattern> initialPatterns = logPatternMiner.minePatterns(parsedTrainingData, 3);
trainingResult.put("patterns_mined", initialPatterns.size());
trainingResult.put("pattern_mining", "SUCCESS");
trainingResult.put("overall_status", "SUCCESS");
} catch (Exception e) {
trainingResult.put("overall_status", "FAILED");
trainingResult.put("error", e.getMessage());
}
return trainingResult;
}
/**
* Get system health status
*/
public Map<String, Object> getHealthStatus() {
Map<String, Object> health = new HashMap<>();
health.put("service", "LogAnalysisService");
health.put("status", "UP");
health.put("patterns_count", logPatternMiner.getDiscoveredPatterns().size());
health.put("timestamp", System.currentTimeMillis());
return health;
}
}

This comprehensive AI-powered log analysis system provides sophisticated log processing, anomaly detection, pattern mining, and intelligent analysis capabilities. The system can be extended with additional machine learning models and integrated with various log storage and visualization systems.