AI-Powered Log Analysis in Java

This comprehensive guide covers implementing AI-powered log analysis in Java applications, including log parsing, anomaly detection, pattern recognition, and intelligent alerting using machine learning and natural language processing.

Architecture Overview

Log Sources → Log Ingestion → Preprocessing → AI Analysis → Application Logs → Parsing/Normalization → Anomaly Detection → System Logs → Feature Extraction → Pattern Recognition → Network Logs → Vectorization → Predictive Analytics → Storage → Elasticsearch/DB → Visualization/Dashboards

Prerequisites and Setup

Maven Dependencies

<properties> <dl4j.version>1.0.0-M2.1</dl4j.version> <opennlp.version>2.2.0</opennlp.version> <tensorflow.version>0.4.1</tensorflow.version> <spring-boot.version>3.1.0</spring-boot.version> <elasticsearch.version>8.8.0</elasticsearch.version> </properties> <dependencies> <!-- Deep Learning --> <dependency> <groupId>org.deeplearning4j</groupId> <artifactId>deeplearning4j-core</artifactId> <version>${dl4j.version}</version> </dependency> <dependency> <groupId>org.nd4j</groupId> <artifactId>nd4j-native-platform</artifactId> <version>${dl4j.version}</version> </dependency> <!-- NLP --> <dependency> <groupId>org.apache.opennlp</groupId> <artifactId>opennlp-tools</artifactId> <version>${opennlp.version}</version> </dependency> <!-- TensorFlow --> <dependency> <groupId>org.tensorflow</groupId> <artifactId>tensorflow-core-api</artifactId> <version>${tensorflow.version}</version> </dependency> <!-- Spring Boot --> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-web</artifactId> <version>${spring-boot.version}</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-elasticsearch</artifactId> <version>${spring-boot.version}</version> </dependency> <!-- Utilities --> <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> <version>2.15.2</version> </dependency> <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-math3</artifactId> <version>3.6.1</version> </dependency> </dependencies>

Configuration Properties

# AI Model Configuration ai.log-analysis.model-path=./models/ ai.log-analysis.anomaly-threshold=0.8 ai.log-analysis.similarity-threshold=0.7 ai.log-analysis.training-enabled=true # Log Processing log.ingestion.batch-size=1000 log.ingestion.buffer-timeout=5000 log.ingestion.max-queue-size=10000 # Elasticsearch spring.elasticsearch.uris=http://localhost:9200 spring.elasticsearch.username=elastic spring.elasticsearch.password=password # Feature Extraction log.features.vector-size=100 log.features.window-size=10 log.features.min-word-frequency=5 # Anomaly Detection anomaly.detection.sensitivity=0.75 anomaly.detection.training-interval=3600000

Core Data Models

1. Log Entry Models

package com.yourapp.loganalysis.model; import java.time.LocalDateTime; import java.util.Map; public class LogEntry { private String id; private LocalDateTime timestamp; private String level; private String logger; private String message; private String thread; private Map<String, String> mdc; private String exception; private String stackTrace; private String application; private String environment; private String host; private String source; // AI Analysis Fields private Double anomalyScore; private String patternId; private String category; private Map<String, Object> features; private Boolean isAnomaly; // Constructors public LogEntry() {} public LogEntry(String message, String level, LocalDateTime timestamp) { this.message = message; this.level = level; this.timestamp = timestamp; } // Getters and Setters public String getId() { return id; } public void setId(String id) { this.id = id; } public LocalDateTime getTimestamp() { return timestamp; } public void setTimestamp(LocalDateTime timestamp) { this.timestamp = timestamp; } public String getLevel() { return level; } public void setLevel(String level) { this.level = level; } public String getLogger() { return logger; } public void setLogger(String logger) { this.logger = logger; } public String getMessage() { return message; } public void setMessage(String message) { this.message = message; } public String getThread() { return thread; } public void setThread(String thread) { this.thread = thread; } public Map<String, String> getMdc() { return mdc; } public void setMdc(Map<String, String> mdc) { this.mdc = mdc; } public String getException() { return exception; } public void setException(String exception) { this.exception = exception; } public String getStackTrace() { return stackTrace; } public void setStackTrace(String stackTrace) { this.stackTrace = stackTrace; } public String getApplication() { return application; } public void setApplication(String application) { this.application = application; } public String getEnvironment() { return environment; } public void setEnvironment(String environment) { this.environment = environment; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public String getSource() { return source; } public void setSource(String source) { this.source = source; } public Double getAnomalyScore() { return anomalyScore; } public void setAnomalyScore(Double anomalyScore) { this.anomalyScore = anomalyScore; } public String getPatternId() { return patternId; } public void setPatternId(String patternId) { this.patternId = patternId; } public String getCategory() { return category; } public void setCategory(String category) { this.category = category; } public Map<String, Object> getFeatures() { return features; } public void setFeatures(Map<String, Object> features) { this.features = features; } public Boolean getIsAnomaly() { return isAnomaly; } public void setIsAnomaly(Boolean isAnomaly) { this.isAnomaly = isAnomaly; } } public class LogPattern { private String patternId; private String pattern; private String regex; private String category; private Integer frequency; private Double confidence; private LocalDateTime firstSeen; private LocalDateTime lastSeen; private Map<String, Object> metadata; // Getters and Setters public String getPatternId() { return patternId; } public void setPatternId(String patternId) { this.patternId = patternId; } public String getPattern() { return pattern; } public void setPattern(String pattern) { this.pattern = pattern; } public String getRegex() { return regex; } public void setRegex(String regex) { this.regex = regex; } public String getCategory() { return category; } public void setCategory(String category) { this.category = category; } public Integer getFrequency() { return frequency; } public void setFrequency(Integer frequency) { this.frequency = frequency; } public Double getConfidence() { return confidence; } public void setConfidence(Double confidence) { this.confidence = confidence; } public LocalDateTime getFirstSeen() { return firstSeen; } public void setFirstSeen(LocalDateTime firstSeen) { this.firstSeen = firstSeen; } public LocalDateTime getLastSeen() { return lastSeen; } public void setLastSeen(LocalDateTime lastSeen) { this.lastSeen = lastSeen; } public Map<String, Object> getMetadata() { return metadata; } public void setMetadata(Map<String, Object> metadata) { this.metadata = metadata; } } public class AnomalyDetectionResult { private String logEntryId; private Double anomalyScore; private String anomalyType; private String explanation; private LocalDateTime detectedAt; private Map<String, Object> context; private Boolean isFalsePositive; private String suggestedAction; // Getters and Setters public String getLogEntryId() { return logEntryId; } public void setLogEntryId(String logEntryId) { this.logEntryId = logEntryId; } public Double getAnomalyScore() { return anomalyScore; } public void setAnomalyScore(Double anomalyScore) { this.anomalyScore = anomalyScore; } public String getAnomalyType() { return anomalyType; } public void setAnomalyType(String anomalyType) { this.anomalyType = anomalyType; } public String getExplanation() { return explanation; } public void setExplanation(String explanation) { this.explanation = explanation; } public LocalDateTime getDetectedAt() { return detectedAt; } public void setDetectedAt(LocalDateTime detectedAt) { this.detectedAt = detectedAt; } public Map<String, Object> getContext() { return context; } public void setContext(Map<String, Object> context) { this.context = context; } public Boolean getIsFalsePositive() { return isFalsePositive; } public void setIsFalsePositive(Boolean isFalsePositive) { this.isFalsePositive = isFalsePositive; } public String getSuggestedAction() { return suggestedAction; } public void setSuggestedAction(String suggestedAction) { this.suggestedAction = suggestedAction; } }

Log Preprocessing & Feature Extraction

2. Log Parser and Normalizer

package com.yourapp.loganalysis.preprocessing; import com.yourapp.loganalysis.model.LogEntry; import opennlp.tools.tokenize.Tokenizer; import opennlp.tools.tokenize.TokenizerME; import opennlp.tools.tokenize.TokenizerModel; import org.springframework.stereotype.Component; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.*; import java.util.regex.Pattern; @Component public class LogParser { private final Tokenizer tokenizer; private final List<LogPattern> knownPatterns; private final Map<String, Pattern> compiledPatterns; public LogParser() throws IOException { // Initialize NLP tokenizer try (InputStream modelIn = new FileInputStream("models/en-token.bin")) { TokenizerModel model = new TokenizerModel(modelIn); this.tokenizer = new TokenizerME(model); } this.knownPatterns = new ArrayList<>(); this.compiledPatterns = new HashMap<>(); initializeKnownPatterns(); } /** * Parse raw log line into structured LogEntry */ public LogEntry parseLogLine(String rawLog, String source) { LogEntry entry = new LogEntry(); entry.setSource(source); entry.setTimestamp(extractTimestamp(rawLog)); entry.setLevel(extractLogLevel(rawLog)); entry.setMessage(extractMessage(rawLog)); entry.setLogger(extractLogger(rawLog)); entry.setThread(extractThread(rawLog)); entry.setMdc(extractMdc(rawLog)); // Apply pattern matching applyPatternMatching(entry); return entry; } /** * Extract timestamp from log line */ private LocalDateTime extractTimestamp(String logLine) { // Common timestamp patterns String[] timestampPatterns = { "\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}", "\\d{2}/\\d{2}/\\d{4} \\d{2}:\\d{2}:\\d{2}", "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z" }; for (String pattern : timestampPatterns) { java.util.regex.Pattern p = java.util.regex.Pattern.compile(pattern); java.util.regex.Matcher m = p.matcher(logLine); if (m.find()) { String timestampStr = m.group(); try { return LocalDateTime.parse(timestampStr, DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")); } catch (Exception e) { // Try other formats... } } } return LocalDateTime.now(); } /** * Extract log level from log line */ private String extractLogLevel(String logLine) { String[] levels = {"ERROR", "WARN", "INFO", "DEBUG", "TRACE"}; for (String level : levels) { if (logLine.contains(level)) { return level; } } return "INFO"; } /** * Extract the actual log message */ private String extractMessage(String logLine) { // Remove timestamp, level, logger, etc. String message = logLine; // Remove timestamp pattern message = message.replaceAll("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}", ""); // Remove log level message = message.replaceAll("\\b(ERROR|WARN|INFO|DEBUG|TRACE)\\b", ""); // Remove logger name in brackets message = message.replaceAll("\\[.*?\\]", ""); return message.trim(); } /** * Extract logger name */ private String extractLogger(String logLine) { java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("\\[(.*?)\\]"); java.util.regex.Matcher matcher = pattern.matcher(logLine); if (matcher.find()) { return matcher.group(1); } return "unknown"; } /** * Extract thread name */ private String extractThread(String logLine) { java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("\\[([^\\]]*?)\\]"); java.util.regex.Matcher matcher = pattern.matcher(logLine); List<String> brackets = new ArrayList<>(); while (matcher.find()) { brackets.add(matcher.group(1)); } return brackets.size() > 1 ? brackets.get(1) : "main"; } /** * Extract MDC context */ private Map<String, String> extractMdc(String logLine) { Map<String, String> mdc = new HashMap<>(); // Extract key-value pairs from MDC java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("\\{([^}]*)\\}"); java.util.regex.Matcher matcher = pattern.matcher(logLine); while (matcher.find()) { String mdcContent = matcher.group(1); String[] pairs = mdcContent.split(","); for (String pair : pairs) { String[] keyValue = pair.split("="); if (keyValue.length == 2) { mdc.put(keyValue[0].trim(), keyValue[1].trim()); } } } return mdc; } /** * Apply pattern matching to categorize log entry */ private void applyPatternMatching(LogEntry entry) { for (LogPattern pattern : knownPatterns) { if (entry.getMessage().matches(pattern.getRegex())) { entry.setPatternId(pattern.getPatternId()); entry.setCategory(pattern.getCategory()); break; } } } /** * Tokenize log message for NLP processing */ public String[] tokenizeMessage(String message) { return tokenizer.tokenize(message); } /** * Normalize log message by removing variables */ public String normalizeMessage(String message) { // Replace numbers with <NUM> message = message.replaceAll("\\b\\d+\\b", "<NUM>"); // Replace UUIDs with <UUID> message = message.replaceAll("[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}", "<UUID>"); // Replace file paths with <PATH> message = message.replaceAll("/[\\w./-]+", "<PATH>"); // Replace IP addresses with <IP> message = message.replaceAll("\\b(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\b", "<IP>"); // Replace email addresses with <EMAIL> message = message.replaceAll("\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b", "<EMAIL>"); return message; } private void initializeKnownPatterns() { // Database connection patterns knownPatterns.add(createPattern("DB_CONNECTION", ".*(connection|connect|database).*(failed|error|timeout).*", "DATABASE")); // Authentication patterns knownPatterns.add(createPattern("AUTH_FAILED", ".*(authentication|auth|login).*(failed|invalid|denied).*", "SECURITY")); // Memory patterns knownPatterns.add(createPattern("MEMORY_ERROR", ".*(memory|heap|outofmemory).*(error|full|exhausted).*", "RESOURCE")); // Network patterns knownPatterns.add(createPattern("NETWORK_ERROR", ".*(connection|socket|network).*(timeout|refused|reset).*", "NETWORK")); } private LogPattern createPattern(String id, String regex, String category) { LogPattern pattern = new LogPattern(); pattern.setPatternId(id); pattern.setRegex(regex); pattern.setCategory(category); pattern.setConfidence(0.9); return pattern; } }

3. Feature Extractor

package com.yourapp.loganalysis.preprocessing; import com.yourapp.loganalysis.model.LogEntry; import org.apache.commons.math3.stat.StatUtils; import org.springframework.stereotype.Component; import java.util.*; import java.util.stream.Collectors; @Component public class FeatureExtractor { private final LogParser logParser; public FeatureExtractor(LogParser logParser) { this.logParser = logParser; } /** * Extract features from log entry for ML models */ public Map<String, Object> extractFeatures(LogEntry entry) { Map<String, Object> features = new HashMap<>(); // Text-based features features.putAll(extractTextFeatures(entry.getMessage())); // Temporal features features.putAll(extractTemporalFeatures(entry)); // Structural features features.putAll(extractStructuralFeatures(entry)); // Contextual features features.putAll(extractContextualFeatures(entry)); return features; } /** * Extract text-based features from log message */ private Map<String, Object> extractTextFeatures(String message) { Map<String, Object> features = new HashMap<>(); String[] tokens = logParser.tokenizeMessage(message); String normalizedMessage = logParser.normalizeMessage(message); features.put("message_length", message.length()); features.put("token_count", tokens.length); features.put("word_count", countWords(message)); features.put("has_exception", containsException(message)); features.put("has_stack_trace", containsStackTrace(message)); features.put("has_url", containsUrl(message)); features.put("has_number", containsNumber(message)); features.put("capital_ratio", calculateCapitalRatio(message)); features.put("special_char_ratio", calculateSpecialCharRatio(message)); features.put("unique_token_ratio", calculateUniqueTokenRatio(tokens)); // NLP features features.put("sentiment_score", calculateSentimentScore(message)); features.put("readability_score", calculateReadabilityScore(message)); return features; } /** * Extract temporal features */ private Map<String, Object> extractTemporalFeatures(LogEntry entry) { Map<String, Object> features = new HashMap<>(); LocalDateTime timestamp = entry.getTimestamp(); features.put("hour_of_day", timestamp.getHour()); features.put("day_of_week", timestamp.getDayOfWeek().getValue()); features.put("is_weekend", isWeekend(timestamp)); features.put("minute_of_hour", timestamp.getMinute()); return features; } /** * Extract structural features */ private Map<String, Object> extractStructuralFeatures(LogEntry entry) { Map<String, Object> features = new HashMap<>(); features.put("log_level", encodeLogLevel(entry.getLevel())); features.put("logger_depth", calculateLoggerDepth(entry.getLogger())); features.put("has_mdc", !entry.getMdc().isEmpty()); features.put("mdc_key_count", entry.getMdc().size()); features.put("thread_type", classifyThread(entry.getThread())); return features; } /** * Extract contextual features */ private Map<String, Object> extractContextualFeatures(LogEntry entry) { Map<String, Object> features = new HashMap<>(); features.put("application", entry.getApplication() != null ? entry.getApplication() : "unknown"); features.put("environment", entry.getEnvironment() != null ? entry.getEnvironment() : "unknown"); features.put("host_type", classifyHost(entry.getHost())); return features; } /** * Extract sequence features from log batch */ public Map<String, Object> extractSequenceFeatures(List<LogEntry> logSequence) { Map<String, Object> sequenceFeatures = new HashMap<>(); if (logSequence.isEmpty()) { return sequenceFeatures; } // Frequency features Map<String, Integer> levelCounts = new HashMap<>(); Map<String, Integer> loggerCounts = new HashMap<>(); for (LogEntry entry : logSequence) { levelCounts.merge(entry.getLevel(), 1, Integer::sum); loggerCounts.merge(entry.getLogger(), 1, Integer::sum); } sequenceFeatures.put("total_entries", logSequence.size()); sequenceFeatures.put("error_rate", (double) levelCounts.getOrDefault("ERROR", 0) / logSequence.size()); sequenceFeatures.put("warn_rate", (double) levelCounts.getOrDefault("WARN", 0) / logSequence.size()); sequenceFeatures.put("unique_loggers", loggerCounts.size()); sequenceFeatures.put("entropy", calculateEntropy(levelCounts, logSequence.size())); // Temporal sequence features double[] timeDiffs = calculateTimeDifferences(logSequence); if (timeDiffs.length > 0) { sequenceFeatures.put("avg_time_diff", StatUtils.mean(timeDiffs)); sequenceFeatures.put("time_diff_std", Math.sqrt(StatUtils.variance(timeDiffs))); sequenceFeatures.put("time_diff_max", StatUtils.max(timeDiffs)); sequenceFeatures.put("time_diff_min", StatUtils.min(timeDiffs)); } return sequenceFeatures; } // Helper methods private int countWords(String text) { return text.split("\\s+").length; } private boolean containsException(String message) { return message.toLowerCase().contains("exception") || message.contains("Exception") || message.contains("Error"); } private boolean containsStackTrace(String message) { return message.contains("at ") && (message.contains(".java:") || message.contains("(Native Method)")); } private boolean containsUrl(String message) { return message.matches(".*https?://[^\\s]+.*"); } private boolean containsNumber(String message) { return message.matches(".*\\b\\d+\\b.*"); } private double calculateCapitalRatio(String text) { long capitalCount = text.chars().filter(Character::isUpperCase).count(); return text.isEmpty() ? 0 : (double) capitalCount / text.length(); } private double calculateSpecialCharRatio(String text) { long specialCount = text.chars().filter(ch -> !Character.isLetterOrDigit(ch) && !Character.isWhitespace(ch)).count(); return text.isEmpty() ? 0 : (double) specialCount / text.length(); } private double calculateUniqueTokenRatio(String[] tokens) { if (tokens.length == 0) return 0; Set<String> uniqueTokens = Arrays.stream(tokens).collect(Collectors.toSet()); return (double) uniqueTokens.size() / tokens.length; } private double calculateSentimentScore(String message) { // Simple sentiment analysis based on keyword matching String[] positiveWords = {"success", "completed", "ok", "ready", "started"}; String[] negativeWords = {"error", "failed", "exception", "timeout", "rejected"}; int positiveCount = 0; int negativeCount = 0; String lowerMessage = message.toLowerCase(); for (String word : positiveWords) { if (lowerMessage.contains(word)) positiveCount++; } for (String word : negativeWords) { if (lowerMessage.contains(word)) negativeCount++; } int total = positiveCount + negativeCount; return total == 0 ? 0.5 : (double) positiveCount / total; } private double calculateReadabilityScore(String message) { // Simple readability measure (words per sentence) String[] sentences = message.split("[.!?]+"); if (sentences.length == 0) return 0; int totalWords = 0; for (String sentence : sentences) { totalWords += countWords(sentence); } return (double) totalWords / sentences.length; } private boolean isWeekend(LocalDateTime timestamp) { int dayOfWeek = timestamp.getDayOfWeek().getValue(); return dayOfWeek == 6 || dayOfWeek == 7; // Saturday or Sunday } private int encodeLogLevel(String level) { return switch (level.toUpperCase()) { case "TRACE" -> 1; case "DEBUG" -> 2; case "INFO" -> 3; case "WARN" -> 4; case "ERROR" -> 5; default -> 3; }; } private int calculateLoggerDepth(String logger) { return logger.split("\\.").length; } private String classifyThread(String thread) { if (thread == null) return "unknown"; if (thread.contains("pool")) return "thread_pool"; if (thread.contains("http")) return "http"; if (thread.equals("main")) return "main"; return "other"; } private String classifyHost(String host) { if (host == null) return "unknown"; if (host.contains("prod")) return "production"; if (host.contains("staging")) return "staging"; if (host.contains("test")) return "test"; if (host.contains("dev")) return "development"; return "other"; } private double calculateEntropy(Map<String, Integer> counts, int total) { double entropy = 0.0; for (int count : counts.values()) { if (count > 0) { double probability = (double) count / total; entropy -= probability * (Math.log(probability) / Math.log(2)); } } return entropy; } private double[] calculateTimeDifferences(List<LogEntry> logSequence) { if (logSequence.size() < 2) { return new double[0]; } double[] diffs = new double[logSequence.size() - 1]; for (int i = 1; i < logSequence.size(); i++) { long diff = java.time.Duration.between( logSequence.get(i - 1).getTimestamp(), logSequence.get(i).getTimestamp() ).toMillis(); diffs[i - 1] = diff; } return diffs; } }

AI-Powered Analysis

4. Anomaly Detection Service

package com.yourapp.loganalysis.ai; import com.yourapp.loganalysis.model.LogEntry; import com.yourapp.loganalysis.model.AnomalyDetectionResult; import com.yourapp.loganalysis.preprocessing.FeatureExtractor; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.util.ModelSerializer; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.factory.Nd4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.File; import java.io.IOException; import java.time.LocalDateTime; import java.util.*; @Service public class AnomalyDetectionService { private final FeatureExtractor featureExtractor; private MultiLayerNetwork autoencoder; private boolean modelLoaded = false; @Value("${ai.log-analysis.model-path:./models/}") private String modelPath; @Value("${ai.log-analysis.anomaly-threshold:0.8}") private double anomalyThreshold; public AnomalyDetectionService(FeatureExtractor featureExtractor) { this.featureExtractor = featureExtractor; loadModel(); } /** * Detect anomalies in a single log entry */ public AnomalyDetectionResult detectAnomaly(LogEntry logEntry) { AnomalyDetectionResult result = new AnomalyDetectionResult(); result.setLogEntryId(logEntry.getId()); result.setDetectedAt(LocalDateTime.now()); try { // Extract features Map<String, Object> features = featureExtractor.extractFeatures(logEntry); logEntry.setFeatures(features); // Convert features to vector INDArray input = convertFeaturesToVector(features); // Get reconstruction error double reconstructionError = calculateReconstructionError(input); result.setAnomalyScore(reconstructionError); // Determine if anomaly boolean isAnomaly = reconstructionError > anomalyThreshold; result.setAnomalyType(determineAnomalyType(logEntry, reconstructionError)); result.setExplanation(generateExplanation(logEntry, reconstructionError, isAnomaly)); result.setSuggestedAction(generateSuggestedAction(logEntry, isAnomaly)); logEntry.setAnomalyScore(reconstructionError); logEntry.setIsAnomaly(isAnomaly); } catch (Exception e) { result.setAnomalyScore(0.0); result.setAnomalyType("PROCESSING_ERROR"); result.setExplanation("Error processing log entry: " + e.getMessage()); result.setSuggestedAction("Check log processing configuration"); } return result; } /** * Detect anomalies in a batch of log entries */ public List<AnomalyDetectionResult> detectBatchAnomalies(List<LogEntry> logEntries) { return logEntries.stream() .map(this::detectAnomaly) .filter(result -> result.getAnomalyScore() > anomalyThreshold) .collect(ArrayList::new, ArrayList::add, ArrayList::addAll); } /** * Detect sequence anomalies in time-ordered log entries */ public List<AnomalyDetectionResult> detectSequenceAnomalies(List<LogEntry> logSequence) { List<AnomalyDetectionResult> anomalies = new ArrayList<>(); if (logSequence.size() < 10) { return anomalies; // Need minimum sequence length } // Extract sequence features Map<String, Object> sequenceFeatures = featureExtractor.extractSequenceFeatures(logSequence); // Detect frequency anomalies detectFrequencyAnomalies(logSequence, anomalies); // Detect temporal anomalies detectTemporalAnomalies(logSequence, anomalies); // Detect pattern anomalies detectPatternAnomalies(logSequence, anomalies); return anomalies; } /** * Train anomaly detection model */ public void trainModel(List<LogEntry> trainingData) { try { // Convert training data to features List<INDArray> trainingFeatures = new ArrayList<>(); for (LogEntry entry : trainingData) { Map<String, Object> features = featureExtractor.extractFeatures(entry); trainingFeatures.add(convertFeaturesToVector(features)); } // Create dataset INDArray featureMatrix = Nd4j.vstack(trainingFeatures.toArray(new INDArray[0])); DataSet dataSet = new DataSet(featureMatrix, featureMatrix); // Autoencoder: input = output // Train model (simplified - in practice, use proper training loop) if (autoencoder == null) { initializeModel(featureMatrix.columns()); } // Training would happen here in a real implementation // autoencoder.fit(dataSet); // Save model saveModel(); } catch (Exception e) { throw new RuntimeException("Failed to train anomaly detection model", e); } } private INDArray convertFeaturesToVector(Map<String, Object> features) { // Convert features to numerical vector // This is a simplified implementation double[] vector = new double[50]; // Fixed size vector int index = 0; for (Object value : features.values()) { if (index >= vector.length) break; if (value instanceof Number) { vector[index++] = ((Number) value).doubleValue(); } else if (value instanceof Boolean) { vector[index++] = (Boolean) value ? 1.0 : 0.0; } else if (value instanceof String) { vector[index++] = ((String) value).hashCode() % 1000 / 1000.0; } } // Pad with zeros if needed while (index < vector.length) { vector[index++] = 0.0; } return Nd4j.create(vector); } private double calculateReconstructionError(INDArray input) { if (!modelLoaded || autoencoder == null) { // Fallback: simple rule-based anomaly detection return calculateRuleBasedAnomalyScore(input); } try { // Autoencoder reconstruction INDArray reconstructed = autoencoder.output(input); // Calculate reconstruction error (MSE) return reconstructed.sub(input).norm2Number().doubleValue(); } catch (Exception e) { return calculateRuleBasedAnomalyScore(input); } } private double calculateRuleBasedAnomalyScore(INDArray input) { // Simple rule-based anomaly scoring double score = 0.0; // High error level if (input.getDouble(3) > 0.8) { // Assuming error level is at index 3 score += 0.3; } // Long message length if (input.getDouble(0) > 1000) { // Message length score += 0.2; } // Contains exception if (input.getDouble(5) > 0.5) { // Has exception score += 0.3; } // Unusual time if (input.getDouble(10) < 6 || input.getDouble(10) > 22) { // Hour of day score += 0.2; } return Math.min(score, 1.0); } private String determineAnomalyType(LogEntry entry, double score) { if (score > 0.9) return "CRITICAL"; if (score > 0.7) return "HIGH"; if (score > 0.5) return "MEDIUM"; return "LOW"; } private String generateExplanation(LogEntry entry, double score, boolean isAnomaly) { if (!isAnomaly) { return "Log entry appears normal"; } List<String> reasons = new ArrayList<>(); if (entry.getLevel().equals("ERROR")) { reasons.add("Error level log entry"); } if (entry.getMessage().length() > 500) { reasons.add("Unusually long log message"); } if (entry.getFeatures() != null) { Object hasException = entry.getFeatures().get("has_exception"); if (hasException instanceof Boolean && (Boolean) hasException) { reasons.add("Contains exception information"); } } if (reasons.isEmpty()) { return "Unusual pattern detected with confidence: " + String.format("%.2f", score); } return "Anomaly detected due to: " + String.join(", ", reasons); } private String generateSuggestedAction(LogEntry entry, boolean isAnomaly) { if (!isAnomaly) { return "No action required"; } if (entry.getLevel().equals("ERROR")) { return "Investigate the root cause of this error"; } if (entry.getMessage().contains("memory") || entry.getMessage().contains("heap")) { return "Check system memory usage and garbage collection"; } if (entry.getMessage().contains("timeout")) { return "Investigate system performance and network connectivity"; } return "Review log entry and monitor for similar patterns"; } private void detectFrequencyAnomalies(List<LogEntry> sequence, List<AnomalyDetectionResult> anomalies) { // Detect unusual frequency of error logs long errorCount = sequence.stream() .filter(entry -> "ERROR".equals(entry.getLevel())) .count(); double errorRate = (double) errorCount / sequence.size(); if (errorRate > 0.1) { // More than 10% errors AnomalyDetectionResult anomaly = new AnomalyDetectionResult(); anomaly.setAnomalyType("FREQUENCY_ANOMALY"); anomaly.setAnomalyScore(errorRate); anomaly.setExplanation("High error rate detected: " + String.format("%.1f", errorRate * 100) + "%"); anomaly.setSuggestedAction("Investigate system stability and error sources"); anomalies.add(anomaly); } } private void detectTemporalAnomalies(List<LogEntry> sequence, List<AnomalyDetectionResult> anomalies) { // Detect bursts of log activity if (sequence.size() < 2) return; double[] timeDiffs = new double[sequence.size() - 1]; for (int i = 1; i < sequence.size(); i++) { timeDiffs[i - 1] = java.time.Duration.between( sequence.get(i - 1).getTimestamp(), sequence.get(i).getTimestamp() ).toMillis(); } double avgDiff = Arrays.stream(timeDiffs).average().orElse(0); double stdDev = Math.sqrt(Arrays.stream(timeDiffs) .map(d -> Math.pow(d - avgDiff, 2)) .average().orElse(0)); // Check for bursts (very small time differences) long burstCount = Arrays.stream(timeDiffs).filter(d -> d < avgDiff - 2 * stdDev).count(); if (burstCount > sequence.size() * 0.1) { // More than 10% are bursts AnomalyDetectionResult anomaly = new AnomalyDetectionResult(); anomaly.setAnomalyType("TEMPORAL_ANOMALY"); anomaly.setAnomalyScore(1.0 - (avgDiff / 1000)); // Normalize score anomaly.setExplanation("Log burst detected with " + burstCount + " rapid successive entries"); anomaly.setSuggestedAction("Investigate system load and potential issues"); anomalies.add(anomaly); } } private void detectPatternAnomalies(List<LogEntry> sequence, List<AnomalyDetectionResult> anomalies) { // Detect unusual sequences of log patterns Map<String, Integer> patternTransitions = new HashMap<>(); for (int i = 1; i < sequence.size(); i++) { String from = sequence.get(i - 1).getPatternId() != null ? sequence.get(i - 1).getPatternId() : "unknown"; String to = sequence.get(i).getPatternId() != null ? sequence.get(i).getPatternId() : "unknown"; String transition = from + "->" + to; patternTransitions.merge(transition, 1, Integer::sum); } // Look for rare transitions (appearing only once) long rareTransitions = patternTransitions.values().stream() .filter(count -> count == 1) .count(); if (rareTransitions > patternTransitions.size() * 0.3) { // More than 30% are rare AnomalyDetectionResult anomaly = new AnomalyDetectionResult(); anomaly.setAnomalyType("PATTERN_ANOMALY"); anomaly.setAnomalyScore((double) rareTransitions / patternTransitions.size()); anomaly.setExplanation("Unusual log sequence pattern with " + rareTransitions + " rare transitions"); anomaly.setSuggestedAction("Review system behavior for unexpected state changes"); anomalies.add(anomaly); } } private void loadModel() { try { File modelFile = new File(modelPath + "anomaly_detector.zip"); if (modelFile.exists()) { this.autoencoder = ModelSerializer.restoreMultiLayerNetwork(modelFile); this.modelLoaded = true; } } catch (IOException e) { System.err.println("Could not load anomaly detection model: " + e.getMessage()); } } private void saveModel() throws IOException { File modelDir = new File(modelPath); if (!modelDir.exists()) { modelDir.mkdirs(); } if (autoencoder != null) { File modelFile = new File(modelPath + "anomaly_detector.zip"); ModelSerializer.writeModel(autoencoder, modelFile, true); } } private void initializeModel(int inputSize) { // This would initialize a proper autoencoder architecture // Simplified for example purposes System.out.println("Initializing anomaly detection model with input size: " + inputSize); } }

5. Log Pattern Miner

package com.yourapp.loganalysis.ai; import com.yourapp.loganalysis.model.LogEntry; import com.yourapp.loganalysis.model.LogPattern; import com.yourapp.loganalysis.preprocessing.LogParser; import org.springframework.stereotype.Service; import java.time.LocalDateTime; import java.util.*; import java.util.regex.Pattern; import java.util.stream.Collectors; @Service public class LogPatternMiner { private final LogParser logParser; private final Map<String, LogPattern> discoveredPatterns; public LogPatternMiner(LogParser logParser) { this.logParser = logParser; this.discoveredPatterns = new HashMap<>(); } /** * Mine patterns from log entries */ public List<LogPattern> minePatterns(List<LogEntry> logEntries, int minSupport) { // Group by normalized message Map<String, List<LogEntry>> groupedLogs = logEntries.stream() .collect(Collectors.groupingBy(entry -> logParser.normalizeMessage(entry.getMessage()))); // Filter by minimum support and create patterns return groupedLogs.entrySet().stream() .filter(entry -> entry.getValue().size() >= minSupport) .map(entry -> createPatternFromGroup(entry.getKey(), entry.getValue())) .collect(Collectors.toList()); } /** * Incremental pattern mining */ public void incrementalMine(List<LogEntry> newEntries) { for (LogEntry entry : newEntries) { String normalized = logParser.normalizeMessage(entry.getMessage()); LogPattern pattern = discoveredPatterns.get(normalized); if (pattern == null) { pattern = createNewPattern(normalized, entry); discoveredPatterns.put(normalized, pattern); } else { updatePattern(pattern, entry); } } // Clean up infrequent patterns cleanupPatterns(); } /** * Find similar patterns for a log entry */ public List<LogPattern> findSimilarPatterns(LogEntry entry, double similarityThreshold) { String normalized = logParser.normalizeMessage(entry.getMessage()); return discoveredPatterns.values().stream() .filter(pattern -> calculateSimilarity(normalized, pattern.getPattern()) >= similarityThreshold) .sorted((p1, p2) -> Double.compare( calculateSimilarity(normalized, p2.getPattern()), calculateSimilarity(normalized, p1.getPattern()))) .collect(Collectors.toList()); } /** * Cluster log entries by pattern similarity */ public Map<String, List<LogEntry>> clusterLogs(List<LogEntry> entries, double similarityThreshold) { Map<String, List<LogEntry>> clusters = new HashMap<>(); for (LogEntry entry : entries) { List<LogPattern> similarPatterns = findSimilarPatterns(entry, similarityThreshold); if (!similarPatterns.isEmpty()) { String clusterId = similarPatterns.get(0).getPatternId(); clusters.computeIfAbsent(clusterId, k -> new ArrayList<>()).add(entry); } else { // Create new cluster String newClusterId = "cluster_" + System.currentTimeMillis(); clusters.put(newClusterId, new ArrayList<>(List.of(entry))); } } return clusters; } /** * Extract emerging patterns (patterns that are becoming more frequent) */ public List<LogPattern> findEmergingPatterns(int timeWindowHours) { LocalDateTime cutoff = LocalDateTime.now().minusHours(timeWindowHours); return discoveredPatterns.values().stream() .filter(pattern -> { // Check if pattern frequency has increased recently long recentCount = pattern.getMetadata() != null ? ((Number) pattern.getMetadata().getOrDefault("recent_count", 0)).longValue() : 0; long historicalCount = pattern.getFrequency() - recentCount; return recentCount > historicalCount * 0.5; // 50% increase }) .sorted((p1, p2) -> { double trend1 = calculateTrend(p1); double trend2 = calculateTrend(p2); return Double.compare(trend2, trend1); }) .collect(Collectors.toList()); } private LogPattern createPatternFromGroup(String normalizedMessage, List<LogEntry> entries) { LogPattern pattern = new LogPattern(); pattern.setPatternId(generatePatternId(normalizedMessage)); pattern.setPattern(normalizedMessage); pattern.setRegex(convertToRegex(normalizedMessage)); pattern.setFrequency(entries.size()); pattern.setConfidence(calculateConfidence(entries)); pattern.setFirstSeen(entries.stream() .map(LogEntry::getTimestamp) .min(LocalDateTime::compareTo) .orElse(LocalDateTime.now())); pattern.setLastSeen(entries.stream() .map(LogEntry::getTimestamp) .max(LocalDateTime::compareTo) .orElse(LocalDateTime.now())); // Extract metadata Map<String, Object> metadata = new HashMap<>(); metadata.put("avg_message_length", entries.stream().mapToInt(e -> e.getMessage().length()).average().orElse(0)); metadata.put("level_distribution", entries.stream().collect(Collectors.groupingBy(LogEntry::getLevel, Collectors.counting()))); metadata.put("common_loggers", entries.stream().collect(Collectors.groupingBy(LogEntry::getLogger, Collectors.counting()))); pattern.setMetadata(metadata); // Categorize pattern pattern.setCategory(categorizePattern(normalizedMessage, entries)); return pattern; } private LogPattern createNewPattern(String normalizedMessage, LogEntry firstEntry) { LogPattern pattern = new LogPattern(); pattern.setPatternId(generatePatternId(normalizedMessage)); pattern.setPattern(normalizedMessage); pattern.setRegex(convertToRegex(normalizedMessage)); pattern.setFrequency(1); pattern.setConfidence(0.5); pattern.setFirstSeen(firstEntry.getTimestamp()); pattern.setLastSeen(firstEntry.getTimestamp()); pattern.setCategory(categorizePattern(normalizedMessage, List.of(firstEntry))); Map<String, Object> metadata = new HashMap<>(); metadata.put("recent_count", 1); pattern.setMetadata(metadata); return pattern; } private void updatePattern(LogPattern pattern, LogEntry newEntry) { pattern.setFrequency(pattern.getFrequency() + 1); pattern.setLastSeen(newEntry.getTimestamp()); pattern.setConfidence(calculateUpdatedConfidence(pattern, newEntry)); // Update metadata Map<String, Object> metadata = pattern.getMetadata(); if (metadata == null) { metadata = new HashMap<>(); pattern.setMetadata(metadata); } // Update recent count if within last hour if (newEntry.getTimestamp().isAfter(LocalDateTime.now().minusHours(1))) { int recentCount = (int) metadata.getOrDefault("recent_count", 0); metadata.put("recent_count", recentCount + 1); } } private void cleanupPatterns() { // Remove patterns that haven't been seen recently and have low frequency LocalDateTime cleanupThreshold = LocalDateTime.now().minusDays(7); discoveredPatterns.entrySet().removeIf(entry -> { LogPattern pattern = entry.getValue(); return pattern.getFrequency() < 5 && pattern.getLastSeen().isBefore(cleanupThreshold); }); } private String generatePatternId(String normalizedMessage) { return "pattern_" + Math.abs(normalizedMessage.hashCode()); } private String convertToRegex(String normalizedMessage) { // Convert normalized message to regex pattern String regex = Pattern.quote(normalizedMessage); // Replace placeholders with appropriate regex patterns regex = regex.replace("<NUM>", "\\\\d+"); regex = regex.replace("<UUID>", "[0-9a-fA-F-]{36}"); regex = regex.replace("<PATH>", "[\\\\w./-]+"); regex = regex.replace("<IP>", "(?:[0-9]{1,3}\\\\.){3}[0-9]{1,3}"); regex = regex.replace("<EMAIL>", "[\\\\w.%+-]+@[\\\\w.-]+\\\\.[A-Za-z]{2,}"); return regex; } private double calculateConfidence(List<LogEntry> entries) { if (entries.size() < 10) { return 0.5; // Low confidence for small samples } // Calculate consistency of log levels Map<String, Long> levelCounts = entries.stream() .collect(Collectors.groupingBy(LogEntry::getLevel, Collectors.counting())); long maxCount = levelCounts.values().stream().max(Long::compareTo).orElse(0L); double levelConsistency = (double) maxCount / entries.size(); // Calculate message length consistency double avgLength = entries.stream().mapToInt(e -> e.getMessage().length()).average().orElse(0); double lengthVariance = entries.stream() .mapToDouble(e -> Math.pow(e.getMessage().length() - avgLength, 2)) .average().orElse(0); double lengthConsistency = 1.0 / (1.0 + Math.sqrt(lengthVariance) / 100.0); return (levelConsistency + lengthConsistency) / 2.0; } private double calculateUpdatedConfidence(LogPattern pattern, LogEntry newEntry) { double currentConfidence = pattern.getConfidence(); double newLevelMatch = pattern.getMetadata().get("level_distribution") instanceof Map ? ((Map<?, ?>) pattern.getMetadata().get("level_distribution")).containsKey(newEntry.getLevel()) ? 1.0 : 0.0 : 0.5; // Update confidence with new evidence return (currentConfidence * pattern.getFrequency() + newLevelMatch) / (pattern.getFrequency() + 1); } private String categorizePattern(String normalizedMessage, List<LogEntry> entries) { // Categorize based on content and context String message = normalizedMessage.toLowerCase(); if (message.contains("exception") || message.contains("error")) { return "ERROR"; } else if (message.contains("warning") || message.contains("warn")) { return "WARNING"; } else if (message.contains("database") || message.contains("sql")) { return "DATABASE"; } else if (message.contains("http") || message.contains("request") || message.contains("response")) { return "NETWORK"; } else if (message.contains("memory") || message.contains("heap") || message.contains("gc")) { return "RESOURCE"; } else if (message.contains("start") || message.contains("stop") || message.contains("shutdown")) { return "LIFECYCLE"; } else if (message.contains("authentication") || message.contains("authorization")) { return "SECURITY"; } else { return "APPLICATION"; } } private double calculateSimilarity(String str1, String str2) { // Calculate Jaccard similarity between token sets Set<String> tokens1 = Set.of(str1.split("\\s+")); Set<String> tokens2 = Set.of(str2.split("\\s+")); Set<String> intersection = new HashSet<>(tokens1); intersection.retainAll(tokens2); Set<String> union = new HashSet<>(tokens1); union.addAll(tokens2); return union.isEmpty() ? 0.0 : (double) intersection.size() / union.size(); } private double calculateTrend(LogPattern pattern) { Map<String, Object> metadata = pattern.getMetadata(); if (metadata == null) return 0.0; int recentCount = (int) metadata.getOrDefault("recent_count", 0); int historicalCount = pattern.getFrequency() - recentCount; if (historicalCount == 0) return 1.0; // New pattern return (double) recentCount / historicalCount; } public Collection<LogPattern> getDiscoveredPatterns() { return discoveredPatterns.values(); } }

Integration and REST API

6. Log Analysis Controller

package com.yourapp.loganalysis.controller; import com.yourapp.loganalysis.model.LogEntry; import com.yourapp.loganalysis.model.AnomalyDetectionResult; import com.yourapp.loganalysis.model.LogPattern; import com.yourapp.loganalysis.service.LogAnalysisService; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import java.util.List; import java.util.Map; @RestController @RequestMapping("/api/log-analysis") public class LogAnalysisController { private final LogAnalysisService logAnalysisService; public LogAnalysisController(LogAnalysisService logAnalysisService) { this.logAnalysisService = logAnalysisService; } @PostMapping("/analyze") public ResponseEntity<Map<String, Object>> analyzeLogEntry(@RequestBody LogEntry logEntry) { Map<String, Object> analysisResult = logAnalysisService.analyzeLogEntry(logEntry); return ResponseEntity.ok(analysisResult); } @PostMapping("/analyze-batch") public ResponseEntity<List<AnomalyDetectionResult>> analyzeLogBatch(@RequestBody List<LogEntry> logEntries) { List<AnomalyDetectionResult> results = logAnalysisService.analyzeLogBatch(logEntries); return ResponseEntity.ok(results); } @PostMapping("/detect-anomalies") public ResponseEntity<List<AnomalyDetectionResult>> detectAnomalies(@RequestBody List<LogEntry> logEntries) { List<AnomalyDetectionResult> anomalies = logAnalysisService.detectAnomalies(logEntries); return ResponseEntity.ok(anomalies); } @GetMapping("/patterns") public ResponseEntity<List<LogPattern>> getDiscoveredPatterns() { List<LogPattern> patterns = logAnalysisService.getDiscoveredPatterns(); return ResponseEntity.ok(patterns); } @PostMapping("/patterns/mine") public ResponseEntity<List<LogPattern>> minePatterns(@RequestBody List<LogEntry> logEntries, @RequestParam(defaultValue = "5") int minSupport) { List<LogPattern> patterns = logAnalysisService.minePatterns(logEntries, minSupport); return ResponseEntity.ok(patterns); } @GetMapping("/patterns/emerging") public ResponseEntity<List<LogPattern>> getEmergingPatterns(@RequestParam(defaultValue = "24") int hours) { List<LogPattern> emergingPatterns = logAnalysisService.findEmergingPatterns(hours); return ResponseEntity.ok(emergingPatterns); } @GetMapping("/health") public ResponseEntity<Map<String, Object>> healthCheck() { Map<String, Object> health = logAnalysisService.getHealthStatus(); return ResponseEntity.ok(health); } @PostMapping("/train") public ResponseEntity<Map<String, Object>> trainModel(@RequestBody List<LogEntry> trainingData) { Map<String, Object> trainingResult = logAnalysisService.trainModels(trainingData); return ResponseEntity.ok(trainingResult); } }

7. Main Analysis Service

package com.yourapp.loganalysis.service; import com.yourapp.loganalysis.ai.AnomalyDetectionService; import com.yourapp.loganalysis.ai.LogPatternMiner; import com.yourapp.loganalysis.model.LogEntry; import com.yourapp.loganalysis.model.AnomalyDetectionResult; import com.yourapp.loganalysis.model.LogPattern; import com.yourapp.loganalysis.preprocessing.LogParser; import org.springframework.stereotype.Service; import java.util.HashMap; import java.util.List; import java.util.Map; @Service public class LogAnalysisService { private final LogParser logParser; private final AnomalyDetectionService anomalyDetectionService; private final LogPatternMiner logPatternMiner; public LogAnalysisService(LogParser logParser, AnomalyDetectionService anomalyDetectionService, LogPatternMiner logPatternMiner) { this.logParser = logParser; this.anomalyDetectionService = anomalyDetectionService; this.logPatternMiner = logPatternMiner; } /** * Comprehensive analysis of a single log entry */ public Map<String, Object> analyzeLogEntry(LogEntry logEntry) { Map<String, Object> analysis = new HashMap<>(); // Parse and preprocess LogEntry parsedEntry = logParser.parseLogLine(logEntry.getMessage(), logEntry.getSource()); analysis.put("parsed_entry", parsedEntry); // Detect anomalies AnomalyDetectionResult anomalyResult = anomalyDetectionService.detectAnomaly(parsedEntry); analysis.put("anomaly_detection", anomalyResult); // Find similar patterns List<LogPattern> similarPatterns = logPatternMiner.findSimilarPatterns(parsedEntry, 0.7); analysis.put("similar_patterns", similarPatterns); // Update pattern mining logPatternMiner.incrementalMine(List.of(parsedEntry)); return analysis; } /** * Analyze batch of log entries */ public List<AnomalyDetectionResult> analyzeLogBatch(List<LogEntry> logEntries) { // Parse all entries List<LogEntry> parsedEntries = logEntries.stream() .map(entry -> logParser.parseLogLine(entry.getMessage(), entry.getSource())) .toList(); // Detect anomalies List<AnomalyDetectionResult> anomalyResults = anomalyDetectionService.detectBatchAnomalies(parsedEntries); // Update pattern mining logPatternMiner.incrementalMine(parsedEntries); return anomalyResults; } /** * Detect anomalies in log entries */ public List<AnomalyDetectionResult> detectAnomalies(List<LogEntry> logEntries) { List<LogEntry> parsedEntries = logEntries.stream() .map(entry -> logParser.parseLogLine(entry.getMessage(), entry.getSource())) .toList(); // Individual anomalies List<AnomalyDetectionResult> individualAnomalies = anomalyDetectionService.detectBatchAnomalies(parsedEntries); // Sequence anomalies List<AnomalyDetectionResult> sequenceAnomalies = anomalyDetectionService.detectSequenceAnomalies(parsedEntries); // Combine results individualAnomalies.addAll(sequenceAnomalies); return individualAnomalies; } /** * Mine patterns from log entries */ public List<LogPattern> minePatterns(List<LogEntry> logEntries, int minSupport) { List<LogEntry> parsedEntries = logEntries.stream() .map(entry -> logParser.parseLogLine(entry.getMessage(), entry.getSource())) .toList(); return logPatternMiner.minePatterns(parsedEntries, minSupport); } /** * Find emerging patterns */ public List<LogPattern> findEmergingPatterns(int hours) { return logPatternMiner.findEmergingPatterns(hours); } /** * Get all discovered patterns */ public List<LogPattern> getDiscoveredPatterns() { return List.copyOf(logPatternMiner.getDiscoveredPatterns()); } /** * Train AI models */ public Map<String, Object> trainModels(List<LogEntry> trainingData) { Map<String, Object> trainingResult = new HashMap<>(); try { // Parse training data List<LogEntry> parsedTrainingData = trainingData.stream() .map(entry -> logParser.parseLogLine(entry.getMessage(), entry.getSource())) .toList(); // Train anomaly detection model anomalyDetectionService.trainModel(parsedTrainingData); trainingResult.put("anomaly_detection_training", "SUCCESS"); // Mine patterns for initial knowledge base List<LogPattern> initialPatterns = logPatternMiner.minePatterns(parsedTrainingData, 3); trainingResult.put("patterns_mined", initialPatterns.size()); trainingResult.put("pattern_mining", "SUCCESS"); trainingResult.put("overall_status", "SUCCESS"); } catch (Exception e) { trainingResult.put("overall_status", "FAILED"); trainingResult.put("error", e.getMessage()); } return trainingResult; } /** * Get system health status */ public Map<String, Object> getHealthStatus() { Map<String, Object> health = new HashMap<>(); health.put("service", "LogAnalysisService"); health.put("status", "UP"); health.put("patterns_count", logPatternMiner.getDiscoveredPatterns().size()); health.put("timestamp", System.currentTimeMillis()); return health; } }

This comprehensive AI-powered log analysis system provides sophisticated log processing, anomaly detection, pattern mining, and intelligent analysis capabilities. The system can be extended with additional machine learning models and integrated with various log storage and visualization systems.