Overview

Grok is a Logstash plugin that enables parsing of unstructured log data into structured, queryable formats. This guide covers implementing Grok-like pattern matching directly in Java for log processing.

Core Grok Pattern Implementation

1. Grok Pattern Engine in Java

import java.util.*;
import java.util.regex.*;
import java.util.concurrent.ConcurrentHashMap;
public class JavaGrok {
private final Map<String, String> patternDefinitions;
private final Map<String, Pattern> compiledPatterns;
// Common Java log patterns
private static final Map<String, String> DEFAULT_PATTERNS = Map.ofEntries(
// Basic patterns
Map.entry("WORD", "\\b\\w+\\b"),
Map.entry("NOTSPACE", "\\S+"),
Map.entry("SPACE", "\\s*"),
Map.entry("DATA", ".*?"),
Map.entry("GREEDYDATA", ".*"),
Map.entry("NUMBER", "(?:%{BASE10NUM})"),
Map.entry("BASE10NUM", "(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))"),
Map.entry("INT", "(?:[+-]?(?:[0-9]+))"),
// Java-specific patterns
Map.entry("JAVA_TIMESTAMP", "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}"),
Map.entry("JAVA_LOGGER", "(?:[a-zA-Z0-9-]+\\.)+[A-Za-z0-9$]+"),
Map.entry("JAVA_CLASS", "(?:[a-zA-Z0-9-]+\\.)+[A-Za-z0-9$]+"),
Map.entry("JAVA_METHOD", "[a-zA-Z_][a-zA-Z0-9_]*"),
Map.entry("JAVA_EXCEPTION", "(?:[a-zA-Z0-9-]+\\.)+[A-Za-z0-9$]+(?:Exception|Error)"),
Map.entry("JAVA_STACKTRACE", "(?m)^\\s*at %{JAVA_CLASS}\\.%{JAVA_METHOD}\\(.*\\)$"),
Map.entry("JAVA_THREAD", "\\[.*\\]"),
Map.entry("JAVA_LEVEL", "DEBUG|INFO|WARN|ERROR|FATAL|TRACE"),
// Date/Time patterns
Map.entry("YEAR", "(?>\\d\\d){1,2}"),
Map.entry("MONTHNUM", "(?:0?[1-9]|1[0-2])"),
Map.entry("MONTHDAY", "(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])"),
Map.entry("HOUR", "(?:2[0123]|[01]?[0-9])"),
Map.entry("MINUTE", "(?:[0-5][0-9])"),
Map.entry("SECOND", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)"),
Map.entry("ISO8601_TIMEZONE", "(?:Z|[+-]%{HOUR}(?::?%{MINUTE}))"),
// Log4j/Logback patterns
Map.entry("LOG4J_DATETIME", "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}"),
Map.entry("LOGLEVEL", "%{JAVA_LEVEL}"),
Map.entry("LOGGER", "%{JAVA_LOGGER}"),
// Web/HTTP patterns
Map.entry("IP", "(?:%{IPV6}|%{IPV4})"),
Map.entry("IPV6", "(([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))"),
Map.entry("IPV4", "(?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.]){3}(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])(?![0-9])"),
Map.entry("HTTPMETHOD", "GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS"),
Map.entry("URIPATH", "(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\\-]*)+"),
Map.entry("HTTP_VERSION", "HTTP/(?:1\\.0|1\\.1|2\\.0)"),
Map.entry("HTTP_STATUS", "(?:[1-5][0-9]{2})"),
Map.entry("USERAGENT", ".+?")
);
public JavaGrok() {
this(DEFAULT_PATTERNS);
}
public JavaGrok(Map<String, String> customPatterns) {
this.patternDefinitions = new HashMap<>(DEFAULT_PATTERNS);
if (customPatterns != null) {
this.patternDefinitions.putAll(customPatterns);
}
this.compiledPatterns = new ConcurrentHashMap<>();
}
public void addPattern(String name, String pattern) {
patternDefinitions.put(name, pattern);
compiledPatterns.clear(); // Clear compiled patterns as definitions changed
}
public Map<String, Object> match(String pattern, String text) {
String resolvedPattern = resolvePattern(pattern);
Pattern compiled = compilePattern(resolvedPattern);
Matcher matcher = compiled.matcher(text);
if (matcher.matches()) {
return extractGroups(matcher);
}
return Collections.emptyMap();
}
public List<Map<String, Object>> matchAll(String pattern, String text) {
String resolvedPattern = resolvePattern(pattern);
Pattern compiled = compilePattern(resolvedPattern);
Matcher matcher = compiled.matcher(text);
List<Map<String, Object>> results = new ArrayList<>();
while (matcher.find()) {
results.add(extractGroups(matcher));
}
return results;
}
private String resolvePattern(String pattern) {
// Resolve nested patterns like %{PATTERN_NAME}
Pattern patternRef = Pattern.compile("%\\{([A-Z0-9_]+)(?::([^}]+))?\\}");
Matcher matcher = patternRef.matcher(pattern);
StringBuffer result = new StringBuffer();
while (matcher.find()) {
String patternName = matcher.group(1);
String semanticName = matcher.group(2);
String replacement = patternDefinitions.get(patternName);
if (replacement == null) {
throw new IllegalArgumentException("Unknown pattern: " + patternName);
}
// Recursively resolve nested patterns
replacement = resolvePattern(replacement);
if (semanticName != null) {
// Create named capturing group
matcher.appendReplacement(result, "(?<" + semanticName + ">" + replacement + ")");
} else {
matcher.appendReplacement(result, replacement);
}
}
matcher.appendTail(result);
return result.toString();
}
private Pattern compilePattern(String regex) {
return compiledPatterns.computeIfAbsent(regex, 
k -> Pattern.compile(regex, Pattern.DOTALL | Pattern.MULTILINE));
}
private Map<String, Object> extractGroups(Matcher matcher) {
Map<String, Object> result = new LinkedHashMap<>();
// Add all named groups
for (String groupName : getGroupNames(matcher)) {
String value = matcher.group(groupName);
if (value != null) {
result.put(groupName, value);
}
}
// Add all numeric groups if no named groups found
if (result.isEmpty()) {
for (int i = 1; i <= matcher.groupCount(); i++) {
String value = matcher.group(i);
if (value != null) {
result.put("group" + i, value);
}
}
}
return result;
}
private Set<String> getGroupNames(Matcher matcher) {
Set<String> groupNames = new HashSet<>();
try {
// Use reflection to access named groups (Java 7+)
java.lang.reflect.Method namedGroupsMethod = 
matcher.pattern().getClass().getDeclaredMethod("namedGroups");
namedGroupsMethod.setAccessible(true);
@SuppressWarnings("unchecked")
Map<String, Integer> namedGroups = 
(Map<String, Integer>) namedGroupsMethod.invoke(matcher.pattern());
if (namedGroups != null) {
groupNames.addAll(namedGroups.keySet());
}
} catch (Exception e) {
// Fallback: try to extract group names from pattern
extractGroupNamesFromPattern(matcher.pattern().pattern(), groupNames);
}
return groupNames;
}
private void extractGroupNamesFromPattern(String pattern, Set<String> groupNames) {
// Simple extraction of named groups: (?<name>...)
Pattern namedGroupPattern = Pattern.compile("\\(\\?<([a-zA-Z][a-zA-Z0-9]*)>");
Matcher m = namedGroupPattern.matcher(pattern);
while (m.find()) {
groupNames.add(m.group(1));
}
}
}

2. Common Java Log Patterns

public class JavaLogPatterns {
// Pre-defined patterns for common Java logging frameworks
public static final Map<String, String> PATTERNS = Map.ofEntries(
// Logback pattern
Map.entry("LOGBACK_BASIC", 
"^%{JAVA_TIMESTAMP} %{LOGLEVEL} %{LOGGER} - %{GREEDYDATA:message}$"),
// Log4j pattern
Map.entry("LOG4J_BASIC", 
"^%{JAVA_TIMESTAMP} %{LOGLEVEL} \\[%{JAVA_THREAD:thread}\\] %{LOGGER} - %{GREEDYDATA:message}$"),
// Log4j with method name
Map.entry("LOG4J_DETAILED", 
"^%{JAVA_TIMESTAMP} %{LOGLEVEL} \\[%{JAVA_THREAD:thread}\\] %{LOGGER}\\.%{JAVA_METHOD:method}\\(.*\\) - %{GREEDYDATA:message}$"),
// Spring Boot default pattern
Map.entry("SPRING_BOOT", 
"^%{JAVA_TIMESTAMP} %{LOGLEVEL} %{NUMBER:pid} --- \\[%{JAVA_THREAD:thread}\\] %{LOGGER} : %{GREEDYDATA:message}$"),
// Tomcat access log
Map.entry("TOMCAT_ACCESS", 
"^%{IP:client_ip} %{NOTSPACE:ident} %{NOTSPACE:auth} \\[%{HTTPDATE:timestamp}\\] \"%{HTTPMETHOD:method} %{URIPATH:request} %{HTTP_VERSION:http_version}\" %{HTTP_STATUS:status} %{NUMBER:bytes}$"),
// Java stack trace
Map.entry("JAVA_EXCEPTION_LINE", 
"^%{JAVA_EXCEPTION:exception}: %{GREEDYDATA:exception_message}$"),
// Stack trace element
Map.entry("STACK_TRACE_ELEMENT", 
"^\\s*at %{JAVA_CLASS:class}\\.%{JAVA_METHOD:method}\\(%{NOTSPACE:file}(?::%{NUMBER:line})?\\)$"),
// JSON log pattern
Map.entry("JSON_LOG", 
"^\\{\"timestamp\":\"%{JAVA_TIMESTAMP:timestamp}\",\"level\":\"%{LOGLEVEL:level}\",\"logger\":\"%{LOGGER:logger}\",\"message\":\"%{GREEDYDATA:message}\".*\\}$"),
// HTTP request log
Map.entry("HTTP_REQUEST", 
"^%{IP:client_ip} - - \\[%{HTTPDATE:timestamp}\\] \"%{HTTPMETHOD:method} %{URIPATH:url} %{HTTP_VERSION:http_version}\" %{HTTP_STATUS:status} %{NUMBER:response_size} \"%{NOTSPACE:referrer}\" \"%{USERAGENT:user_agent}\" %{NUMBER:response_time}$")
);
private final JavaGrok grok;
public JavaLogPatterns() {
this.grok = new JavaGrok(PATTERNS);
}
public JavaLogPatterns(JavaGrok grok) {
this.grok = grok;
}
public LogEvent parseLogback(String logLine) {
Map<String, Object> matches = grok.match("LOGBACK_BASIC", logLine);
return createLogEvent(matches);
}
public LogEvent parseLog4j(String logLine) {
Map<String, Object> matches = grok.match("LOG4J_BASIC", logLine);
return createLogEvent(matches);
}
public LogEvent parseSpringBoot(String logLine) {
Map<String, Object> matches = grok.match("SPRING_BOOT", logLine);
return createLogEvent(matches);
}
public HttpRequestLog parseTomcatAccess(String logLine) {
Map<String, Object> matches = grok.match("TOMCAT_ACCESS", logLine);
return createHttpRequestLog(matches);
}
public StackTraceElement parseStackTraceLine(String line) {
Map<String, Object> matches = grok.match("STACK_TRACE_ELEMENT", line);
return createStackTraceElement(matches);
}
public ExceptionInfo parseExceptionLine(String line) {
Map<String, Object> matches = grok.match("JAVA_EXCEPTION_LINE", line);
return createExceptionInfo(matches);
}
public Map<String, Object> parseJsonLog(String logLine) {
// For JSON logs, we might want to use a JSON parser instead
// This is a fallback for when JSON parsing fails
return grok.match("JSON_LOG", logLine);
}
private LogEvent createLogEvent(Map<String, Object> matches) {
if (matches.isEmpty()) {
return null;
}
return new LogEvent(
(String) matches.get("timestamp"),
(String) matches.get("level"),
(String) matches.get("logger"),
(String) matches.get("thread"),
(String) matches.get("message"),
(String) matches.get("method")
);
}
private HttpRequestLog createHttpRequestLog(Map<String, Object> matches) {
if (matches.isEmpty()) {
return null;
}
return new HttpRequestLog(
(String) matches.get("client_ip"),
(String) matches.get("timestamp"),
(String) matches.get("method"),
(String) matches.get("request"),
(String) matches.get("http_version"),
(String) matches.get("status"),
parseLong(matches.get("bytes")),
(String) matches.get("referrer"),
(String) matches.get("user_agent")
);
}
private StackTraceElement createStackTraceElement(Map<String, Object> matches) {
if (matches.isEmpty()) {
return null;
}
return new StackTraceElement(
(String) matches.get("class"),
(String) matches.get("method"),
(String) matches.get("file"),
parseInt(matches.get("line"))
);
}
private ExceptionInfo createExceptionInfo(Map<String, Object> matches) {
if (matches.isEmpty()) {
return null;
}
return new ExceptionInfo(
(String) matches.get("exception"),
(String) matches.get("exception_message")
);
}
private Long parseLong(Object value) {
if (value == null) return null;
try {
return Long.parseLong(value.toString());
} catch (NumberFormatException e) {
return null;
}
}
private Integer parseInt(Object value) {
if (value == null) return null;
try {
return Integer.parseInt(value.toString());
} catch (NumberFormatException e) {
return null;
}
}
// Data classes
public static class LogEvent {
private final String timestamp;
private final String level;
private final String logger;
private final String thread;
private final String message;
private final String method;
public LogEvent(String timestamp, String level, String logger, 
String thread, String message, String method) {
this.timestamp = timestamp;
this.level = level;
this.logger = logger;
this.thread = thread;
this.message = message;
this.method = method;
}
// Getters
public String getTimestamp() { return timestamp; }
public String getLevel() { return level; }
public String getLogger() { return logger; }
public String getThread() { return thread; }
public String getMessage() { return message; }
public String getMethod() { return method; }
}
public static class HttpRequestLog {
private final String clientIp;
private final String timestamp;
private final String method;
private final String request;
private final String httpVersion;
private final String status;
private final Long bytes;
private final String referrer;
private final String userAgent;
public HttpRequestLog(String clientIp, String timestamp, String method, 
String request, String httpVersion, String status, 
Long bytes, String referrer, String userAgent) {
this.clientIp = clientIp;
this.timestamp = timestamp;
this.method = method;
this.request = request;
this.httpVersion = httpVersion;
this.status = status;
this.bytes = bytes;
this.referrer = referrer;
this.userAgent = userAgent;
}
// Getters
public String getClientIp() { return clientIp; }
public String getTimestamp() { return timestamp; }
public String getMethod() { return method; }
public String getRequest() { return request; }
public String getHttpVersion() { return httpVersion; }
public String getStatus() { return status; }
public Long getBytes() { return bytes; }
public String getReferrer() { return referrer; }
public String getUserAgent() { return userAgent; }
}
public static class ExceptionInfo {
private final String exceptionClass;
private final String message;
public ExceptionInfo(String exceptionClass, String message) {
this.exceptionClass = exceptionClass;
this.message = message;
}
// Getters
public String getExceptionClass() { return exceptionClass; }
public String getMessage() { return message; }
}
}

3. Log Processor with Multiple Pattern Support

import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
public class LogProcessor {
private final JavaGrok grok;
private final List<LogPattern> patterns;
private final Map<String, LogPattern> patternCache;
public LogProcessor() {
this.grok = new JavaGrok();
this.patterns = new ArrayList<>();
this.patternCache = new ConcurrentHashMap<>();
initializeDefaultPatterns();
}
private void initializeDefaultPatterns() {
// Add common log patterns with their conditions
patterns.add(new LogPattern("LOGBACK_BASIC", 
"timestamp level logger message", 
line -> line.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.*")));
patterns.add(new LogPattern("LOG4J_BASIC", 
"timestamp level thread logger message", 
line -> line.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.*\\[.*\\].*")));
patterns.add(new LogPattern("SPRING_BOOT", 
"timestamp level pid thread logger message", 
line -> line.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.*--- \\[.*\\].*")));
patterns.add(new LogPattern("TOMCAT_ACCESS", 
"client_ip ident auth timestamp method request http_version status bytes", 
line -> line.matches("^\\d+\\.\\d+\\.\\d+\\.\\d+.*\\[.*\\].*HTTP/.*")));
patterns.add(new LogPattern("JAVA_EXCEPTION", 
"exception exception_message", 
line -> line.matches("^[a-zA-Z0-9.]*Exception:.*")));
patterns.add(new LogPattern("STACK_TRACE", 
"class method file line", 
line -> line.matches("^\\s*at [a-zA-Z0-9.]+\\.*")));
}
public ProcessedLog processLine(String logLine) {
// Try cached pattern first
LogPattern cachedPattern = patternCache.get(logLine);
if (cachedPattern != null) {
Map<String, Object> matches = grok.match(cachedPattern.getPatternName(), logLine);
return new ProcessedLog(logLine, cachedPattern.getPatternName(), matches);
}
// Find matching pattern
for (LogPattern pattern : patterns) {
if (pattern.getCondition().test(logLine)) {
Map<String, Object> matches = grok.match(pattern.getPatternName(), logLine);
// Cache successful pattern for this line format
if (!matches.isEmpty()) {
patternCache.putIfAbsent(logLine, pattern);
return new ProcessedLog(logLine, pattern.getPatternName(), matches);
}
}
}
// No pattern matched
return new ProcessedLog(logLine, "UNKNOWN", Collections.singletonMap("raw_message", logLine));
}
public List<ProcessedLog> processLines(List<String> logLines) {
return logLines.stream()
.map(this::processLine)
.collect(Collectors.toList());
}
public void addPattern(LogPattern pattern) {
patterns.add(pattern);
patternCache.clear(); // Clear cache as patterns changed
}
public void addPattern(String name, String pattern, Function<String, Boolean> condition) {
addPattern(new LogPattern(name, pattern, condition));
}
public Map<String, Long> analyzeLogLevels(List<ProcessedLog> logs) {
return logs.stream()
.filter(log -> log.getFields().containsKey("level"))
.collect(Collectors.groupingBy(
log -> log.getFields().get("level").toString(),
Collectors.counting()
));
}
public Map<String, Long> analyzeLoggers(List<ProcessedLog> logs) {
return logs.stream()
.filter(log -> log.getFields().containsKey("logger"))
.collect(Collectors.groupingBy(
log -> log.getFields().get("logger").toString(),
Collectors.counting()
));
}
public List<ProcessedLog> findErrors(List<ProcessedLog> logs) {
return logs.stream()
.filter(log -> {
Object level = log.getFields().get("level");
return level != null && 
("ERROR".equals(level) || "FATAL".equals(level));
})
.collect(Collectors.toList());
}
public List<ProcessedLog> filterByLogger(List<ProcessedLog> logs, String loggerPattern) {
Pattern pattern = Pattern.compile(loggerPattern);
return logs.stream()
.filter(log -> {
Object logger = log.getFields().get("logger");
return logger != null && pattern.matcher(logger.toString()).matches();
})
.collect(Collectors.toList());
}
// Pattern definition class
public static class LogPattern {
private final String patternName;
private final String description;
private final Function<String, Boolean> condition;
public LogPattern(String patternName, String description, 
Function<String, Boolean> condition) {
this.patternName = patternName;
this.description = description;
this.condition = condition;
}
// Getters
public String getPatternName() { return patternName; }
public String getDescription() { return description; }
public Function<String, Boolean> getCondition() { return condition; }
}
// Processed log result
public static class ProcessedLog {
private final String originalLine;
private final String patternName;
private final Map<String, Object> fields;
private final long timestamp;
public ProcessedLog(String originalLine, String patternName, 
Map<String, Object> fields) {
this.originalLine = originalLine;
this.patternName = patternName;
this.fields = Collections.unmodifiableMap(fields);
this.timestamp = System.currentTimeMillis();
}
// Getters
public String getOriginalLine() { return originalLine; }
public String getPatternName() { return patternName; }
public Map<String, Object> getFields() { return fields; }
public long getTimestamp() { return timestamp; }
public Object getField(String name) {
return fields.get(name);
}
public boolean hasField(String name) {
return fields.containsKey(name);
}
}
}

4. Spring Boot Integration

import org.springframework.stereotype.Component;
import org.springframework.beans.factory.annotation.Value;
import javax.annotation.PostConstruct;
import java.io.*;
import java.nio.file.*;
import java.util.stream.Collectors;
@Component
public class LogstashService {
private final LogProcessor logProcessor;
private final JavaLogPatterns javaPatterns;
@Value("${app.log.patterns.file:patterns.conf}")
private String patternsFile;
@Value("${app.log.process.batch-size:1000}")
private int batchSize;
public LogstashService() {
this.logProcessor = new LogProcessor();
this.javaPatterns = new JavaLogPatterns();
}
@PostConstruct
public void init() {
loadCustomPatterns();
}
private void loadCustomPatterns() {
try {
Path path = Paths.get(patternsFile);
if (Files.exists(path)) {
List<String> lines = Files.readAllLines(path);
parseCustomPatterns(lines);
}
} catch (IOException e) {
System.err.println("Failed to load custom patterns: " + e.getMessage());
}
}
private void parseCustomPatterns(List<String> lines) {
for (String line : lines) {
line = line.trim();
if (line.isEmpty() || line.startsWith("#")) {
continue;
}
// Parse pattern definition: PATTERN_NAME regex
String[] parts = line.split("\\s+", 2);
if (parts.length == 2) {
String patternName = parts[0];
String pattern = parts[1];
logProcessor.addPattern(patternName, pattern, 
line -> line.matches(".*" + pattern + ".*")); // Simple condition
}
}
}
public List<LogProcessor.ProcessedLog> processLogFile(String filePath) throws IOException {
List<String> lines = Files.readAllLines(Paths.get(filePath));
return logProcessor.processLines(lines);
}
public void processLogFileStreaming(String filePath, 
Consumer<LogProcessor.ProcessedLog> consumer) 
throws IOException {
try (BufferedReader reader = Files.newBufferedReader(Paths.get(filePath))) {
String line;
while ((line = reader.readLine()) != null) {
LogProcessor.ProcessedLog processed = logProcessor.processLine(line);
consumer.accept(processed);
}
}
}
public Map<String, Object> analyzeLogFile(String filePath) throws IOException {
List<LogProcessor.ProcessedLog> logs = processLogFile(filePath);
Map<String, Long> levelCounts = logProcessor.analyzeLogLevels(logs);
Map<String, Long> loggerCounts = logProcessor.analyzeLoggers(logs);
List<LogProcessor.ProcessedLog> errors = logProcessor.findErrors(logs);
return Map.of(
"total_logs", logs.size(),
"level_distribution", levelCounts,
"logger_distribution", loggerCounts,
"error_count", errors.size(),
"errors", errors.stream()
.map(LogProcessor.ProcessedLog::getFields)
.collect(Collectors.toList())
);
}
public void exportToJson(List<LogProcessor.ProcessedLog> logs, String outputPath) 
throws IOException {
List<Map<String, Object>> jsonLogs = logs.stream()
.map(log -> {
Map<String, Object> jsonLog = new HashMap<>();
jsonLog.put("original_line", log.getOriginalLine());
jsonLog.put("pattern", log.getPatternName());
jsonLog.put("timestamp", log.getTimestamp());
jsonLog.putAll(log.getFields());
return jsonLog;
})
.collect(Collectors.toList());
// Using Jackson or similar library for JSON serialization
ObjectMapper mapper = new ObjectMapper();
mapper.writeValue(new File(outputPath), jsonLogs);
}
// Real-time log processing
public void watchLogFile(String filePath, 
Consumer<LogProcessor.ProcessedLog> processor) 
throws IOException, InterruptedException {
WatchService watchService = FileSystems.getDefault().newWatchService();
Path path = Paths.get(filePath).getParent();
path.register(watchService, StandardWatchEventKinds.ENTRY_MODIFY);
long lastPosition = Files.size(Paths.get(filePath));
while (true) {
WatchKey key = watchService.take();
for (WatchEvent<?> event : key.pollEvents()) {
if (event.kind() == StandardWatchEventKinds.ENTRY_MODIFY) {
processNewLines(filePath, lastPosition, processor);
lastPosition = Files.size(Paths.get(filePath));
}
}
key.reset();
}
}
private void processNewLines(String filePath, long lastPosition,
Consumer<LogProcessor.ProcessedLog> processor) 
throws IOException {
try (RandomAccessFile file = new RandomAccessFile(filePath, "r")) {
file.seek(lastPosition);
String line;
while ((line = file.readLine()) != null) {
LogProcessor.ProcessedLog processed = logProcessor.processLine(line);
processor.accept(processed);
}
}
}
}

5. Advanced Pattern Matching with Custom Rules

public class AdvancedGrokProcessor {
private final JavaGrok grok;
private final List<MatchingRule> rules;
public AdvancedGrokProcessor() {
this.grok = new JavaGrok();
this.rules = new ArrayList<>();
initializeRules();
}
private void initializeRules() {
// Rule for detecting high response times
rules.add(new MatchingRule(
"HIGH_RESPONSE_TIME",
"Response time exceeds threshold",
fields -> {
Object responseTime = fields.get("response_time");
if (responseTime instanceof String) {
try {
long rt = Long.parseLong((String) responseTime);
return rt > 1000; // 1 second threshold
} catch (NumberFormatException e) {
return false;
}
}
return false;
},
fields -> Map.of("alert", "HIGH_RESPONSE_TIME", 
"response_time", fields.get("response_time"))
));
// Rule for detecting error patterns
rules.add(new MatchingRule(
"ERROR_PATTERN",
"Multiple errors from same source",
fields -> {
Object level = fields.get("level");
Object logger = fields.get("logger");
return "ERROR".equals(level) && 
logger != null && 
logger.toString().contains("Database");
},
fields -> Map.of("alert", "DATABASE_ERROR", 
"logger", fields.get("logger"))
));
// Rule for suspicious activity
rules.add(new MatchingRule(
"SUSPICIOUS_ACTIVITY",
"Multiple failed login attempts",
fields -> {
Object message = fields.get("message");
return message != null && 
message.toString().toLowerCase().contains("failed login");
},
fields -> Map.of("alert", "FAILED_LOGIN_ATTEMPTS")
));
}
public EnrichedLog processAndEnrich(String logLine) {
// First, parse with standard patterns
LogProcessor.ProcessedLog processed = new LogProcessor().processLine(logLine);
// Then apply enrichment rules
List<Map<String, Object>> enrichments = new ArrayList<>();
for (MatchingRule rule : rules) {
if (rule.getCondition().test(processed.getFields())) {
enrichments.add(rule.getEnrichment().apply(processed.getFields()));
}
}
return new EnrichedLog(processed, enrichments);
}
public List<EnrichedLog> processWithRules(List<String> logLines) {
return logLines.stream()
.map(this::processAndEnrich)
.collect(Collectors.toList());
}
public void addRule(MatchingRule rule) {
rules.add(rule);
}
// Rule definition
public static class MatchingRule {
private final String name;
private final String description;
private final Function<Map<String, Object>, Boolean> condition;
private final Function<Map<String, Object>, Map<String, Object>> enrichment;
public MatchingRule(String name, String description,
Function<Map<String, Object>, Boolean> condition,
Function<Map<String, Object>, Map<String, Object>> enrichment) {
this.name = name;
this.description = description;
this.condition = condition;
this.enrichment = enrichment;
}
// Getters
public String getName() { return name; }
public String getDescription() { return description; }
public Function<Map<String, Object>, Boolean> getCondition() { return condition; }
public Function<Map<String, Object>, Map<String, Object>> getEnrichment() { return enrichment; }
}
// Enriched log result
public static class EnrichedLog {
private final LogProcessor.ProcessedLog processedLog;
private final List<Map<String, Object>> enrichments;
public EnrichedLog(LogProcessor.ProcessedLog processedLog, 
List<Map<String, Object>> enrichments) {
this.processedLog = processedLog;
this.enrichments = Collections.unmodifiableList(enrichments);
}
// Getters
public LogProcessor.ProcessedLog getProcessedLog() { return processedLog; }
public List<Map<String, Object>> getEnrichments() { return enrichments; }
public boolean hasEnrichments() { return !enrichments.isEmpty(); }
}
}

6. Performance-Optimized Batch Processing

import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
public class BatchLogProcessor {
private final LogProcessor logProcessor;
private final ExecutorService executor;
private final BlockingQueue<String> logQueue;
private final AtomicLong processedCount;
private final AtomicLong errorCount;
public BatchLogProcessor(int queueSize, int threadCount) {
this.logProcessor = new LogProcessor();
this.executor = Executors.newFixedThreadPool(threadCount);
this.logQueue = new LinkedBlockingQueue<>(queueSize);
this.processedCount = new AtomicLong(0);
this.errorCount = new AtomicLong(0);
}
public void startProcessing(Consumer<LogProcessor.ProcessedLog> resultHandler) {
for (int i = 0; i < Runtime.getRuntime().availableProcessors(); i++) {
executor.submit(() -> {
while (!Thread.currentThread().isInterrupted()) {
try {
String logLine = logQueue.poll(100, TimeUnit.MILLISECONDS);
if (logLine != null) {
processLogLine(logLine, resultHandler);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
});
}
}
public void submitLogLine(String logLine) {
try {
logQueue.put(logLine);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
public void submitLogLines(List<String> logLines) {
for (String line : logLines) {
submitLogLine(line);
}
}
private void processLogLine(String logLine, Consumer<LogProcessor.ProcessedLog> resultHandler) {
try {
LogProcessor.ProcessedLog processed = logProcessor.processLine(logLine);
processedCount.incrementAndGet();
resultHandler.accept(processed);
} catch (Exception e) {
errorCount.incrementAndGet();
System.err.println("Failed to process log line: " + logLine);
}
}
public void shutdown() {
executor.shutdown();
try {
if (!executor.awaitTermination(60, TimeUnit.SECONDS)) {
executor.shutdownNow();
}
} catch (InterruptedException e) {
executor.shutdownNow();
Thread.currentThread().interrupt();
}
}
public ProcessingStats getStats() {
return new ProcessingStats(
processedCount.get(),
errorCount.get(),
logQueue.size()
);
}
public static class ProcessingStats {
private final long processedCount;
private final long errorCount;
private final long queueSize;
public ProcessingStats(long processedCount, long errorCount, long queueSize) {
this.processedCount = processedCount;
this.errorCount = errorCount;
this.queueSize = queueSize;
}
// Getters
public long getProcessedCount() { return processedCount; }
public long getErrorCount() { return errorCount; }
public long getQueueSize() { return queueSize; }
public double getErrorRate() { 
return processedCount > 0 ? (double) errorCount / processedCount * 100 : 0.0; 
}
}
}

Usage Examples

public class GrokExamples {
public static void main(String[] args) throws Exception {
// Basic usage
JavaGrok grok = new JavaGrok();
String logLine = "2024-01-15 10:30:45 INFO  com.example.Service - Processing request";
Map<String, Object> result = grok.match("LOGBACK_BASIC", logLine);
System.out.println("Parsed fields: " + result);
// Using pattern processor
LogProcessor processor = new LogProcessor();
LogProcessor.ProcessedLog processed = processor.processLine(logLine);
System.out.println("Pattern: " + processed.getPatternName());
System.out.println("Fields: " + processed.getFields());
// Batch processing
List<String> logLines = Arrays.asList(
"2024-01-15 10:30:45 INFO  com.example.Service - Processing request",
"2024-01-15 10:30:46 ERROR com.example.Service - Database connection failed",
"127.0.0.1 - - [15/Jan/2024:10:30:47 +0000] \"GET /api/users HTTP/1.1\" 200 1234"
);
List<LogProcessor.ProcessedLog> results = processor.processLines(logLines);
for (LogProcessor.ProcessedLog log : results) {
System.out.println(log.getPatternName() + ": " + log.getFields());
}
// Advanced processing with rules
AdvancedGrokProcessor advancedProcessor = new AdvancedGrokProcessor();
EnrichedLog enriched = advancedProcessor.processAndEnrich(logLine);
if (enriched.hasEnrichments()) {
System.out.println("Enrichments: " + enriched.getEnrichments());
}
}
}

This implementation provides a comprehensive Grok-like pattern matching system in Java, suitable for parsing Java application logs with support for custom patterns, batch processing, and real-time analysis.