Overview
Log rotation and archiving are essential practices for managing application logs efficiently. They prevent log files from consuming excessive disk space, maintain system performance, and facilitate organized log analysis and compliance.
Core Concepts
1. Log Rotation
- Size-based: Rotate when log reaches specific size
- Time-based: Daily, weekly, or monthly rotation
- Hybrid approaches: Combine size and time triggers
2. Log Archiving
- Compression: Reduce storage footprint
- Retention Policies: Define how long to keep logs
- Storage Tiers: Move old logs to cheaper storage
Java Logging Frameworks with Rotation Support
1. Logback with SizeAndTimeBasedRollingPolicy
<!-- logback.xml configuration -->
<configuration>
<appender name="ROLLING" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>logs/application.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!-- Daily rotation with size limit -->
<fileNamePattern>logs/application.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
<!-- Each file max 100MB -->
<maxFileSize>100MB</maxFileSize>
<!-- Keep 30 days of history -->
<maxHistory>30</maxHistory>
<!-- Total size cap -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Clean history on start -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss} - %msg%n</pattern>
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="ROLLING" />
</root>
</configuration>
2. Log4j2 with RollingFileAppender
<!-- log4j2.xml configuration -->
<Configuration status="WARN">
<Appenders>
<RollingFile name="RollingFile" fileName="logs/app.log"
filePattern="logs/$${date:yyyy-MM}/app-%d{MM-dd-yyyy}-%i.log.gz">
<PatternLayout>
<Pattern>%d %p %c{1.} [%t] %m%n</Pattern>
</PatternLayout>
<Policies>
<!-- Rotate daily -->
<TimeBasedTriggeringPolicy />
<!-- Or when file reaches 100MB -->
<SizeBasedTriggeringPolicy size="100 MB" />
</Policies>
<!-- Keep max 100 files, delete oldest -->
<DefaultRolloverStrategy max="100">
<Delete basePath="logs" maxDepth="2">
<IfFileName glob="*/app-*.log.gz">
<IfLastModified age="30d" />
</IfFileName>
</Delete>
</DefaultRolloverStrategy>
</RollingFile>
</Appenders>
<Loggers>
<Root level="info">
<AppenderRef ref="RollingFile"/>
</Root>
</Loggers>
</Configuration>
Custom Log Rotation Implementation
1. Basic Log Rotator
public class CustomLogRotator {
private static final Logger logger = LoggerFactory.getLogger(CustomLogRotator.class);
private final Path logDirectory;
private final String baseFileName;
private final long maxFileSize;
private final int maxBackupIndex;
private final Duration rotationInterval;
private volatile int currentFileIndex = 0;
private volatile long lastRotationTime = System.currentTimeMillis();
private volatile long currentFileSize = 0;
public CustomLogRotator(Path logDirectory, String baseFileName,
long maxFileSize, int maxBackupIndex,
Duration rotationInterval) {
this.logDirectory = logDirectory;
this.baseFileName = baseFileName;
this.maxFileSize = maxFileSize;
this.maxBackupIndex = maxBackupIndex;
this.rotationInterval = rotationInterval;
initializeLogFiles();
}
private void initializeLogFiles() {
try {
Files.createDirectories(logDirectory);
// Find current file index
currentFileIndex = findCurrentFileIndex();
currentFileSize = getCurrentFileSize();
scheduleTimeBasedRotation();
} catch (IOException e) {
logger.error("Failed to initialize log rotator", e);
}
}
public synchronized void log(String message) {
checkRotationNeeded();
try {
Path currentFile = getCurrentFilePath();
String logEntry = String.format("%s - %s%n",
Instant.now().toString(), message);
byte[] bytes = logEntry.getBytes(StandardCharsets.UTF_8);
Files.write(currentFile, bytes, StandardOpenOption.CREATE,
StandardOpenOption.APPEND);
currentFileSize += bytes.length;
} catch (IOException e) {
logger.error("Failed to write log entry", e);
}
}
private void checkRotationNeeded() {
boolean sizeExceeded = currentFileSize >= maxFileSize;
boolean timeExceeded = System.currentTimeMillis() - lastRotationTime
>= rotationInterval.toMillis();
if (sizeExceeded || timeExceeded) {
rotateLog();
}
}
private synchronized void rotateLog() {
try {
Path currentFile = getCurrentFilePath();
if (Files.exists(currentFile) && Files.size(currentFile) > 0) {
// Compress and archive current file
compressCurrentFile();
// Rotate to next file
currentFileIndex = (currentFileIndex + 1) % maxBackupIndex;
currentFileSize = 0;
lastRotationTime = System.currentTimeMillis();
// Create new log file
Files.createFile(getCurrentFilePath());
cleanupOldArchives();
}
} catch (IOException e) {
logger.error("Failed to rotate log file", e);
}
}
private void compressCurrentFile() throws IOException {
Path currentFile = getCurrentFilePath();
Path compressedFile = getCompressedFilePath();
try (FileInputStream fis = new FileInputStream(currentFile.toFile());
GZIPOutputStream gzos = new GZIPOutputStream(
new FileOutputStream(compressedFile.toFile()))) {
byte[] buffer = new byte[8192];
int length;
while ((length = fis.read(buffer)) > 0) {
gzos.write(buffer, 0, length);
}
}
// Delete original file after compression
Files.delete(currentFile);
}
private Path getCurrentFilePath() {
return logDirectory.resolve(baseFileName + ".log");
}
private Path getCompressedFilePath() {
String timestamp = LocalDate.now().toString();
return logDirectory.resolve(
String.format("%s-%s-%d.log.gz", baseFileName, timestamp, currentFileIndex)
);
}
private int findCurrentFileIndex() {
// Implementation to find the latest file index
return 0;
}
private long getCurrentFileSize() {
try {
Path currentFile = getCurrentFilePath();
return Files.exists(currentFile) ? Files.size(currentFile) : 0;
} catch (IOException e) {
return 0;
}
}
private void scheduleTimeBasedRotation() {
ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
scheduler.scheduleAtFixedRate(this::checkRotationNeeded,
rotationInterval.toMinutes(), rotationInterval.toMinutes(), TimeUnit.MINUTES);
}
private void cleanupOldArchives() {
// Implementation to delete archives older than retention period
}
}
2. Advanced Log Archiver with Retention
public class LogArchiver {
private static final Logger logger = LoggerFactory.getLogger(LogArchiver.class);
private final Path archiveDirectory;
private final Duration retentionPeriod;
private final CompressionType compressionType;
public enum CompressionType {
GZIP, ZIP, NONE
}
public LogArchiver(Path archiveDirectory, Duration retentionPeriod,
CompressionType compressionType) {
this.archiveDirectory = archiveDirectory;
this.retentionPeriod = retentionPeriod;
this.compressionType = compressionType;
initializeArchiveDirectory();
}
private void initializeArchiveDirectory() {
try {
Files.createDirectories(archiveDirectory);
} catch (IOException e) {
logger.error("Failed to create archive directory", e);
}
}
public void archiveLogFile(Path logFile, String category) throws IOException {
if (!Files.exists(logFile) || Files.size(logFile) == 0) {
return;
}
String timestamp = Instant.now().toString().replace(":", "-");
String archiveFileName = String.format("%s-%s%s",
category, timestamp, getCompressionExtension());
Path archivePath = archiveDirectory.resolve(archiveFileName);
switch (compressionType) {
case GZIP:
compressWithGZIP(logFile, archivePath);
break;
case ZIP:
compressWithZIP(logFile, archivePath);
break;
case NONE:
Files.copy(logFile, archivePath);
break;
}
logger.info("Archived log file: {} to {}", logFile, archivePath);
// Cleanup old archives after new archive is created
cleanupExpiredArchives();
}
private void compressWithGZIP(Path source, Path target) throws IOException {
try (FileInputStream fis = new FileInputStream(source.toFile());
FileOutputStream fos = new FileOutputStream(target.toFile());
GZIPOutputStream gzos = new GZIPOutputStream(fos)) {
byte[] buffer = new byte[8192];
int length;
while ((length = fis.read(buffer)) > 0) {
gzos.write(buffer, 0, length);
}
}
}
private void compressWithZIP(Path source, Path target) throws IOException {
try (FileOutputStream fos = new FileOutputStream(target.toFile());
ZipOutputStream zos = new ZipOutputStream(fos)) {
ZipEntry entry = new ZipEntry(source.getFileName().toString());
zos.putNextEntry(entry);
Files.copy(source, zos);
zos.closeEntry();
}
}
private String getCompressionExtension() {
switch (compressionType) {
case GZIP: return ".gz";
case ZIP: return ".zip";
default: return ".log";
}
}
public void cleanupExpiredArchives() {
try {
Instant cutoffTime = Instant.now().minus(retentionPeriod);
Files.list(archiveDirectory)
.filter(this::isArchiveFile)
.filter(path -> isFileOlderThan(path, cutoffTime))
.forEach(this::deleteArchiveFile);
} catch (IOException e) {
logger.error("Failed to cleanup expired archives", e);
}
}
private boolean isArchiveFile(Path path) {
String fileName = path.getFileName().toString();
return fileName.endsWith(".gz") || fileName.endsWith(".zip") ||
fileName.endsWith(".log");
}
private boolean isFileOlderThan(Path path, Instant cutoffTime) {
try {
FileTime lastModified = Files.getLastModifiedTime(path);
return lastModified.toInstant().isBefore(cutcutoffTime);
} catch (IOException e) {
return false;
}
}
private void deleteArchiveFile(Path path) {
try {
Files.delete(path);
logger.info("Deleted expired archive: {}", path);
} catch (IOException e) {
logger.error("Failed to delete archive: {}", path, e);
}
}
public List<Path> listArchives(String category, Duration period) {
try {
Instant startTime = Instant.now().minus(period);
return Files.list(archiveDirectory)
.filter(path -> path.getFileName().toString().startsWith(category))
.filter(path -> !isFileOlderThan(path, startTime))
.sorted(Comparator.comparing(this::getFileCreationTime).reversed())
.collect(Collectors.toList());
} catch (IOException e) {
logger.error("Failed to list archives", e);
return Collections.emptyList();
}
}
private Instant getFileCreationTime(Path path) {
try {
return Files.getLastModifiedTime(path).toInstant();
} catch (IOException e) {
return Instant.MIN;
}
}
}
Cloud-Based Log Rotation and Archiving
1. AWS S3 Archiver
public class S3LogArchiver {
private final AmazonS3 s3Client;
private final String bucketName;
private final String basePath;
public S3LogArchiver(String bucketName, String basePath) {
this.s3Client = AmazonS3ClientBuilder.defaultClient();
this.bucketName = bucketName;
this.basePath = basePath;
}
public void archiveToS3(Path logFile, String application, String environment)
throws IOException {
if (!Files.exists(logFile)) {
return;
}
String datePath = LocalDate.now().toString();
String s3Key = String.format("%s/%s/%s/%s.gz",
basePath, application, environment, logFile.getFileName());
// Compress file before upload
Path compressedFile = compressFile(logFile);
try {
s3Client.putObject(bucketName, s3Key, compressedFile.toFile());
logger.info("Successfully archived log to S3: s3://{}/{}", bucketName, s3Key);
// Delete local compressed file after upload
Files.delete(compressedFile);
} catch (AmazonServiceException e) {
logger.error("Failed to upload log to S3", e);
throw new IOException("S3 upload failed", e);
}
}
private Path compressFile(Path source) throws IOException {
Path compressed = Files.createTempFile("log-archive-", ".gz");
try (FileInputStream fis = new FileInputStream(source.toFile());
FileOutputStream fos = new FileOutputStream(compressed.toFile());
GZIPOutputStream gzos = new GZIPOutputStream(fos)) {
byte[] buffer = new byte[8192];
int length;
while ((length = fis.read(buffer)) > 0) {
gzos.write(buffer, 0, length);
}
}
return compressed;
}
public void applyS3LifecyclePolicy(Duration expiration) {
// Set lifecycle policy for automatic deletion after expiration period
// Implementation depends on specific AWS SDK version
}
}
2. Multi-Strategy Rotation Manager
public class LogRotationManager {
private final ScheduledExecutorService scheduler;
private final List<RotationStrategy> strategies;
private final LogArchiver archiver;
public LogRotationManager(LogArchiver archiver) {
this.scheduler = Executors.newScheduledThreadPool(2);
this.strategies = new ArrayList<>();
this.archiver = archiver;
}
public void addStrategy(RotationStrategy strategy) {
strategies.add(strategy);
}
public void start() {
// Check rotation every minute
scheduler.scheduleAtFixedRate(this::checkAllStrategies, 1, 1, TimeUnit.MINUTES);
// Cleanup every hour
scheduler.scheduleAtFixedRate(archiver::cleanupExpiredArchives,
1, 1, TimeUnit.HOURS);
}
public void stop() {
scheduler.shutdown();
try {
if (!scheduler.awaitTermination(30, TimeUnit.SECONDS)) {
scheduler.shutdownNow();
}
} catch (InterruptedException e) {
scheduler.shutdownNow();
Thread.currentThread().interrupt();
}
}
private void checkAllStrategies() {
for (RotationStrategy strategy : strategies) {
if (strategy.shouldRotate()) {
performRotation(strategy);
}
}
}
private void performRotation(RotationStrategy strategy) {
try {
Path logFile = strategy.getLogFile();
String category = strategy.getCategory();
// Archive current log
archiver.archiveLogFile(logFile, category);
// Reset log file (truncate or create new)
strategy.resetLogFile();
logger.info("Performed log rotation for: {}", category);
} catch (IOException e) {
logger.error("Failed to perform log rotation", e);
}
}
public interface RotationStrategy {
boolean shouldRotate();
Path getLogFile();
String getCategory();
void resetLogFile() throws IOException;
}
public static class SizeBasedStrategy implements RotationStrategy {
private final Path logFile;
private final long maxSize;
private final String category;
public SizeBasedStrategy(Path logFile, long maxSize, String category) {
this.logFile = logFile;
this.maxSize = maxSize;
this.category = category;
}
@Override
public boolean shouldRotate() {
try {
return Files.exists(logFile) && Files.size(logFile) >= maxSize;
} catch (IOException e) {
return false;
}
}
@Override
public Path getLogFile() { return logFile; }
@Override
public String getCategory() { return category; }
@Override
public void resetLogFile() throws IOException {
Files.write(logFile, new byte[0], StandardOpenOption.TRUNCATE_EXISTING);
}
}
public static class TimeBasedStrategy implements RotationStrategy {
private final Path logFile;
private final Duration rotationInterval;
private final String category;
private volatile long lastRotationTime;
public TimeBasedStrategy(Path logFile, Duration rotationInterval, String category) {
this.logFile = logFile;
this.rotationInterval = rotationInterval;
this.category = category;
this.lastRotationTime = System.currentTimeMillis();
}
@Override
public boolean shouldRotate() {
long currentTime = System.currentTimeMillis();
return (currentTime - lastRotationTime) >= rotationInterval.toMillis();
}
@Override
public Path getLogFile() { return logFile; }
@Override
public String getCategory() { return category; }
@Override
public void resetLogFile() throws IOException {
lastRotationTime = System.currentTimeMillis();
// For time-based rotation, we might want to create a new file
String timestamp = Instant.now().toString().replace(":", "-");
Path newFile = logFile.getParent().resolve(
logFile.getFileName() + "." + timestamp
);
Files.move(logFile, newFile);
Files.createFile(logFile);
}
}
}
Best Practices and Configuration
1. Production Configuration Example
@Configuration
public class LogRotationConfig {
@Value("${logging.directory:/var/log/myapp}")
private String logDirectory;
@Value("${logging.max-file-size:100MB}")
private String maxFileSize;
@Value("${logging.max-history:30}")
private int maxHistory;
@Value("${logging.retention-days:90}")
private int retentionDays;
@Bean
public LogArchiver logArchiver() {
return new LogArchiver(
Paths.get(logDirectory, "archives"),
Duration.ofDays(retentionDays),
LogArchiver.CompressionType.GZIP
);
}
@Bean
public LogRotationManager logRotationManager(LogArchiver archiver) {
LogRotationManager manager = new LogRotationManager(archiver);
// Add size-based strategy for application logs
manager.addStrategy(new LogRotationManager.SizeBasedStrategy(
Paths.get(logDirectory, "application.log"),
parseSize(maxFileSize),
"application"
));
// Add daily rotation for audit logs
manager.addStrategy(new LogRotationManager.TimeBasedStrategy(
Paths.get(logDirectory, "audit.log"),
Duration.ofDays(1),
"audit"
));
return manager;
}
private long parseSize(String size) {
// Parse human-readable size (e.g., "100MB", "1GB")
return SizeUnit.parse(size);
}
@EventListener(ContextRefreshedEvent.class)
public void startLogRotation(ContextRefreshedEvent event) {
logRotationManager(logArchiver()).start();
}
}
2. Monitoring and Alerting
public class LogRotationMonitor {
private final MeterRegistry meterRegistry;
private final Counter rotationCounter;
private final Counter archiveCounter;
private final Gauge logSizeGauge;
public LogRotationMonitor(MeterRegistry meterRegistry, Path logDirectory) {
this.meterRegistry = meterRegistry;
this.rotationCounter = Counter.builder("log.rotations")
.description("Number of log rotations performed")
.register(meterRegistry);
this.archiveCounter = Counter.builder("log.archives")
.description("Number of log archives created")
.register(meterRegistry);
this.logSizeGauge = Gauge.builder("log.file.size")
.description("Current log file size")
.tag("file", "application.log")
.register(meterRegistry, this, monitor -> getCurrentLogSize(logDirectory));
}
public void recordRotation() {
rotationCounter.increment();
}
public void recordArchive() {
archiveCounter.increment();
}
private double getCurrentLogSize(Path logDirectory) {
try {
Path logFile = logDirectory.resolve("application.log");
if (Files.exists(logFile)) {
return Files.size(logFile) / (1024.0 * 1024.0); // Size in MB
}
} catch (IOException e) {
// Ignore measurement errors
}
return 0.0;
}
}
Conclusion
Effective log rotation and archiving in Java involves:
- Choosing the right strategy based on application needs
- Implementing proper compression to save storage space
- Enforcing retention policies to manage disk usage
- Monitoring rotation activities for operational visibility
- Integrating with cloud storage for long-term archiving
By implementing robust log rotation and archiving solutions, organizations can maintain system performance, comply with data retention policies, and ensure logs remain available for troubleshooting and analysis when needed.