MACRO NEPAL
Scatter/Gather I/O Operations in Java
awiskar acharya
November 4, 2025
2:19 am

Overview

Scatter/Gather I/O is a method of reading from and writing to multiple buffers in a single I/O operation. This technique improves performance by reducing system calls and allowing more efficient data processing.

Key Concepts

  • Scattering Read: Reading data from a single channel into multiple buffers
  • Gathering Write: Writing data from multiple buffers to a single channel
  • Buffer Management: Efficient handling of multiple buffers in one operation

Core Classes

  • ScatteringByteChannel: Interface for scattering reads
  • GatheringByteChannel: Interface for gathering writes
  • FileChannel: Implements both interfaces
  • SocketChannel: Implements both interfaces
  • ByteBuffer: Primary buffer type used

Basic Usage

1. Scattering Read Example

import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
public class ScatterGatherExample {
public static void scatteringReadExample() throws Exception {
// Create a test file with different data types
createTestFile("test.dat");
try (RandomAccessFile file = new RandomAccessFile("test.dat", "r");
FileChannel channel = file.getChannel()) {
// Create multiple buffers for different data types
ByteBuffer headerBuffer = ByteBuffer.allocate(128);    // Header data
ByteBuffer bodyBuffer = ByteBuffer.allocate(1024);     // Body data
ByteBuffer footerBuffer = ByteBuffer.allocate(64);     // Footer data
// Array of buffers for scattering read
ByteBuffer[] buffers = { headerBuffer, bodyBuffer, footerBuffer };
// Perform scattering read - fills all buffers in sequence
long bytesRead = channel.read(buffers);
System.out.println("Total bytes read: " + bytesRead);
// Process each buffer
headerBuffer.flip();
bodyBuffer.flip();
footerBuffer.flip();
System.out.println("Header: " + bytesToHex(headerBuffer));
System.out.println("Body size: " + bodyBuffer.remaining());
System.out.println("Footer: " + new String(footerBuffer.array()).trim());
}
}
private static void createTestFile(String filename) throws Exception {
try (RandomAccessFile file = new RandomAccessFile(filename, "rw");
FileChannel channel = file.getChannel()) {
ByteBuffer header = ByteBuffer.wrap("HEADER:File Format v1.0".getBytes());
ByteBuffer body = ByteBuffer.allocate(1024);
for (int i = 0; i < 1024; i++) {
body.put((byte) (i % 256));
}
body.flip();
ByteBuffer footer = ByteBuffer.wrap("FOOTER:End of File".getBytes());
ByteBuffer[] buffers = { header, body, footer };
channel.write(buffers);
}
}
private static String bytesToHex(ByteBuffer buffer) {
StringBuilder hex = new StringBuilder();
while (buffer.hasRemaining()) {
hex.append(String.format("%02x ", buffer.get()));
}
return hex.toString();
}
}

2. Gathering Write Example

public class GatheringWriteExample {
public static void gatheringWriteExample() throws Exception {
try (RandomAccessFile file = new RandomAccessFile("output.dat", "rw");
FileChannel channel = file.getChannel()) {
// Create multiple buffers with different data
ByteBuffer header = createHeader();
ByteBuffer payload = createPayload();
ByteBuffer trailer = createTrailer();
// Array of buffers for gathering write
ByteBuffer[] buffers = { header, payload, trailer };
// Perform gathering write - writes all buffers in sequence
long bytesWritten = channel.write(buffers);
System.out.println("Total bytes written: " + bytesWritten);
// Verify the write
channel.position(0);
ByteBuffer readBuffer = ByteBuffer.allocate((int) channel.size());
channel.read(readBuffer);
readBuffer.flip();
System.out.println("File content starts with: " + 
new String(readBuffer.array(), 0, 50));
}
}
private static ByteBuffer createHeader() {
String header = "=== FILE HEADER ===\nVersion: 1.0\nCreated: " + 
new java.util.Date() + "\n";
return ByteBuffer.wrap(header.getBytes());
}
private static ByteBuffer createPayload() {
ByteBuffer buffer = ByteBuffer.allocate(512);
for (int i = 0; i < 512; i++) {
buffer.put((byte) ('A' + (i % 26)));
}
buffer.flip();
return buffer;
}
private static ByteBuffer createTrailer() {
String trailer = "\n=== FILE TRAILER ===\nEnd of File";
return ByteBuffer.wrap(trailer.getBytes());
}
}

Practical Examples

Example 1: Network Protocol Implementation

public class NetworkProtocolHandler {
private static final int HEADER_SIZE = 16;
private static final int MAX_PAYLOAD_SIZE = 8192;
public static class Message {
private final int messageId;
private final int sequenceNumber;
private final int payloadLength;
private final ByteBuffer payload;
private final int checksum;
public Message(int messageId, int sequenceNumber, ByteBuffer payload) {
this.messageId = messageId;
this.sequenceNumber = sequenceNumber;
this.payload = payload;
this.payloadLength = payload.remaining();
this.checksum = calculateChecksum(payload);
}
public ByteBuffer[] toBuffers() {
ByteBuffer header = ByteBuffer.allocate(HEADER_SIZE);
header.putInt(messageId);
header.putInt(sequenceNumber);
header.putInt(payloadLength);
header.putInt(checksum);
header.flip();
return new ByteBuffer[]{ header, payload.slice() };
}
private int calculateChecksum(ByteBuffer buffer) {
int sum = 0;
ByteBuffer readOnly = buffer.asReadOnlyBuffer();
while (readOnly.hasRemaining()) {
sum += readOnly.get() & 0xFF;
}
return sum & 0xFFFF;
}
public static Message parse(ByteBuffer[] buffers) {
if (buffers.length < 2) {
throw new IllegalArgumentException("Need at least two buffers");
}
ByteBuffer header = buffers[0];
ByteBuffer payload = buffers[1];
int messageId = header.getInt();
int sequenceNumber = header.getInt();
int payloadLength = header.getInt();
int expectedChecksum = header.getInt();
// Validate payload length
if (payload.remaining() != payloadLength) {
throw new IllegalArgumentException("Payload length mismatch");
}
// Validate checksum
int actualChecksum = calculateChecksum(payload);
if (actualChecksum != expectedChecksum) {
throw new IllegalArgumentException("Checksum mismatch");
}
return new Message(messageId, sequenceNumber, payload);
}
// Getters
public int getMessageId() { return messageId; }
public int getSequenceNumber() { return sequenceNumber; }
public ByteBuffer getPayload() { return payload; }
}
public static void sendMessage(Message message, GatheringByteChannel channel) 
throws Exception {
ByteBuffer[] buffers = message.toBuffers();
long totalWritten = 0;
while (totalWritten < getTotalSize(buffers)) {
long written = channel.write(buffers);
if (written == -1) {
throw new IOException("Channel closed");
}
totalWritten += written;
}
}
public static Message receiveMessage(ScatteringByteChannel channel) 
throws Exception {
ByteBuffer header = ByteBuffer.allocate(HEADER_SIZE);
ByteBuffer payload = ByteBuffer.allocate(MAX_PAYLOAD_SIZE);
ByteBuffer[] buffers = { header, payload };
long totalRead = 0;
long expectedSize = HEADER_SIZE;
// First read: get header to determine payload size
while (totalRead < expectedSize) {
long read = channel.read(buffers);
if (read == -1) {
throw new IOException("Channel closed");
}
totalRead += read;
// After reading header, we know the payload size
if (header.position() == HEADER_SIZE) {
header.flip();
int payloadLength = header.getInt(8); // payload length at position 8
expectedSize = HEADER_SIZE + payloadLength;
header.rewind();
// Resize payload buffer if needed
if (payloadLength > MAX_PAYLOAD_SIZE) {
throw new IOException("Payload too large: " + payloadLength);
}
if (payload.capacity() < payloadLength) {
payload = ByteBuffer.allocate(payloadLength);
buffers[1] = payload;
}
}
}
header.flip();
payload.flip();
return Message.parse(buffers);
}
private static long getTotalSize(ByteBuffer[] buffers) {
long total = 0;
for (ByteBuffer buffer : buffers) {
total += buffer.remaining();
}
return total;
}
}

Example 2: File Format Parser

public class StructuredFileParser {
public static class FileStructure {
private final ByteBuffer magicNumber;
private final ByteBuffer fileHeader;
private final ByteBuffer[] dataBlocks;
private final ByteBuffer fileFooter;
public FileStructure(ByteBuffer magic, ByteBuffer header, 
ByteBuffer[] data, ByteBuffer footer) {
this.magicNumber = magic;
this.fileHeader = header;
this.dataBlocks = data;
this.fileFooter = footer;
}
public void printStructure() {
System.out.println("Magic: " + new String(magicNumber.array()).trim());
System.out.println("Header size: " + fileHeader.remaining());
System.out.println("Data blocks: " + dataBlocks.length);
System.out.println("Footer: " + new String(fileFooter.array()).trim());
}
}
public static FileStructure parseStructuredFile(String filename) throws Exception {
try (RandomAccessFile file = new RandomAccessFile(filename, "r");
FileChannel channel = file.getChannel()) {
// Read magic number (first 8 bytes)
ByteBuffer magicBuffer = ByteBuffer.allocate(8);
channel.read(magicBuffer);
magicBuffer.flip();
// Read file header (next 256 bytes)
ByteBuffer headerBuffer = ByteBuffer.allocate(256);
channel.read(headerBuffer);
headerBuffer.flip();
// Parse header to determine data structure
int numBlocks = headerBuffer.getInt(16); // Assume block count at position 16
int blockSize = headerBuffer.getInt(20); // Assume block size at position 20
// Read data blocks
ByteBuffer[] dataBlocks = new ByteBuffer[numBlocks];
for (int i = 0; i < numBlocks; i++) {
dataBlocks[i] = ByteBuffer.allocate(blockSize);
}
// Scattering read for all data blocks
long dataBytesRead = channel.read(dataBlocks);
System.out.println("Data bytes read: " + dataBytesRead);
for (ByteBuffer block : dataBlocks) {
block.flip();
}
// Read footer (last 128 bytes)
ByteBuffer footerBuffer = ByteBuffer.allocate(128);
channel.read(footerBuffer);
footerBuffer.flip();
return new FileStructure(magicBuffer, headerBuffer, dataBlocks, footerBuffer);
}
}
public static void createStructuredFile(String filename) throws Exception {
try (RandomAccessFile file = new RandomAccessFile(filename, "rw");
FileChannel channel = file.getChannel()) {
ByteBuffer magic = ByteBuffer.wrap("MYFILEv1".getBytes());
ByteBuffer header = ByteBuffer.allocate(256);
header.putInt(1);    // Version
header.putInt(5);    // Number of data blocks
header.putInt(512);  // Block size
header.putLong(System.currentTimeMillis()); // Timestamp
header.flip();
// Create data blocks
ByteBuffer[] dataBlocks = new ByteBuffer[5];
for (int i = 0; i < dataBlocks.length; i++) {
dataBlocks[i] = ByteBuffer.allocate(512);
String content = "Data Block " + (i + 1) + " - " + 
"Created at: " + System.currentTimeMillis();
dataBlocks[i].put(content.getBytes());
dataBlocks[i].flip();
}
ByteBuffer footer = ByteBuffer.wrap("END_OF_FILE".getBytes());
// Gathering write of all components
ByteBuffer[] allBuffers = new ByteBuffer[2 + dataBlocks.length + 1];
allBuffers[0] = magic;
allBuffers[1] = header;
System.arraycopy(dataBlocks, 0, allBuffers, 2, dataBlocks.length);
allBuffers[allBuffers.length - 1] = footer;
long totalWritten = channel.write(allBuffers);
System.out.println("Total bytes written: " + totalWritten);
}
}
public static void main(String[] args) throws Exception {
createStructuredFile("structured.dat");
FileStructure structure = parseStructuredFile("structured.dat");
structure.printStructure();
}
}

Example 3: HTTP-like Response Builder

public class HttpResponseBuilder {
public static class HttpResponse {
private final String statusLine;
private final Map<String, String> headers;
private final ByteBuffer body;
public HttpResponse(String statusLine, Map<String, String> headers, ByteBuffer body) {
this.statusLine = statusLine;
this.headers = headers;
this.body = body;
}
public ByteBuffer[] toBuffers() {
List<ByteBuffer> buffers = new ArrayList<>();
// Status line
buffers.add(ByteBuffer.wrap((statusLine + "\r\n").getBytes()));
// Headers
for (Map.Entry<String, String> header : headers.entrySet()) {
String headerLine = header.getKey() + ": " + header.getValue() + "\r\n";
buffers.add(ByteBuffer.wrap(headerLine.getBytes()));
}
// Empty line separating headers and body
buffers.add(ByteBuffer.wrap("\r\n".getBytes()));
// Body
buffers.add(body.slice());
return buffers.toArray(new ByteBuffer[0]);
}
}
public static void sendHttpResponse(GatheringByteChannel channel, HttpResponse response) 
throws Exception {
ByteBuffer[] buffers = response.toBuffers();
long totalWritten = 0;
long totalToWrite = calculateTotalSize(buffers);
while (totalWritten < totalToWrite) {
long written = channel.write(buffers);
if (written == -1) {
throw new IOException("Channel closed while writing");
}
totalWritten += written;
}
}
public static HttpResponse createSuccessResponse(String content) {
String statusLine = "HTTP/1.1 200 OK";
Map<String, String> headers = new LinkedHashMap<>();
headers.put("Content-Type", "text/html; charset=UTF-8");
headers.put("Content-Length", String.valueOf(content.length()));
headers.put("Server", "JavaScatterGather/1.0");
headers.put("Connection", "close");
ByteBuffer body = ByteBuffer.wrap(content.getBytes());
return new HttpResponse(statusLine, headers, body);
}
public static HttpResponse createErrorResponse(int code, String message) {
String statusLine = "HTTP/1.1 " + code + " " + message;
Map<String, String> headers = new LinkedHashMap<>();
String errorContent = "<html><body><h1>Error " + code + "</h1><p>" + message + "</p></body></html>";
headers.put("Content-Type", "text/html; charset=UTF-8");
headers.put("Content-Length", String.valueOf(errorContent.length()));
headers.put("Connection", "close");
ByteBuffer body = ByteBuffer.wrap(errorContent.getBytes());
return new HttpResponse(statusLine, headers, body);
}
private static long calculateTotalSize(ByteBuffer[] buffers) {
long total = 0;
for (ByteBuffer buffer : buffers) {
total += buffer.remaining();
}
return total;
}
}
// Usage example with socket channel
class HttpServerExample {
public static void main(String[] args) throws Exception {
// Simulate HTTP response
String htmlContent = "<html><body><h1>Hello, Scatter/Gather!</h1></body></html>";
HttpResponse response = HttpResponseBuilder.createSuccessResponse(htmlContent);
// In a real server, you'd get the channel from ServerSocketChannel.accept()
try (java.nio.channels.SocketChannel socketChannel = 
java.nio.channels.SocketChannel.open()) {
// Connect to somewhere (in real usage, this would be the accepted connection)
// socketChannel.connect(new InetSocketAddress("localhost", 8080));
// Send the response using gathering write
// HttpResponseBuilder.sendHttpResponse(socketChannel, response);
}
// Print the response structure
ByteBuffer[] buffers = response.toBuffers();
System.out.println("Response consists of " + buffers.length + " buffers");
for (int i = 0; i < buffers.length; i++) {
System.out.printf("Buffer %d: %d bytes%n", i, buffers[i].remaining());
}
}
}

Advanced Techniques

1. Dynamic Buffer Allocation

public class DynamicScatterGather {
public static class BufferPool {
private final Queue<ByteBuffer> bufferPool = new LinkedList<>();
private final int defaultBufferSize;
public BufferPool(int defaultBufferSize) {
this.defaultBufferSize = defaultBufferSize;
}
public ByteBuffer getBuffer() {
ByteBuffer buffer = bufferPool.poll();
if (buffer == null) {
buffer = ByteBuffer.allocate(defaultBufferSize);
}
buffer.clear();
return buffer;
}
public void returnBuffer(ByteBuffer buffer) {
bufferPool.offer(buffer);
}
}
public static List<ByteBuffer> readVariableLengthData(ScatteringByteChannel channel, 
BufferPool bufferPool) 
throws Exception {
List<ByteBuffer> buffers = new ArrayList<>();
long totalRead = 0;
boolean moreData = true;
while (moreData) {
ByteBuffer buffer = bufferPool.getBuffer();
long bytesRead = channel.read(new ByteBuffer[]{buffer});
if (bytesRead == -1) {
moreData = false;
bufferPool.returnBuffer(buffer); // Return unused buffer
} else if (bytesRead > 0) {
buffer.flip();
buffers.add(buffer);
totalRead += bytesRead;
// If buffer wasn't filled, we've reached end of data
if (bytesRead < buffer.capacity()) {
moreData = false;
}
}
}
System.out.println("Read " + totalRead + " bytes in " + buffers.size() + " buffers");
return buffers;
}
public static void writeVariableLengthData(GatheringByteChannel channel, 
List<ByteBuffer> buffers) 
throws Exception {
ByteBuffer[] bufferArray = buffers.toArray(new ByteBuffer[0]);
long totalWritten = 0;
long totalToWrite = calculateTotalSize(bufferArray);
while (totalWritten < totalToWrite) {
long written = channel.write(bufferArray);
if (written == -1) {
throw new IOException("Channel closed during write");
}
totalWritten += written;
}
System.out.println("Written " + totalWritten + " bytes from " + buffers.size() + " buffers");
}
}

2. Performance Comparison

public class ScatterGatherPerformance {
public static long testTraditionalIO(String filename, int bufferSize, int iterations) 
throws Exception {
long startTime = System.nanoTime();
byte[] data = new byte[bufferSize];
Arrays.fill(data, (byte) 'X');
try (FileOutputStream fos = new FileOutputStream(filename)) {
for (int i = 0; i < iterations; i++) {
fos.write(data);
}
}
return System.nanoTime() - startTime;
}
public static long testScatterGatherIO(String filename, int bufferSize, int iterations) 
throws Exception {
long startTime = System.nanoTime();
try (RandomAccessFile file = new RandomAccessFile(filename, "rw");
FileChannel channel = file.getChannel()) {
// Create multiple smaller buffers
int numBuffers = 4;
int smallBufferSize = bufferSize / numBuffers;
ByteBuffer[] buffers = new ByteBuffer[numBuffers];
for (int i = 0; i < numBuffers; i++) {
buffers[i] = ByteBuffer.allocate(smallBufferSize);
for (int j = 0; j < smallBufferSize; j++) {
buffers[i].put((byte) ('A' + (i % 26)));
}
buffers[i].flip();
}
for (int i = 0; i < iterations; i++) {
channel.write(buffers);
// Reset buffer positions for next write
for (ByteBuffer buffer : buffers) {
buffer.rewind();
}
}
}
return System.nanoTime() - startTime;
}
public static void main(String[] args) throws Exception {
int bufferSize = 4096; // 4KB
int iterations = 1000;
long traditionalTime = testTraditionalIO("traditional.dat", bufferSize, iterations);
long scatterGatherTime = testScatterGatherIO("scatter.dat", bufferSize, iterations);
System.out.printf("Traditional I/O: %d ns%n", traditionalTime);
System.out.printf("Scatter/Gather I/O: %d ns%n", scatterGatherTime);
System.out.printf("Ratio: %.2f%n", (double) traditionalTime / scatterGatherTime);
}
}

Best Practices

  1. Buffer Sizing: Choose appropriate buffer sizes for your use case
  2. Buffer Reuse: Reuse buffers when possible to reduce GC pressure
  3. Error Handling: Always check return values from read/write operations
  4. Resource Management: Use try-with-resources for channel management
  5. Buffer Position Management: Properly flip/rewind buffers between operations
  6. Concurrency: Be cautious with buffer sharing between threads

Common Use Cases

  • Network Protocols: HTTP, custom protocols with headers and payloads
  • File Formats: Structured files with headers, data blocks, and footers
  • Database Systems: Reading/writing records with different field types
  • Message Queues: Handling messages with headers and payloads
  • Stream Processing: Processing data in chunks with different processing requirements

Scatter/Gather I/O provides significant performance benefits and cleaner code organization when dealing with structured data that naturally fits into multiple buffers.

Category : JAVA

Leave a Reply

Your email address will not be published. Required fields are marked *


Macro Nepal Helper