Java多线程读取大文件技巧

AsynchronousFileChannel channel = AsynchronousFileChannel.open(Paths.get("largefile.txt"), StandardOpenOption.READ);
ByteBuffer buffer = ByteBuffer.allocate(8192);
Future result = channel.read(buffer, 0);
// 非阻塞：可以继续做其他事
while (!result.isDone()) {
// 可执行其他任务
}
Integer bytesRead = result.get();

channel.read(buffer, 0, buffer, new CompletionHandler() {
    public void completed(Integer result, ByteBuffer attachment) {
        if (result == -1) {
            System.out.println("读取完成");
            return;
        }
        attachment.flip();
        // 处理数据...
        attachment.clear();
    }
public void failed(Throwable exc, ByteBuffer attachment) {
    exc.printStackTrace();
}
});

int corePoolSize = Runtime.getRuntime().availableProcessors() + 1;
ExecutorService executor = new ThreadPoolExecutor(
    corePoolSize,
    corePoolSize * 2,
    60L,
    TimeUnit.SECONDS,
    new LinkedBlockingQueue<>(100),
    new ThreadPoolTaskDecorator()
);

long chunkSize = 1024 * 1024;
long fileSize = channel.size();
for (long pos = 0; pos < fileSize; pos += chunkSize) {
long finalPos = pos;
executor.submit(() -> {
ByteBuffer buf = ByteBuffer.allocate((int) Math.min(chunkSize, fileSize - finalPos));
Future readOp = channel.read(buf, finalPos);
Integer n = readOp.get();
buf.flip();
// 处理该块数据
return n;
});
}

CompletableFuture.supplyAsync(() -> {
    // 异步读取某一块
    return readFileChunk(channel, position, size);
}, executor)
.thenApplyAsync(data -> processChunk(data), executor)
.thenAccept(result -> saveResult(result));