11package com .maxmind .db ;
22
33import com .maxmind .db .Reader .FileMode ;
4+ import java .io .ByteArrayOutputStream ;
45import java .io .File ;
56import java .io .IOException ;
67import java .io .InputStream ;
@@ -13,6 +14,10 @@ final class BufferHolder {
1314 // DO NOT PASS OUTSIDE THIS CLASS. Doing so will remove thread safety.
1415 private final Buffer buffer ;
1516
17+ // Reasonable I/O buffer size for reading from InputStream.
18+ // This is separate from chunk size which determines MultiBuffer chunk allocation.
19+ private static final int IO_BUFFER_SIZE = 16 * 1024 ; // 16KB
20+
1621 BufferHolder (File database , FileMode mode ) throws IOException {
1722 this (database , mode , MultiBuffer .DEFAULT_CHUNK_SIZE );
1823 }
@@ -78,29 +83,49 @@ final class BufferHolder {
7883 if (null == stream ) {
7984 throw new NullPointerException ("Unable to use a NULL InputStream" );
8085 }
81- var chunks = new ArrayList <ByteBuffer >();
82- var total = 0L ;
83- var tmp = new byte [chunkSize ];
86+
87+ // Read data from the stream in chunks to support databases >2GB.
88+ // Invariant: All chunks except the last are exactly chunkSize bytes.
89+ var chunks = new ArrayList <byte []>();
90+ var currentChunkStream = new ByteArrayOutputStream ();
91+ var tmp = new byte [IO_BUFFER_SIZE ];
8492 int read ;
8593
8694 while (-1 != (read = stream .read (tmp ))) {
87- var chunk = ByteBuffer .allocate (read );
88- chunk .put (tmp , 0 , read );
89- chunk .flip ();
90- chunks .add (chunk );
91- total += read ;
92- }
95+ var offset = 0 ;
96+ while (offset < read ) {
97+ var spaceInCurrentChunk = chunkSize - currentChunkStream .size ();
98+ var toWrite = Math .min (spaceInCurrentChunk , read - offset );
9399
94- if (total <= chunkSize ) {
95- var data = new byte [(int ) total ];
96- var pos = 0 ;
97- for (var chunk : chunks ) {
98- System .arraycopy (chunk .array (), 0 , data , pos , chunk .capacity ());
99- pos += chunk .capacity ();
100+ currentChunkStream .write (tmp , offset , toWrite );
101+ offset += toWrite ;
102+
103+ // When chunk is exactly full, save it and start a new one.
104+ // This guarantees all non-final chunks are exactly chunkSize.
105+ if (currentChunkStream .size () == chunkSize ) {
106+ chunks .add (currentChunkStream .toByteArray ());
107+ currentChunkStream = new ByteArrayOutputStream ();
108+ }
100109 }
101- this .buffer = SingleBuffer .wrap (data );
110+ }
111+
112+ // Handle last partial chunk (could be empty if total is multiple of chunkSize)
113+ if (currentChunkStream .size () > 0 ) {
114+ chunks .add (currentChunkStream .toByteArray ());
115+ }
116+
117+ if (chunks .size () == 1 ) {
118+ // For databases that fit in a single chunk, use SingleBuffer
119+ this .buffer = SingleBuffer .wrap (chunks .get (0 ));
102120 } else {
103- this .buffer = new MultiBuffer (chunks .toArray (new ByteBuffer [0 ]), chunkSize );
121+ // For large databases, wrap chunks in ByteBuffers and use MultiBuffer
122+ // Guaranteed: chunks[0..n-2] all have length == chunkSize
123+ // chunks[n-1] may have length < chunkSize
124+ var buffers = new ByteBuffer [chunks .size ()];
125+ for (var i = 0 ; i < chunks .size (); i ++) {
126+ buffers [i ] = ByteBuffer .wrap (chunks .get (i ));
127+ }
128+ this .buffer = new MultiBuffer (buffers , chunkSize );
104129 }
105130 }
106131
0 commit comments