Skip to content

ORC-2049: Move MAX_ARRAY_SIZE to RecordReaderUtils from IOUtils #2472

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
dongjoon-hyun wants to merge 1 commit into from
+20 −19
Closed

ORC-2049: Move MAX_ARRAY_SIZE to RecordReaderUtils from IOUtils #2472

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions java/core/src/java/org/apache/orc/impl/IOUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,6 @@ public final class IOUtils {

public static final int DEFAULT_BUFFER_SIZE = 8192;

/**
* The maximum size of array to allocate, value being the same as {@link java.util.Hashtable}
*/
public static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8;

/**
* Returns a new byte array of size {@link #DEFAULT_BUFFER_SIZE}.
*
Expand Down
13 changes: 9 additions & 4 deletions java/core/src/java/org/apache/orc/impl/RecordReaderUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@
* Stateless methods shared between RecordReaderImpl and EncodedReaderImpl.
*/
public class RecordReaderUtils {
/**
* The maximum size of array to allocate, value being the same as {@link java.util.Hashtable}
*/
public static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8;

private static final HadoopShims SHIMS = HadoopShimsFactory.get();
private static final boolean supportVectoredIO =
SHIMS.supportVectoredIO(VersionInfo.getVersion());
Expand Down Expand Up @@ -836,12 +841,12 @@ static ChunkReader create(BufferChunk from, BufferChunk to) {
current = (BufferChunk) current.next;
}
reqBytes += currentEnd - currentStart;
if (reqBytes > IOUtils.MAX_ARRAY_SIZE) {
throw new IllegalArgumentException("invalid reqBytes value " + reqBytes + ",out of bounds " + IOUtils.MAX_ARRAY_SIZE);
if (reqBytes > MAX_ARRAY_SIZE) {
throw new IllegalArgumentException("invalid reqBytes value " + reqBytes + ",out of bounds " + MAX_ARRAY_SIZE);
}
long readBytes = end - start;
if (readBytes > IOUtils.MAX_ARRAY_SIZE) {
throw new IllegalArgumentException("invalid readBytes value " + readBytes + ",out of bounds " + IOUtils.MAX_ARRAY_SIZE);
if (readBytes > MAX_ARRAY_SIZE) {
throw new IllegalArgumentException("invalid readBytes value " + readBytes + ",out of bounds " + MAX_ARRAY_SIZE);
}
return new ChunkReader(from, to, (int) readBytes, (int) reqBytes);
}
Expand Down
21 changes: 11 additions & 10 deletions java/core/src/test/org/apache/orc/impl/TestRecordReaderUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,30 +209,31 @@ public void testChunkReaderCreateOffsetExceedsMaxInt() {

@Test
public void testChunkReaderCreateReqBytesAndReadBytesValidation() {
int MAX_ARRAY_SIZE = RecordReaderUtils.MAX_ARRAY_SIZE;
BufferChunkList rangeList = new TestOrcLargeStripe.RangeBuilder()
.range(0, IOUtils.MAX_ARRAY_SIZE)
.range(1L + IOUtils.MAX_ARRAY_SIZE, IOUtils.MAX_ARRAY_SIZE + 1)
.range(2L * IOUtils.MAX_ARRAY_SIZE, IOUtils.MAX_ARRAY_SIZE - 4)
.range(3L * IOUtils.MAX_ARRAY_SIZE, 2)
.range(0, MAX_ARRAY_SIZE)
.range(1L + MAX_ARRAY_SIZE, MAX_ARRAY_SIZE + 1)
.range(2L * MAX_ARRAY_SIZE, MAX_ARRAY_SIZE - 4)
.range(3L * MAX_ARRAY_SIZE, 2)
.build();

// reqBytes,readBytes boundary value
RecordReaderUtils.ChunkReader chunkReader = RecordReaderUtils.ChunkReader.create(rangeList.get(0), 0);
assertEquals(chunkReader.getReadBytes(), IOUtils.MAX_ARRAY_SIZE);
assertEquals(chunkReader.getReqBytes(), IOUtils.MAX_ARRAY_SIZE);
assertEquals(chunkReader.getReadBytes(), MAX_ARRAY_SIZE);
assertEquals(chunkReader.getReqBytes(), MAX_ARRAY_SIZE);

// reqBytes > IOUtils.MAX_ARRAY_SIZE validation
// reqBytes > MAX_ARRAY_SIZE validation
assertThrowsExactly(IllegalArgumentException.class,
() -> RecordReaderUtils.ChunkReader.create(rangeList.get(1), 0),
() -> String.format("invalid reqBytes value %d,out of bounds %d",
rangeList.get(1).getLength(), IOUtils.MAX_ARRAY_SIZE)
rangeList.get(1).getLength(), MAX_ARRAY_SIZE)
);

// readBytes > IOUtils.MAX_ARRAY_SIZE validation
// readBytes > MAX_ARRAY_SIZE validation
assertThrowsExactly(IllegalArgumentException.class,
() -> RecordReaderUtils.ChunkReader.create(rangeList.get(2), 100),
() -> String.format("invalid readBytes value %d,out of bounds %d",
rangeList.get(3).getEnd() - rangeList.get(2).getOffset(), IOUtils.MAX_ARRAY_SIZE)
rangeList.get(3).getEnd() - rangeList.get(2).getOffset(), MAX_ARRAY_SIZE)
);
}

Expand Down
Loading