Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 140 additions & 5 deletions src/main/java/org/xerial/snappy/Snappy.java
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,8 @@ public static int compress(byte[] input, int inputOffset, int inputLength, byte[
* @param compressed output of the compressed data. Uses range [pos()..].
* @return byte size of the compressed data.
* @throws SnappyError when the input is not a direct buffer
* @throws IllegalArgumentException when the output buffer has less than {@link #maxCompressedLength(int)} bytes
* remaining
*/
public static int compress(ByteBuffer uncompressed, ByteBuffer compressed)
throws IOException
Expand All @@ -150,6 +152,7 @@ public static int compress(ByteBuffer uncompressed, ByteBuffer compressed)
int uPos = uncompressed.position();
int uLen = uncompressed.remaining();
int cPos = compressed.position();
checkOutputSpace(compressed.remaining(), 0, requiredCompressedLength(uLen));
int compressedSize = impl.rawCompress(uncompressed, uPos, uLen, compressed,
cPos);

Expand Down Expand Up @@ -354,6 +357,7 @@ public static boolean isValidCompressedBuffer(byte[] input, int offset, int leng
if (input == null) {
throw new NullPointerException("input is null");
}
checkArrayRange(input, offset, length, "input");
return impl.isValidCompressedBuffer(input, offset, length);
}

Expand Down Expand Up @@ -446,7 +450,11 @@ public static long rawUncompress(long inputAddr, long inputSize, long destAddr)
public static byte[] rawCompress(Object data, int byteSize)
throws IOException
{
byte[] buf = new byte[Snappy.maxCompressedLength(byteSize)];
if (data == null) {
throw new NullPointerException("input is null");
}
checkArrayRange(data, 0, byteSize, "input");
byte[] buf = new byte[requiredCompressedLength(byteSize)];
int compressedByteSize = impl.rawCompress(data, 0, byteSize, buf, 0);
byte[] result = new byte[compressedByteSize];
System.arraycopy(buf, 0, result, 0, compressedByteSize);
Expand All @@ -464,13 +472,17 @@ public static byte[] rawCompress(Object data, int byteSize)
* @param outputOffset byte offset at the output array
* @return byte size of the compressed data
* @throws IOException
* @throws IllegalArgumentException if the input range is out of bounds, or the output array does not have
* {@link #maxCompressedLength(int)} bytes available after outputOffset
*/
public static int rawCompress(Object input, int inputOffset, int inputLength, byte[] output, int outputOffset)
throws IOException
{
if (input == null || output == null) {
throw new NullPointerException("input or output is null");
}
checkArrayRange(input, inputOffset, inputLength, "input");
checkOutputSpace(output.length, outputOffset, requiredCompressedLength(inputLength));

int compressedSize = impl
.rawCompress(input, inputOffset, inputLength, output, outputOffset);
Expand All @@ -494,16 +506,127 @@ public static int rawCompress(Object input, int inputOffset, int inputLength, by
* @param outputOffset byte offset in the output buffer
* @return the byte size of the uncompressed data
* @throws IOException when failed to uncompress the input data
* @throws IllegalArgumentException if the input range is out of bounds, or the output array does not have
* enough space after outputOffset for the uncompressed data
*/
public static int rawUncompress(byte[] input, int inputOffset, int inputLength, Object output, int outputOffset)
throws IOException
{
if (input == null || output == null) {
throw new NullPointerException("input or output is null");
}
// uncompressedLength validates the input range
int requiredSize = uncompressedLength(input, inputOffset, inputLength);
checkOutputSpace(byteLengthOf(output), outputOffset, requiredSize);
return impl.rawUncompress(input, inputOffset, inputLength, output, outputOffset);
}

/**
* Returns the byte size of the given primitive array
*/
private static long byteLengthOf(Object array)
{
if (array instanceof byte[]) {
return ((byte[]) array).length;
}
if (array instanceof boolean[]) {
return ((boolean[]) array).length;
}
if (array instanceof char[]) {
return 2L * ((char[]) array).length;
}
if (array instanceof short[]) {
return 2L * ((short[]) array).length;
}
if (array instanceof int[]) {
return 4L * ((int[]) array).length;
}
if (array instanceof float[]) {
return 4L * ((float[]) array).length;
}
if (array instanceof long[]) {
return 8L * ((long[]) array).length;
}
if (array instanceof double[]) {
return 8L * ((double[]) array).length;
}
throw new IllegalArgumentException("not a primitive array: " + array.getClass().getName());
}

/**
* Ensures the byte range [offset, offset+length) lies within the given primitive array, since the native code
* accesses the array through a raw pointer without any bounds check.
*/
private static void checkArrayRange(Object array, int offset, int length, String name)
{
long size = byteLengthOf(array);
if (offset < 0 || length < 0 || (long) offset + length > size) {
throw new IllegalArgumentException(String.format(
"%s range [offset:%,d, length:%,d] is out of bounds for %s of %,d bytes",
name, offset, length, name, size));
}
}

/**
* Ensures the output has at least requiredSize bytes available after outputOffset, since the native code writes
* the output through a raw pointer without any bounds check.
*/
private static void checkOutputSpace(long outputSize, long outputOffset, long requiredSize)
{
if (outputOffset < 0 || outputOffset > outputSize) {
throw new IllegalArgumentException(String.format(
"output offset %,d is out of bounds for output of %,d bytes", outputOffset, outputSize));
}
if (outputSize - outputOffset < requiredSize) {
throw new IllegalArgumentException(String.format(
"not enough space for output: need %,d bytes, but only %,d remaining",
requiredSize, outputSize - outputOffset));
}
}

/**
* Returns the output buffer size required by the native compressor for the given input size.
*/
private static int requiredCompressedLength(int inputLength)
{
int maxCompressedLength = maxCompressedLength(inputLength);
// maxCompressedLength overflows int for inputs close to 2GB
if (maxCompressedLength < 0) {
throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input is too large to compress: " + inputLength);
}
return maxCompressedLength;
}

/**
* Rejects an uncompressed length declared in a compressed stream header that no valid Snappy stream of the
* given compressed size can produce. The densest Snappy element is a 3-byte copy that expands to 64 bytes, so
* a valid stream never declares more than 64/3 times its compressed size. This prevents allocating huge (or
* negative-size) buffers from a few bytes of crafted input before decompression detects the corruption.
*/
private static long checkUncompressedLength(long uncompressedLength, long compressedLength)
throws IOException
{
if (uncompressedLength < 0 || uncompressedLength > compressedLength * 64 / 3) {
throw new SnappyIOException(SnappyErrorCode.PARSING_ERROR, String.format(
"invalid uncompressed length %,d declared in %,d bytes of compressed data",
uncompressedLength, compressedLength));
}
return uncompressedLength;
}

/**
* Rejects an uncompressed length that is not a multiple of the element size of the typed array to produce.
*/
private static void checkElementAlignment(int uncompressedLength, int elementSize)
throws IOException
{
if (uncompressedLength % elementSize != 0) {
throw new SnappyIOException(SnappyErrorCode.FAILED_TO_UNCOMPRESS, String.format(
"uncompressed length %,d is not a multiple of the element size %d",
uncompressedLength, elementSize));
}
}

/**
* High-level API for uncompressing the input byte array.
*
Expand Down Expand Up @@ -557,6 +680,8 @@ public static int uncompress(byte[] input, int inputOffset, int inputLength, byt
* @return uncompressed data size
* @throws IOException when failed to uncompress the given input
* @throws SnappyError when the input is not a direct buffer
* @throws IllegalArgumentException when the output buffer does not have enough space remaining for the
* uncompressed data
*/
public static int uncompress(ByteBuffer compressed, ByteBuffer uncompressed)
throws IOException
Expand All @@ -572,6 +697,7 @@ public static int uncompress(ByteBuffer compressed, ByteBuffer uncompressed)
int cPos = compressed.position();
int cLen = compressed.remaining();
int uPos = uncompressed.position();
checkOutputSpace(uncompressed.remaining(), 0, uncompressedLength(compressed));

// pos limit
// [ ......UUUUUU.........]
Expand Down Expand Up @@ -608,6 +734,7 @@ public static char[] uncompressCharArray(byte[] input, int offset, int length)
throws IOException
{
int uncompressedLength = Snappy.uncompressedLength(input, offset, length);
checkElementAlignment(uncompressedLength, 2);
char[] result = new char[uncompressedLength / 2];
impl.rawUncompress(input, offset, length, result, 0);
return result;
Expand Down Expand Up @@ -639,6 +766,7 @@ public static double[] uncompressDoubleArray(byte[] input, int offset, int lengt
throws IOException
{
int uncompressedLength = Snappy.uncompressedLength(input, offset, length);
checkElementAlignment(uncompressedLength, 8);
double[] result = new double[uncompressedLength / 8];
impl.rawUncompress(input, offset, length, result, 0);
return result;
Expand All @@ -656,7 +784,7 @@ public static double[] uncompressDoubleArray(byte[] input, int offset, int lengt
public static int uncompressedLength(byte[] input)
throws IOException
{
return impl.uncompressedLength(input, 0, input.length);
return uncompressedLength(input, 0, input.length);
}

/**
Expand All @@ -676,8 +804,9 @@ public static int uncompressedLength(byte[] input, int offset, int length)
if (input == null) {
throw new NullPointerException("input is null");
}
checkArrayRange(input, offset, length, "input");

return impl.uncompressedLength(input, offset, length);
return (int) checkUncompressedLength(impl.uncompressedLength(input, offset, length), length);
}

/**
Expand All @@ -697,7 +826,9 @@ public static int uncompressedLength(ByteBuffer compressed)
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer");
}

return impl.uncompressedLength(compressed, compressed.position(), compressed.remaining());
return (int) checkUncompressedLength(
impl.uncompressedLength(compressed, compressed.position(), compressed.remaining()),
compressed.remaining());
}

/**
Expand All @@ -712,7 +843,7 @@ public static int uncompressedLength(ByteBuffer compressed)
public static long uncompressedLength(long inputAddr, long len)
throws IOException
{
return impl.uncompressedLength(inputAddr, len);
return checkUncompressedLength(impl.uncompressedLength(inputAddr, len), len);
}

/**
Expand Down Expand Up @@ -741,6 +872,7 @@ public static float[] uncompressFloatArray(byte[] input, int offset, int length)
throws IOException
{
int uncompressedLength = Snappy.uncompressedLength(input, offset, length);
checkElementAlignment(uncompressedLength, 4);
float[] result = new float[uncompressedLength / 4];
impl.rawUncompress(input, offset, length, result, 0);
return result;
Expand Down Expand Up @@ -772,6 +904,7 @@ public static int[] uncompressIntArray(byte[] input, int offset, int length)
throws IOException
{
int uncompressedLength = Snappy.uncompressedLength(input, offset, length);
checkElementAlignment(uncompressedLength, 4);
int[] result = new int[uncompressedLength / 4];
impl.rawUncompress(input, offset, length, result, 0);
return result;
Expand Down Expand Up @@ -803,6 +936,7 @@ public static long[] uncompressLongArray(byte[] input, int offset, int length)
throws IOException
{
int uncompressedLength = Snappy.uncompressedLength(input, offset, length);
checkElementAlignment(uncompressedLength, 8);
long[] result = new long[uncompressedLength / 8];
impl.rawUncompress(input, offset, length, result, 0);
return result;
Expand Down Expand Up @@ -834,6 +968,7 @@ public static short[] uncompressShortArray(byte[] input, int offset, int length)
throws IOException
{
int uncompressedLength = Snappy.uncompressedLength(input, offset, length);
checkElementAlignment(uncompressedLength, 2);
short[] result = new short[uncompressedLength / 2];
impl.rawUncompress(input, offset, length, result, 0);
return result;
Expand Down
24 changes: 15 additions & 9 deletions src/main/java/org/xerial/snappy/SnappyFramedInputStream.java
Original file line number Diff line number Diff line change
Expand Up @@ -517,18 +517,22 @@ private boolean ensureBuffer()
return false;
}

if (!readBlockHeader()) {
eof = true;
return false;
}

// get action based on header
final FrameMetaData frameMetaData = getFrameMetaData(frameHeader);
// Skip chunks in a loop rather than recursively: a zero-length skippable chunk is only 4 bytes, so
// recursing per chunk lets a small input exhaust the stack
FrameMetaData frameMetaData;
while (true) {
if (!readBlockHeader()) {
eof = true;
return false;
}

if (FrameAction.SKIP == frameMetaData.frameAction) {
// get action based on header
frameMetaData = getFrameMetaData(frameHeader);
if (FrameAction.SKIP != frameMetaData.frameAction) {
break;
}
SnappyFramed.skip(rbc, frameMetaData.length,
ByteBuffer.wrap(buffer));
return ensureBuffer();
}

if (frameMetaData.length > input.capacity()) {
Expand All @@ -550,6 +554,8 @@ private boolean ensureBuffer()

input.position(frameData.offset);

// uncompressedLength rejects lengths that cannot be produced from the frame size, so the buffers
// allocated below stay proportional to the frame actually read
final int uncompressedLength = Snappy.uncompressedLength(input);

if (uncompressedLength > uncompressedDirect.capacity()) {
Expand Down
Loading
Loading