Browse Source
When deserializing KRPC (which is used for RPCs sent to Kafka, Kafka Metadata records, and some
other things), check that we have at least N bytes remaining before allocating an array of size N.
Remove DataInputStreamReadable since it was hard to make this class aware of how many bytes were
remaining. Instead, when reading an individual record in the Raft layer, simply create a
ByteBufferAccessor with a ByteBuffer containing just the bytes we're interested in.
Add SimpleArraysMessageTest and ByteBufferAccessorTest. Also add some additional tests in
RequestResponseTest.
Reviewers: Tom Bentley <tbentley@redhat.com>, Mickael Maison <mickael.maison@gmail.com>, Colin McCabe <colin@cmccabe.xyz>
Co-authored-by: Colin McCabe <colin@cmccabe.xyz>
Co-authored-by: Manikumar Reddy <manikumar.reddy@gmail.com>
Co-authored-by: Mickael Maison <mickael.maison@gmail.com>
pull/12302/merge
Colin Patrick McCabe
3 years ago
committed by
Manikumar Reddy
16 changed files with 433 additions and 188 deletions
@ -1,139 +0,0 @@
@@ -1,139 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.kafka.common.protocol; |
||||
|
||||
import org.apache.kafka.common.utils.ByteUtils; |
||||
|
||||
import java.io.Closeable; |
||||
import java.io.DataInputStream; |
||||
import java.io.IOException; |
||||
import java.nio.ByteBuffer; |
||||
|
||||
public class DataInputStreamReadable implements Readable, Closeable { |
||||
protected final DataInputStream input; |
||||
|
||||
public DataInputStreamReadable(DataInputStream input) { |
||||
this.input = input; |
||||
} |
||||
|
||||
@Override |
||||
public byte readByte() { |
||||
try { |
||||
return input.readByte(); |
||||
} catch (IOException e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public short readShort() { |
||||
try { |
||||
return input.readShort(); |
||||
} catch (IOException e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public int readInt() { |
||||
try { |
||||
return input.readInt(); |
||||
} catch (IOException e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public long readLong() { |
||||
try { |
||||
return input.readLong(); |
||||
} catch (IOException e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public double readDouble() { |
||||
try { |
||||
return input.readDouble(); |
||||
} catch (IOException e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public void readArray(byte[] arr) { |
||||
try { |
||||
input.readFully(arr); |
||||
} catch (IOException e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public int readUnsignedVarint() { |
||||
try { |
||||
return ByteUtils.readUnsignedVarint(input); |
||||
} catch (IOException e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public ByteBuffer readByteBuffer(int length) { |
||||
byte[] arr = new byte[length]; |
||||
readArray(arr); |
||||
return ByteBuffer.wrap(arr); |
||||
} |
||||
|
||||
@Override |
||||
public int readVarint() { |
||||
try { |
||||
return ByteUtils.readVarint(input); |
||||
} catch (IOException e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public long readVarlong() { |
||||
try { |
||||
return ByteUtils.readVarlong(input); |
||||
} catch (IOException e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public int remaining() { |
||||
try { |
||||
return input.available(); |
||||
} catch (IOException e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public void close() { |
||||
try { |
||||
input.close(); |
||||
} catch (IOException e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,54 @@
@@ -0,0 +1,54 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.kafka.common.message; |
||||
|
||||
import org.apache.kafka.common.protocol.ByteBufferAccessor; |
||||
import org.junit.jupiter.api.Test; |
||||
|
||||
import java.nio.ByteBuffer; |
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals; |
||||
import static org.junit.jupiter.api.Assertions.assertThrows; |
||||
|
||||
public class SimpleArraysMessageTest { |
||||
@Test |
||||
public void testArrayBoundsChecking() { |
||||
// SimpleArraysMessageData takes 2 arrays
|
||||
final ByteBuffer buf = ByteBuffer.wrap(new byte[] { |
||||
(byte) 0x7f, // Set size of first array to 126 which is larger than the size of this buffer
|
||||
(byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00 |
||||
}); |
||||
final SimpleArraysMessageData out = new SimpleArraysMessageData(); |
||||
ByteBufferAccessor accessor = new ByteBufferAccessor(buf); |
||||
assertEquals("Tried to allocate a collection of size 126, but there are only 7 bytes remaining.", |
||||
assertThrows(RuntimeException.class, () -> out.read(accessor, (short) 2)).getMessage()); |
||||
} |
||||
|
||||
@Test |
||||
public void testArrayBoundsCheckingOtherArray() { |
||||
// SimpleArraysMessageData takes 2 arrays
|
||||
final ByteBuffer buf = ByteBuffer.wrap(new byte[] { |
||||
(byte) 0x01, // Set size of first array to 0
|
||||
(byte) 0x7e, // Set size of second array to 125 which is larger than the size of this buffer
|
||||
(byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00 |
||||
}); |
||||
final SimpleArraysMessageData out = new SimpleArraysMessageData(); |
||||
ByteBufferAccessor accessor = new ByteBufferAccessor(buf); |
||||
assertEquals("Tried to allocate a collection of size 125, but there are only 6 bytes remaining.", |
||||
assertThrows(RuntimeException.class, () -> out.read(accessor, (short) 2)).getMessage()); |
||||
} |
||||
} |
||||
@ -0,0 +1,58 @@
@@ -0,0 +1,58 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.kafka.common.protocol; |
||||
|
||||
import org.junit.jupiter.api.Test; |
||||
|
||||
import java.nio.ByteBuffer; |
||||
import java.nio.charset.StandardCharsets; |
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals; |
||||
import static org.junit.jupiter.api.Assertions.assertEquals; |
||||
import static org.junit.jupiter.api.Assertions.assertThrows; |
||||
|
||||
public class ByteBufferAccessorTest { |
||||
@Test |
||||
public void testReadArray() { |
||||
ByteBuffer buf = ByteBuffer.allocate(1024); |
||||
ByteBufferAccessor accessor = new ByteBufferAccessor(buf); |
||||
final byte[] testArray = new byte[] {0x4b, 0x61, 0x46}; |
||||
accessor.writeByteArray(testArray); |
||||
accessor.writeInt(12345); |
||||
accessor.flip(); |
||||
final byte[] testArray2 = accessor.readArray(3); |
||||
assertArrayEquals(testArray, testArray2); |
||||
assertEquals(12345, accessor.readInt()); |
||||
assertEquals("Error reading byte array of 3 byte(s): only 0 byte(s) available", |
||||
assertThrows(RuntimeException.class, |
||||
() -> accessor.readArray(3)).getMessage()); |
||||
} |
||||
|
||||
@Test |
||||
public void testReadString() { |
||||
ByteBuffer buf = ByteBuffer.allocate(1024); |
||||
ByteBufferAccessor accessor = new ByteBufferAccessor(buf); |
||||
String testString = "ABC"; |
||||
final byte[] testArray = testString.getBytes(StandardCharsets.UTF_8); |
||||
accessor.writeByteArray(testArray); |
||||
accessor.flip(); |
||||
assertEquals("ABC", accessor.readString(3)); |
||||
assertEquals("Error reading byte array of 2 byte(s): only 0 byte(s) available", |
||||
assertThrows(RuntimeException.class, |
||||
() -> accessor.readString(2)).getMessage()); |
||||
} |
||||
} |
||||
@ -0,0 +1,29 @@
@@ -0,0 +1,29 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one or more |
||||
// contributor license agreements. See the NOTICE file distributed with |
||||
// this work for additional information regarding copyright ownership. |
||||
// The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
// (the "License"); you may not use this file except in compliance with |
||||
// the License. You may obtain a copy of the License at |
||||
// |
||||
// http://www.apache.org/licenses/LICENSE-2.0 |
||||
// |
||||
// Unless required by applicable law or agreed to in writing, software |
||||
// distributed under the License is distributed on an "AS IS" BASIS, |
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
// See the License for the specific language governing permissions and |
||||
// limitations under the License. |
||||
{ |
||||
"name": "SimpleArraysMessage", |
||||
"type": "header", |
||||
"validVersions": "0-2", |
||||
"flexibleVersions": "1+", |
||||
"fields": [ |
||||
{ "name": "Goats", "type": "[]StructArray", "versions": "1+", |
||||
"fields": [ |
||||
{ "name": "Color", "type": "int8", "versions": "1+"}, |
||||
{ "name": "Name", "type": "string", "versions": "2+"} |
||||
] |
||||
}, |
||||
{ "name": "Sheep", "type": "[]int32", "versions": "0+" } |
||||
] |
||||
} |
||||
Loading…
Reference in new issue