Skip to content

perf(decoder): avoid lots of calls to ensure data and field length/offset lookups #144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mkjois wants to merge 5 commits into
base: master
Choose a base branch
from
Open

perf(decoder): avoid lots of calls to ensure data and field length/offset lookups #144

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a638299
perf(decoder): save constant results for field length and offsets
Mar 22, 2023
b859f1e
perf(decoder): read bit sets in larger chunks
Mar 23, 2023
b34d05a
perf(decoder): do fewer reads for vendors and publisher restrictions
Mar 23, 2023
4116c98
style: tiny trailing whitespace removal
Mar 23, 2023
5340b30
refactor(decoder): only use powers of 2 for reading bit sets
Mar 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion iabtcf-decoder/src/main/java/com/iabtcf/decoder/TCStringDecoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ public static TCString decode(String consentString, DecoderOption... options)
case 1:
return TCStringV1.fromBitVector(bitVector);
case 2:
TCString tcString = null;
TCString tcString;
if (split.length > 1) {
BitReader[] remaining = new BitReader[split.length - 1];
for (int i = 1; i < split.length; i++) {
Expand Down
62 changes: 31 additions & 31 deletions iabtcf-decoder/src/main/java/com/iabtcf/decoder/TCStringV2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,20 +147,16 @@ public IntIterable getPubPurposesConsent() {
* @throws InvalidRangeFieldException
*/
static BitSetIntIterable fillVendors(BitReader bbv, FieldDefs maxVendor, FieldDefs vendorField) {
BitSet bs = new BitSet();

int maxV = bbv.readBits16(maxVendor);
boolean isRangeEncoding = bbv.readBits1(maxVendor.getEnd(bbv));
final int maxV = bbv.readBits16(maxVendor);
final boolean isRangeEncoding = bbv.readBits1(maxVendor.getEnd(bbv));
final int vendorFieldOffset = vendorField.getOffset(bbv);
BitSet bs = new BitSet();

if (isRangeEncoding) {
vendorIdsFromRange(bbv, bs, vendorField, Optional.of(maxVendor));
} else {
for (int i = 0; i < maxV; i++) {
boolean hasVendorConsent = bbv.readBits1(vendorField.getOffset(bbv) + i);
if (hasVendorConsent) {
bs.set(i + 1);
}
}
bs = bbv.readBitSet(vendorFieldOffset, maxV, 1);
}
return BitSetIntIterable.from(bs);
}
Expand All @@ -172,32 +168,36 @@ static BitSetIntIterable fillVendors(BitReader bbv, FieldDefs maxVendor, FieldDe
*/
static int vendorIdsFromRange(BitReader bbv, BitSet bs, int numberOfVendorEntriesOffset,
Optional<FieldDefs> maxVendor) {
int numberOfVendorEntries = bbv.readBits12(numberOfVendorEntriesOffset);

final int numberOfVendorEntries = bbv.readBits12(numberOfVendorEntriesOffset);
final int maxV = maxVendor.map(maxVF -> bbv.readBits16(maxVF)).orElse(Integer.MAX_VALUE);
final int vendorIdFieldLength = FieldDefs.START_OR_ONLY_VENDOR_ID.getLength(bbv);
final int vendorIdMask = 0xFFFFFFFF >>> (32 - vendorIdFieldLength);
int offset = numberOfVendorEntriesOffset + FieldDefs.NUM_ENTRIES.getLength(bbv);
int maxV = maxVendor.map(maxVF -> bbv.readBits16(maxVF)).orElse(Integer.MAX_VALUE);

for (int j = 0; j < numberOfVendorEntries; j++) {
boolean isRangeEntry = bbv.readBits1(offset++);
int startOrOnlyVendorId = bbv.readBits16(offset);
offset += FieldDefs.START_OR_ONLY_VENDOR_ID.getLength(bbv);
if (isRangeEntry) {
int endVendorId = bbv.readBits16(offset);
offset += FieldDefs.START_OR_ONLY_VENDOR_ID.getLength(bbv);
final int content = bbv.readBits32(offset);
Copy link
Collaborator

@laktech laktech Sep 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there is significant code here to eliminate a call to readBits16. I don't think it's worth it unless you can show a benchmark that it adds value.

final int startVendorId = (content >>> vendorIdFieldLength) & vendorIdMask;
final int endVendorId = content & vendorIdMask;
offset += 2 * vendorIdFieldLength;

if (startOrOnlyVendorId > endVendorId) {
if (startVendorId > endVendorId) {
throw new InvalidRangeFieldException(String.format(
"start vendor id (%d) is greater than endVendorId (%d)", startOrOnlyVendorId,
endVendorId));
"start vendor id (%d) is greater than endVendorId (%d)", startVendorId, endVendorId));
}

if (endVendorId > maxV) {
throw new InvalidRangeFieldException(
String.format("end vendor id (%d) is greater than max (%d)", endVendorId, maxV));
}

bs.set(startOrOnlyVendorId, endVendorId + 1);
bs.set(startVendorId, endVendorId + 1);
} else {
bs.set(startOrOnlyVendorId);
final int onlyVendorId = bbv.readBits16(offset);
offset += vendorIdFieldLength;
bs.set(onlyVendorId);
}
}

Expand All @@ -217,22 +217,22 @@ static void vendorIdsFromRange(BitReader bbv, BitSet bs, FieldDefs vendorField,
private int fillPublisherRestrictions(
List<PublisherRestriction> publisherRestrictions, int currentPointer, BitReader bitVector) {

int numberOfPublisherRestrictions = bitVector.readBits12(currentPointer);
final int numberOfPublisherRestrictions = bitVector.readBits12(currentPointer);
final int purposeIdFieldLength = FieldDefs.PURPOSE_ID.getLength(bitVector);
currentPointer += FieldDefs.NUM_ENTRIES.getLength(bitVector);

for (int i = 0; i < numberOfPublisherRestrictions; i++) {
int purposeId = bitVector.readBits6(currentPointer);
currentPointer += FieldDefs.PURPOSE_ID.getLength(bitVector);

int restrictionTypeId = bitVector.readBits2(currentPointer);
currentPointer += 2;
RestrictionType restrictionType = RestrictionType.from(restrictionTypeId);
final byte content = bitVector.readByteBits(currentPointer, 8);
final int purposeId = (content >>> 2) & (0xFF >>> (8 - purposeIdFieldLength));
final int restrictionTypeId = content & 0b11;
currentPointer += purposeIdFieldLength + 2;

BitSet bs = new BitSet();
final BitSet bs = new BitSet();
currentPointer = vendorIdsFromRange(bbv, bs, currentPointer, Optional.empty());
PublisherRestriction publisherRestriction =
new PublisherRestriction(purposeId, restrictionType, BitSetIntIterable.from(bs));
publisherRestrictions.add(publisherRestriction);
publisherRestrictions.add(
new PublisherRestriction(
purposeId, RestrictionType.from(restrictionTypeId),
BitSetIntIterable.from(bs)));
}
return currentPointer;
}
Expand Down
103 changes: 97 additions & 6 deletions iabtcf-decoder/src/main/java/com/iabtcf/utils/BitReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,11 @@ public byte readBits6(int offset) {
*
* @throws ByteParseException
*/
private byte readByteBits(int offset, int nbits) {
public byte readByteBits(int offset, int nbits) {
if (nbits < 0 || nbits > 8) {
throw new ByteParseException("Only 0 to 8 bytes can be read into a byte");
}

int startByte = offset >>> 3;
int bitPos = offset % 8;
int n = 8 - bitPos;
Expand Down Expand Up @@ -286,6 +290,30 @@ public int readBits24(int offset) {
}
}

/**
* @throws ByteParseException
*/
public int readBits32(int offset) {
int startByte = offset >>> 3;
int bitPos = offset % 8;
int n = 8 - bitPos;

if (n < 8) {
ensureReadable(startByte, 5);
return ((unsafeReadLsb(buffer[startByte], bitPos, n) & 0xFF) << 24)
| (buffer[startByte + 1] & 0xFF) << (16 + bitPos)
| (buffer[startByte + 2] & 0xFF) << (8 + bitPos)
| (buffer[startByte + 3] & 0xFF) << bitPos
| (unsafeReadMsb(buffer[startByte + 4], 0, bitPos) & 0xFF);
} else {
ensureReadable(startByte, 4);
return (buffer[startByte] & 0xFF) << 24
| (buffer[startByte + 1] & 0xFF) << 16
| (buffer[startByte + 2] & 0xFF) << 8
| (buffer[startByte + 3] & 0xFF);
}
}

/**
* @throws ByteParseException
*/
Expand Down Expand Up @@ -320,20 +348,83 @@ public long readBits36(int offset) {
}
}

/**
* @throws ByteParseException
*/
public long readBits64(int offset) {
int startByte = offset >>> 3;
int bitPos = offset % 8;
int n = 8 - bitPos; // # bits to read

if (n < 8) {
ensureReadable(startByte, 9);
return ((long) unsafeReadLsb(buffer[startByte], bitPos, n) & 0xFF) << 56
| ((long) buffer[startByte + 1] & 0xFF) << (48 + bitPos)
| ((long) buffer[startByte + 2] & 0xFF) << (40 + bitPos)
| ((long) buffer[startByte + 3] & 0xFF) << (32 + bitPos)
| ((long) buffer[startByte + 4] & 0xFF) << (24 + bitPos)
| ((long) buffer[startByte + 5] & 0xFF) << (16 + bitPos)
| ((long) buffer[startByte + 6] & 0xFF) << (8 + bitPos)
| ((long) buffer[startByte + 7] & 0xFF) << bitPos
| ((long) unsafeReadMsb(buffer[startByte + 8], 0, bitPos) & 0xFF);
} else {
ensureReadable(startByte, 8);
return ((long) buffer[startByte] & 0xFF) << 56
| ((long) buffer[startByte + 1] & 0xFF) << 48
| ((long) buffer[startByte + 2] & 0xFF) << 40
| ((long) buffer[startByte + 3] & 0xFF) << 32
| ((long) buffer[startByte + 4] & 0xFF) << 24
| ((long) buffer[startByte + 5] & 0xFF) << 16
| ((long) buffer[startByte + 6] & 0xFF) << 8
| ((long) buffer[startByte + 7] & 0xFF);
}
}

/**
* @throws ByteParseException
*/
public BitSet readBitSet(int offset, int length) {
// TODO(mk): can we read larger chunks at a time?
BitSet bs = new BitSet(length);
for (int i = 0; i < length; i++) {
if (readBits1(offset + i)) {
bs.set(i);
return readBitSet(offset, length, 0);
}

/**
* @throws ByteParseException
*/
public BitSet readBitSet(int offset, int length, int resultShift) {
final BitSet bs = new BitSet(length);
int i = 0;
while (i < length) {
final int remaining = length - i;
final int readIndex = offset + i;
final int writeIndex = resultShift + i;
if (remaining >= 64) {
fillBitSetWithContent(bs, readBits64(readIndex), 64, writeIndex);
i += 64;
} else if (remaining >= 32) {
fillBitSetWithContent(bs, readBits32(readIndex), 32, writeIndex);
i += 32;
} else if (remaining >= 16) {
fillBitSetWithContent(bs, readBits16(readIndex), 16, writeIndex);
i += 16;
} else if (remaining >= 8) {
fillBitSetWithContent(bs, readByteBits(readIndex, 8), 8, writeIndex);
i += 8;
} else {
fillBitSetWithContent(bs, readByteBits(readIndex, remaining), remaining, writeIndex);
i += remaining;
}
}
return bs;
}

private void fillBitSetWithContent(BitSet bs, long content, int size, int offset) {
for (int j = 0; j < size; j++) {
if (((content >>> (size - 1 - j)) & 1) == 1) {
bs.set(offset + j);
}
}
}

private byte unsafeReadMsb(byte from, int offset, int length) {
return length == 0 ? 0 : (byte) ((from >>> ((8 - length) - offset)) & ((1 << length) - 1));
}
Expand Down
113 changes: 113 additions & 0 deletions iabtcf-decoder/src/test/java/com/iabtcf/decoder/BitReaderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,119 @@ private void checkTestReadBits24N_Random() {
}
}

@Test
public void testReadBits32_0() {
BitReader bv = new BitReader(new byte[] {
0b00000001, 0b00000001, 0b00000001, 0b00000001,
});
assertEquals(0x1010101, bv.readBits32(0));
}

@Test
public void testReadBits32_1() {
BitReader bv = new BitReader(new byte[] {(byte) 0x00,
(byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000,
});
assertEquals(0x1010101, bv.readBits32(1));
}

@Test
public void testReadBits32_2() {
byte[] g = new byte[] {(byte) 0x00,
(byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000,
};
BitReader bv = new BitReader(g);
assertEquals(0x1010101, bv.readBits32(1));

shift(g);

bv = new BitReader(g);
assertEquals(0x1010101, bv.readBits32(2));
}

@Test
public void testReadBits32N_Random() {
for (int i = 0; i < 1000; i++) {
checkTestReadBits32N_Random();
}
}

private void checkTestReadBits32N_Random() {
byte[] rb = new byte[5];
r.nextBytes(rb);

byte[] g = new byte[] {(byte) 0b0000001,
rb[0], rb[1], rb[2], rb[3], rb[4],
(byte) 0x00, (byte) 0x00};

BitReader bv = new BitReader(g);
long expect = bv.readBits32(4);

for (int i = 1; i < 16; i++) {
shift(g);
bv = new BitReader(g);
assertEquals(String.format("%d", i), expect, bv.readBits32(4 + i));
}
}

@Test
public void testReadBits64_0() {
BitReader bv = new BitReader(new byte[] {
0b00000001, 0b00000001, 0b00000001, 0b00000001,
0b00000001, 0b00000001, 0b00000001, 0b00000001,
});
assertEquals(0x101010101010101L, bv.readBits64(0));
}

@Test
public void testReadBits64_1() {
BitReader bv = new BitReader(new byte[] {(byte) 0x00,
(byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000,
(byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000,
});
assertEquals(0x101010101010101L, bv.readBits64(1));
}

@Test
public void testReadBits64_2() {
byte[] g = new byte[] {(byte) 0x00,
(byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000,
(byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000,
};
BitReader bv = new BitReader(g);
assertEquals(0x101010101010101L, bv.readBits64(1));

shift(g);

bv = new BitReader(g);
assertEquals(0x101010101010101L, bv.readBits64(2));
}

@Test
public void testReadBits64N_Random() {
for (int i = 0; i < 1000; i++) {
checkTestReadBits64N_Random();
}
}

private void checkTestReadBits64N_Random() {
byte[] rb = new byte[9];
r.nextBytes(rb);

byte[] g = new byte[] {(byte) 0b0000001,
rb[0], rb[1], rb[2], rb[3], rb[4], rb[5], rb[6], rb[7], rb[8],
(byte) 0x00, (byte) 0x00};

BitReader bv = new BitReader(g);
long expect = bv.readBits64(4);

for (int i = 1; i < 16; i++) {
shift(g);
bv = new BitReader(g);
assertEquals(String.format("%d", i), expect, bv.readBits64(4 + i));
}
}

@Test
public void testShift() {
BitReader bv;
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@
</plugins>
</build>
</profile>

</profiles>

<build>
Expand Down