Skip to content

Commit 1795832

Browse files
rtadepallilidavidm
rtadepalli
and
lidavidm
authored
GH-774: Consoliate BitVectorHelper.getValidityBufferSize and BaseValueVector.getValidityBufferSizeFromCount (#775)
## What's Changed Just removing some duplicate functions in anticipation of cleaning out some transferPair duplication across complex vectors. Closes #774 --------- Co-authored-by: David Li <li.davidm96@gmail.com>
1 parent fad4e14 commit 1795832

22 files changed

+117
-91
lines changed

‎vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -359,7 +359,7 @@ public int getBufferSizeFor(final int count) {
359359
if (count == 0) {
360360
return 0;
361361
}
362-
return (count * typeWidth) + getValidityBufferSizeFromCount(count);
362+
return (count * typeWidth) + BitVectorHelper.getValidityBufferSizeFromCount(count);
363363
}
364364

365365
/**
@@ -372,7 +372,7 @@ public int getBufferSize() {
372372
if (valueCount == 0) {
373373
return 0;
374374
}
375-
return (valueCount * typeWidth) + getValidityBufferSizeFromCount(valueCount);
375+
return (valueCount * typeWidth) + BitVectorHelper.getValidityBufferSizeFromCount(valueCount);
376376
}
377377

378378
/**
@@ -536,10 +536,10 @@ private void setReaderAndWriterIndex() {
536536
validityBuffer.writerIndex(0);
537537
valueBuffer.writerIndex(0);
538538
} else {
539-
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
539+
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
540540
if (typeWidth == 0) {
541541
/* specialized handling for BitVector */
542-
valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
542+
valueBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
543543
} else {
544544
valueBuffer.writerIndex((long) valueCount * typeWidth);
545545
}
@@ -664,7 +664,7 @@ private void splitAndTransferValidityBuffer(
664664
int startIndex, int length, BaseFixedWidthVector target) {
665665
int firstByteSource = BitVectorHelper.byteIndex(startIndex);
666666
int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
667-
int byteSizeTarget = getValidityBufferSizeFromCount(length);
667+
int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length);
668668
int offset = startIndex % 8;
669669

670670
if (length > 0) {

‎vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,7 @@ private void setReaderAndWriterIndex() {
379379
valueBuffer.writerIndex(0);
380380
} else {
381381
final long lastDataOffset = getStartOffset(valueCount);
382-
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
382+
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
383383
offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH);
384384
valueBuffer.writerIndex(lastDataOffset);
385385
}
@@ -633,7 +633,7 @@ public int getBufferSizeFor(final int valueCount) {
633633
return 0;
634634
}
635635

636-
final long validityBufferSize = getValidityBufferSizeFromCount(valueCount);
636+
final long validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount);
637637
final long offsetBufferSize = (long) (valueCount + 1) * OFFSET_WIDTH;
638638
/* get the end offset for this valueCount */
639639
final long dataBufferSize = getStartOffset(valueCount);
@@ -816,7 +816,7 @@ private void splitAndTransferValidityBuffer(
816816
int startIndex, int length, BaseLargeVariableWidthVector target) {
817817
int firstByteSource = BitVectorHelper.byteIndex(startIndex);
818818
int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
819-
int byteSizeTarget = getValidityBufferSizeFromCount(length);
819+
int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length);
820820
int offset = startIndex % 8;
821821

822822
if (length > 0) {

‎vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,14 @@ protected ArrowBuf releaseBuffer(ArrowBuf buffer) {
110110
return buffer;
111111
}
112112

113-
/* number of bytes for the validity buffer for the given valueCount */
113+
/**
114+
* Compute the size of validity buffer required to manage a given number of elements in a vector.
115+
*
116+
* @param valueCount number of elements in the vector
117+
* @return buffer size
118+
* @deprecated -- use {@link BitVectorHelper#getValidityBufferSizeFromCount} instead.
119+
*/
120+
@Deprecated(forRemoval = true, since = "18.4.0")
114121
protected static int getValidityBufferSizeFromCount(final int valueCount) {
115122
return DataSizeRoundingUtil.divideBy8Ceil(valueCount);
116123
}

‎vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -395,7 +395,7 @@ private void setReaderAndWriterIndex() {
395395
valueBuffer.writerIndex(0);
396396
} else {
397397
final int lastDataOffset = getStartOffset(valueCount);
398-
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
398+
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
399399
offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH);
400400
valueBuffer.writerIndex(lastDataOffset);
401401
}
@@ -673,7 +673,7 @@ public int getBufferSizeFor(final int valueCount) {
673673
return 0;
674674
}
675675

676-
final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
676+
final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount);
677677
final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH;
678678
/* get the end offset for this valueCount */
679679
final int dataBufferSize = offsetBuffer.getInt((long) valueCount * OFFSET_WIDTH);
@@ -867,7 +867,7 @@ private void splitAndTransferValidityBuffer(
867867

868868
final int firstByteSource = BitVectorHelper.byteIndex(startIndex);
869869
final int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
870-
final int byteSizeTarget = getValidityBufferSizeFromCount(length);
870+
final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length);
871871
final int offset = startIndex % 8;
872872

873873
if (offset == 0) {

‎vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -400,7 +400,7 @@ private void setReaderAndWriterIndex() {
400400
validityBuffer.writerIndex(0);
401401
viewBuffer.writerIndex(0);
402402
} else {
403-
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
403+
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
404404
viewBuffer.writerIndex(valueCount * ELEMENT_SIZE);
405405
}
406406
}
@@ -683,7 +683,7 @@ public int getBufferSizeFor(final int valueCount) {
683683
return 0;
684684
}
685685

686-
final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
686+
final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount);
687687
final int viewBufferSize = valueCount * ELEMENT_SIZE;
688688
final int dataBufferSize = getDataBufferSize();
689689
return validityBufferSize + viewBufferSize + dataBufferSize;
@@ -872,7 +872,7 @@ private void splitAndTransferValidityBuffer(
872872

873873
final int firstByteSource = BitVectorHelper.byteIndex(startIndex);
874874
final int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
875-
final int byteSizeTarget = getValidityBufferSizeFromCount(length);
875+
final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length);
876876
final int offset = startIndex % 8;
877877

878878
if (offset == 0) {

‎vector/src/main/java/org/apache/arrow/vector/BitVector.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ public MinorType getMinorType() {
9898
*/
9999
@Override
100100
public void setInitialCapacity(int valueCount) {
101-
final int size = getValidityBufferSizeFromCount(valueCount);
101+
final int size = BitVectorHelper.getValidityBufferSizeFromCount(valueCount);
102102
if (size * 2L > MAX_ALLOCATION_SIZE) {
103103
throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
104104
}
@@ -121,7 +121,7 @@ public int getBufferSizeFor(final int count) {
121121
if (count == 0) {
122122
return 0;
123123
}
124-
return 2 * getValidityBufferSizeFromCount(count);
124+
return 2 * BitVectorHelper.getValidityBufferSizeFromCount(count);
125125
}
126126

127127
/**
@@ -165,7 +165,7 @@ private ArrowBuf splitAndTransferBuffer(
165165
int startIndex, int length, ArrowBuf sourceBuffer, ArrowBuf destBuffer) {
166166
int firstByteSource = BitVectorHelper.byteIndex(startIndex);
167167
int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
168-
int byteSizeTarget = getValidityBufferSizeFromCount(length);
168+
int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length);
169169
int offset = startIndex % 8;
170170

171171
if (length > 0) {

‎vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -135,11 +135,11 @@ public static void setValidityBit(ArrowBuf validityBuffer, int index, int value)
135135
public static ArrowBuf setValidityBit(
136136
ArrowBuf validityBuffer, BufferAllocator allocator, int valueCount, int index, int value) {
137137
if (validityBuffer == null) {
138-
validityBuffer = allocator.buffer(getValidityBufferSize(valueCount));
138+
validityBuffer = allocator.buffer(getValidityBufferSizeFromCount(valueCount));
139139
}
140140
setValidityBit(validityBuffer, index, value);
141141
if (index == (valueCount - 1)) {
142-
validityBuffer.writerIndex(getValidityBufferSize(valueCount));
142+
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
143143
}
144144

145145
return validityBuffer;
@@ -165,7 +165,7 @@ public static int get(final ArrowBuf buffer, int index) {
165165
* @param valueCount number of elements in the vector
166166
* @return buffer size
167167
*/
168-
public static int getValidityBufferSize(int valueCount) {
168+
public static int getValidityBufferSizeFromCount(int valueCount) {
169169
return DataSizeRoundingUtil.divideBy8Ceil(valueCount);
170170
}
171171

@@ -182,7 +182,7 @@ public static int getNullCount(final ArrowBuf validityBuffer, final int valueCou
182182
return 0;
183183
}
184184
int count = 0;
185-
final int sizeInBytes = getValidityBufferSize(valueCount);
185+
final int sizeInBytes = getValidityBufferSizeFromCount(valueCount);
186186
// If value count is not a multiple of 8, then calculate number of used bits in the last byte
187187
final int remainder = valueCount % 8;
188188
final int fullBytesCount = remainder == 0 ? sizeInBytes : sizeInBytes - 1;
@@ -233,7 +233,7 @@ public static boolean checkAllBitsEqualTo(
233233
if (valueCount == 0) {
234234
return true;
235235
}
236-
final int sizeInBytes = getValidityBufferSize(valueCount);
236+
final int sizeInBytes = getValidityBufferSizeFromCount(valueCount);
237237

238238
// boundary check
239239
validityBuffer.checkBytes(0, sizeInBytes);
@@ -325,7 +325,7 @@ public static ArrowBuf loadValidityBuffer(
325325
sourceValidityBuffer == null || sourceValidityBuffer.capacity() == 0;
326326
if (isValidityBufferNull
327327
&& (fieldNode.getNullCount() == 0 || fieldNode.getNullCount() == valueCount)) {
328-
newBuffer = allocator.buffer(getValidityBufferSize(valueCount));
328+
newBuffer = allocator.buffer(getValidityBufferSizeFromCount(valueCount));
329329
newBuffer.setZero(0, newBuffer.capacity());
330330
if (fieldNode.getNullCount() != 0) {
331331
/* all NULLs */

‎vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,8 @@ public FixedSizeListVector(
110110
this.listSize = ((ArrowType.FixedSizeList) field.getFieldType().getType()).getListSize();
111111
Preconditions.checkArgument(listSize >= 0, "list size must be non-negative");
112112
this.valueCount = 0;
113-
this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
113+
this.validityAllocationSizeInBytes =
114+
BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
114115
}
115116

116117
@Override
@@ -189,7 +190,7 @@ public List<ArrowBuf> getFieldBuffers() {
189190

190191
private void setReaderAndWriterIndex() {
191192
validityBuffer.readerIndex(0);
192-
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
193+
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
193194
}
194195

195196
/**
@@ -268,7 +269,8 @@ private void reallocValidityBuffer() {
268269
if (validityAllocationSizeInBytes > 0) {
269270
newAllocationSize = validityAllocationSizeInBytes;
270271
} else {
271-
newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L;
272+
newAllocationSize =
273+
BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L;
272274
}
273275
}
274276

@@ -311,7 +313,7 @@ public UnionFixedSizeListWriter getWriter() {
311313

312314
@Override
313315
public void setInitialCapacity(int numRecords) {
314-
validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords);
316+
validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords);
315317
vector.setInitialCapacity(numRecords * listSize);
316318
}
317319

@@ -328,15 +330,15 @@ public int getBufferSize() {
328330
if (getValueCount() == 0) {
329331
return 0;
330332
}
331-
return getValidityBufferSizeFromCount(valueCount) + vector.getBufferSize();
333+
return BitVectorHelper.getValidityBufferSizeFromCount(valueCount) + vector.getBufferSize();
332334
}
333335

334336
@Override
335337
public int getBufferSizeFor(int valueCount) {
336338
if (valueCount == 0) {
337339
return 0;
338340
}
339-
return getValidityBufferSizeFromCount(valueCount)
341+
return BitVectorHelper.getValidityBufferSizeFromCount(valueCount)
340342
+ vector.getBufferSizeFor(valueCount * listSize);
341343
}
342344

@@ -654,7 +656,7 @@ private void splitAndTransferValidityBuffer(
654656
int startIndex, int length, FixedSizeListVector target) {
655657
int firstByteSource = BitVectorHelper.byteIndex(startIndex);
656658
int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
657-
int byteSizeTarget = getValidityBufferSizeFromCount(length);
659+
int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length);
658660
int offset = startIndex % 8;
659661

660662
if (length > 0) {

‎vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,8 @@ public LargeListVector(Field field, BufferAllocator allocator, CallBack callBack
131131
this.field = field;
132132
this.validityBuffer = allocator.getEmpty();
133133
this.callBack = callBack;
134-
this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
134+
this.validityAllocationSizeInBytes =
135+
BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
135136
this.lastSet = -1;
136137
this.offsetBuffer = allocator.getEmpty();
137138
this.vector = vector == null ? DEFAULT_DATA_VECTOR : vector;
@@ -156,7 +157,7 @@ public void initializeChildrenFromFields(List<Field> children) {
156157

157158
@Override
158159
public void setInitialCapacity(int numRecords) {
159-
validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords);
160+
validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords);
160161
offsetAllocationSizeInBytes = (long) (numRecords + 1) * OFFSET_WIDTH;
161162
if (vector instanceof BaseFixedWidthVector || vector instanceof BaseVariableWidthVector) {
162163
vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
@@ -184,7 +185,7 @@ public void setInitialCapacity(int numRecords) {
184185
*/
185186
@Override
186187
public void setInitialCapacity(int numRecords, double density) {
187-
validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords);
188+
validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords);
188189
if ((numRecords * density) >= Integer.MAX_VALUE) {
189190
throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
190191
}
@@ -311,7 +312,7 @@ private void setReaderAndWriterIndex() {
311312
validityBuffer.writerIndex(0);
312313
offsetBuffer.writerIndex(0);
313314
} else {
314-
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
315+
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
315316
offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH);
316317
}
317318
}
@@ -442,7 +443,8 @@ private void reallocValidityBuffer() {
442443
if (validityAllocationSizeInBytes > 0) {
443444
newAllocationSize = validityAllocationSizeInBytes;
444445
} else {
445-
newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L;
446+
newAllocationSize =
447+
BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L;
446448
}
447449
}
448450
newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize);
@@ -699,7 +701,7 @@ private void splitAndTransferValidityBuffer(
699701
int startIndex, int length, LargeListVector target) {
700702
int firstByteSource = BitVectorHelper.byteIndex(startIndex);
701703
int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
702-
int byteSizeTarget = getValidityBufferSizeFromCount(length);
704+
int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length);
703705
int offset = startIndex % 8;
704706

705707
if (length > 0) {
@@ -821,7 +823,7 @@ public int getBufferSize() {
821823
return 0;
822824
}
823825
final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH;
824-
final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
826+
final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount);
825827
return offsetBufferSize + validityBufferSize + vector.getBufferSize();
826828
}
827829

@@ -830,7 +832,7 @@ public int getBufferSizeFor(int valueCount) {
830832
if (valueCount == 0) {
831833
return 0;
832834
}
833-
final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
835+
final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount);
834836
long innerVectorValueCount = offsetBuffer.getLong((long) valueCount * OFFSET_WIDTH);
835837

836838
return ((valueCount + 1) * OFFSET_WIDTH)

0 commit comments

Comments
 (0)