ARROW-1373: Implement getBuffer() methods for ValueVector

cc @jacques-n , @StevenMPhillips

Patch Summary:

As part of ARROW-801, we recently added getValidityBufferAddress(), getOffsetBufferAddress(), getDataBufferAddress() interfaces to get the virtual address of the ArrowBuf.

We now have the following new interfaces to get the corresponding ArrowBuf:

getValidityBuffer()
getDataBuffer()
getOffsetBuffer()

Background:

Currently we have getBuffer() method implemented as part of BaseDataValueVector abstract class. As part of patch for ARROW-276, NullableValueVectors no longer extends BaseDataValueVector -- they don't have to since they don't need the underlying data buffer  (ArrowBuf data field) of BaseDataValueVector.

The call to getBuffer() on NullableValueVectors simply delegates the operation to getBuffer() of underlying data/value vector.

Problem:

If a piece of code is working with ValueVector abstraction and the expected runtime type is Nullable<something>Vector, the compiler obviously complains about doing
(v of type ValueVector).getBuffer().

Until now this worked as we kept the compiler happy by casting the ValueVector to BaseDataValueVector and then do ((BaseDataValueVector)(v of type ValueVector)).getBuffer(). This code broke since NullableValueVectors are no longer a subtype of BaseDataValueVector -- the inheritance hierarchy was changed as part of ARROW-276.

Solution:

Similar to what was done in ARROW-801, we have new methods at ValueVector interface to get the underlying buffer. ValueVector has always had the methods getBuffers(), getBufferSizeFor(), getBufferSize(), so it makes sense to augment the ValueVector interface with new APIs.

It looks like new unit tests are not needed since the unit tests added for ARROW-801 test the new APIs as well --> getDataBufferAddress() underneath invokes getDataBuffer() to get the memory address of ArrowBuf so we are good.

Author: siddharth <siddharth@dremio.com>

Closes apache#976 from siddharthteotia/ARROW-1373 and squashes the following commits:

1ef2022 [siddharth] Fixed failures and added javadocs
e5ff023 [siddharth] ARROW-1373: Implement getBuffer() methods for ValueVector

Author: siddharthteotia

vector/src/main/codegen/templates/FixedValueVectors.java

@@ -96,6 +96,24 @@ public int getBufferSizeFor(final int valueCount) {
     return valueCount * ${type.width};
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    /* this operation is not supported for non-nullable vectors */
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    /* we are not throwing away getBuffer() of BaseDataValueVector so use it wherever applicable */
+    return getBuffer();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    /* this operation is not supported for fixed-width vectors */
+    throw new UnsupportedOperationException();
+  }
+
   @Override
   public int getValueCapacity(){
     return (int) (data.capacity() *1.0 / ${type.width});

vector/src/main/codegen/templates/NullableValueVectors.java

@@ -219,7 +219,7 @@ public int getBufferSizeFor(final int valueCount) {
   }
 
   public ArrowBuf getBuffer() {
-    return values.getBuffer();
+    return values.getDataBuffer();
   }
 
   @Override
@@ -437,23 +437,50 @@ public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){
 
   @Override
   public long getValidityBufferAddress() {
-    return (bits.getBuffer().memoryAddress());
+    /* address of the databuffer associated with the bitVector */
+    return (bits.getDataBuffer().memoryAddress());
   }
 
   @Override
   public long getDataBufferAddress() {
-    return (values.getBuffer().memoryAddress());
+    /* address of the dataBuffer associated with the valueVector */
+    return (values.getDataBuffer().memoryAddress());
   }
 
   @Override
   public long getOffsetBufferAddress() {
+    /* address of the dataBuffer associated with the offsetVector
+     * this operation is not supported for fixed-width vector types.
+     */
     <#if type.major != "VarLen">
         throw new UnsupportedOperationException();
     <#else>
         return (values.getOffsetAddr());
     </#if>
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    /* dataBuffer associated with the bitVector */
+    return (bits.getDataBuffer());
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    /* dataBuffer associated with the valueVector */
+    return (values.getDataBuffer());
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    /* dataBuffer associated with the offsetVector of the valueVector */
+    <#if type.major != "VarLen">
+        throw new UnsupportedOperationException();
+    <#else>
+        return (values.getOffsetBuffer());
+    </#if>
+  }
+
   public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> {
     final BitVector.Accessor bAccessor = bits.getAccessor();
     final ${valuesName}.Accessor vAccessor = values.getAccessor();

vector/src/main/codegen/templates/UnionVector.java

@@ -134,7 +134,7 @@ private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
 
   @Override
   public long getValidityBufferAddress() {
-    return typeVector.getBuffer().memoryAddress();
+    return typeVector.getDataBuffer().memoryAddress();
   }
 
   @Override
@@ -147,6 +147,15 @@ public long getOffsetBufferAddress() {
     throw new UnsupportedOperationException();
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() { return typeVector.getDataBuffer(); }
+
+  @Override
+  public ArrowBuf getDataBuffer() { throw new UnsupportedOperationException(); }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); }
+
   public NullableMapVector getMap() {
     if (mapVector == null) {
       int vectorCount = internalMap.size();

vector/src/main/codegen/templates/VariableLengthVectors.java

@@ -123,6 +123,24 @@ public int getBufferSizeFor(final int valueCount) {
     return offsetVector.getBufferSizeFor(valueCount + 1) + idx;
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    /* this operation is not supported for non-nullable vectors */
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    /* we are not throwing away getBuffer() of BaseDataValueVector so use it wherever applicable */
+    return getBuffer();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    /* dataBuffer associated with the underlying offsetVector */
+    return offsetVector.getDataBuffer();
+  }
+
   @Override
   public int getValueCapacity(){
     return Math.max(offsetVector.getValueCapacity() - 1, 0);
@@ -170,7 +188,7 @@ public ArrowBuf[] getBuffers(boolean clear) {
   }
 
   public long getOffsetAddr(){
-    return offsetVector.getBuffer().memoryAddress();
+    return offsetVector.getDataBuffer().memoryAddress();
   }
 
   public UInt${type.width}Vector getOffsetVector(){

vector/src/main/java/org/apache/arrow/vector/BitVector.java

@@ -110,6 +110,24 @@ public int getBufferSizeFor(final int valueCount) {
     return getSizeFromCount(valueCount);
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    /* this operation is not supported for non-nullable vectors */
+    throw new  UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    /* we are not throwing away getBuffer() of BaseDataValueVector so use it wherever applicable */
+    return getBuffer();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    /* this operation is not supported for fixed-width vectors */
+    throw new UnsupportedOperationException();
+  }
+
   int getSizeFromCount(int valueCount) {
     return (int) Math.ceil(valueCount / 8.0);
   }

vector/src/main/java/org/apache/arrow/vector/ValueVector.java

@@ -234,4 +234,25 @@ interface Mutator {
     @Deprecated
     void generateTestData(int values);
   }
+
+  /**
+   * Gets the underlying buffer associated with validity vector
+   *
+   * @return buffer
+   */
+  public ArrowBuf getValidityBuffer();
+
+  /**
+   * Gets the underlying buffer associated with data vector
+   *
+   * @return buffer
+   */
+  public ArrowBuf getDataBuffer();
+
+  /**
+   * Gets the underlying buffer associated with offset vector
+   *
+   * @return buffer
+   */
+  public ArrowBuf getOffsetBuffer();
 }

vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

@@ -244,4 +244,19 @@ public long getDataBufferAddress() {
   public long getOffsetBufferAddress() {
     throw new UnsupportedOperationException();
   }
+
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    throw new UnsupportedOperationException();
+  }
 }

vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java

@@ -295,7 +295,7 @@ public UnionVector promoteToUnion() {
 
   @Override
   public long getValidityBufferAddress() {
-    return (bits.getBuffer().memoryAddress());
+    return (bits.getDataBuffer().memoryAddress());
   }
 
   @Override
@@ -308,6 +308,21 @@ public long getOffsetBufferAddress() {
     throw new UnsupportedOperationException();
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    return (bits.getDataBuffer());
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
   public class Accessor extends BaseValueVector.BaseAccessor {
 
     @Override

vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

@@ -180,7 +180,7 @@ public TransferPair makeTransferPair(ValueVector target) {
 
   @Override
   public long getValidityBufferAddress() {
-    return (bits.getBuffer().memoryAddress());
+    return (bits.getDataBuffer().memoryAddress());
   }
 
   @Override
@@ -190,9 +190,20 @@ public long getDataBufferAddress() {
 
   @Override
   public long getOffsetBufferAddress() {
-    return (offsets.getBuffer().memoryAddress());
+    return (offsets.getDataBuffer().memoryAddress());
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() { return bits.getDataBuffer(); }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() { return offsets.getDataBuffer(); }
+
   private class TransferImpl implements TransferPair {
 
     ListVector to;

vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java

@@ -32,6 +32,8 @@
 import com.google.common.collect.Ordering;
 import com.google.common.primitives.Ints;
 
+import io.netty.buffer.ArrowBuf;
+
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.BaseValueVector;
 import org.apache.arrow.vector.FieldVector;
@@ -129,6 +131,21 @@ public int getBufferSizeFor(final int valueCount) {
     return (int) bufferSize;
   }
 
+  @Override
+  public ArrowBuf getValidityBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getDataBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ArrowBuf getOffsetBuffer() {
+    throw new UnsupportedOperationException();
+  }
+
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
     return getTransferPair(name, allocator, null);