Skip to content

Commit 45f4b45

Browse files
julienledemkou
julienledem
authored and
kou
committed
ARROW-264: File format
This is work in progress Author: Julien Le Dem <julien@dremio.com> Closes apache#123 from julienledem/arrow_264_file_format and squashes the following commits: 252de6d [Julien Le Dem] remove outdated comment 04d797f [Julien Le Dem] maps are not nullable yet e8359b3 [Julien Le Dem] align on 8 byte boundaries; more tests 8b8b823 [Julien Le Dem] refactoring 31e95e6 [Julien Le Dem] fix list vector b824938 [Julien Le Dem] fix types; add licenses; more tests; more complex 2fd3bc1 [Julien Le Dem] cleanup 50fe680 [Julien Le Dem] nested support b0bf6bc [Julien Le Dem] cleanup 4247b1a [Julien Le Dem] fix whitespace d6a1788 [Julien Le Dem] refactoring 81863c5 [Julien Le Dem] fixed loader aa1b766 [Julien Le Dem] better test 2067e01 [Julien Le Dem] update format aacf61e [Julien Le Dem] fix pom b907aa5 [Julien Le Dem] simplify e43f26b [Julien Le Dem] add layout spec 0cc9718 [Julien Le Dem] add vector type ac6902a [Julien Le Dem] ARROW-264: File format 807db51 [Julien Le Dem] move information to schema f2f0596 [Julien Le Dem] Update FieldNode structure to be more explicit and reflect schema
1 parent 7b83cf6 commit 45f4b45

42 files changed

Lines changed: 2675 additions & 159 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎format/pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@
106106
<argument>-o</argument>
107107
<argument>target/generated-sources/</argument>
108108
<argument>../../format/Message.fbs</argument>
109+
<argument>../../format/File.fbs</argument>
109110
</arguments>
110111
</configuration>
111112
</execution>

‎memory/src/main/java/io/netty/buffer/ArrowBuf.java‎

Lines changed: 42 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@
1717
*/
1818
package io.netty.buffer;
1919

20-
import io.netty.util.internal.PlatformDependent;
21-
2220
import java.io.IOException;
2321
import java.io.InputStream;
2422
import java.io.OutputStream;
@@ -30,16 +28,18 @@
3028
import java.util.concurrent.atomic.AtomicInteger;
3129
import java.util.concurrent.atomic.AtomicLong;
3230

31+
import org.apache.arrow.memory.AllocationManager.BufferLedger;
3332
import org.apache.arrow.memory.BaseAllocator;
33+
import org.apache.arrow.memory.BaseAllocator.Verbosity;
3434
import org.apache.arrow.memory.BoundsChecking;
3535
import org.apache.arrow.memory.BufferAllocator;
3636
import org.apache.arrow.memory.BufferManager;
37-
import org.apache.arrow.memory.AllocationManager.BufferLedger;
38-
import org.apache.arrow.memory.BaseAllocator.Verbosity;
3937
import org.apache.arrow.memory.util.HistoricalLog;
4038

4139
import com.google.common.base.Preconditions;
4240

41+
import io.netty.util.internal.PlatformDependent;
42+
4343
public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable {
4444
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ArrowBuf.class);
4545

@@ -307,7 +307,7 @@ public ByteOrder order() {
307307
}
308308

309309
@Override
310-
public ByteBuf order(ByteOrder endianness) {
310+
public ArrowBuf order(ByteOrder endianness) {
311311
return this;
312312
}
313313

@@ -344,7 +344,7 @@ public ByteBuf copy(int index, int length) {
344344
}
345345

346346
@Override
347-
public ByteBuf slice() {
347+
public ArrowBuf slice() {
348348
return slice(readerIndex(), readableBytes());
349349
}
350350

@@ -467,7 +467,7 @@ public boolean equals(Object obj) {
467467
}
468468

469469
@Override
470-
public ByteBuf retain(int increment) {
470+
public ArrowBuf retain(int increment) {
471471
Preconditions.checkArgument(increment > 0, "retain(%d) argument is not positive", increment);
472472

473473
if (isEmpty) {
@@ -484,7 +484,7 @@ public ByteBuf retain(int increment) {
484484
}
485485

486486
@Override
487-
public ByteBuf retain() {
487+
public ArrowBuf retain() {
488488
return retain(1);
489489
}
490490

@@ -535,109 +535,109 @@ public short getShort(int index) {
535535
}
536536

537537
@Override
538-
public ByteBuf setShort(int index, int value) {
538+
public ArrowBuf setShort(int index, int value) {
539539
chk(index, 2);
540540
PlatformDependent.putShort(addr(index), (short) value);
541541
return this;
542542
}
543543

544544
@Override
545-
public ByteBuf setInt(int index, int value) {
545+
public ArrowBuf setInt(int index, int value) {
546546
chk(index, 4);
547547
PlatformDependent.putInt(addr(index), value);
548548
return this;
549549
}
550550

551551
@Override
552-
public ByteBuf setLong(int index, long value) {
552+
public ArrowBuf setLong(int index, long value) {
553553
chk(index, 8);
554554
PlatformDependent.putLong(addr(index), value);
555555
return this;
556556
}
557557

558558
@Override
559-
public ByteBuf setChar(int index, int value) {
559+
public ArrowBuf setChar(int index, int value) {
560560
chk(index, 2);
561561
PlatformDependent.putShort(addr(index), (short) value);
562562
return this;
563563
}
564564

565565
@Override
566-
public ByteBuf setFloat(int index, float value) {
566+
public ArrowBuf setFloat(int index, float value) {
567567
chk(index, 4);
568568
PlatformDependent.putInt(addr(index), Float.floatToRawIntBits(value));
569569
return this;
570570
}
571571

572572
@Override
573-
public ByteBuf setDouble(int index, double value) {
573+
public ArrowBuf setDouble(int index, double value) {
574574
chk(index, 8);
575575
PlatformDependent.putLong(addr(index), Double.doubleToRawLongBits(value));
576576
return this;
577577
}
578578

579579
@Override
580-
public ByteBuf writeShort(int value) {
580+
public ArrowBuf writeShort(int value) {
581581
ensure(2);
582582
PlatformDependent.putShort(addr(writerIndex), (short) value);
583583
writerIndex += 2;
584584
return this;
585585
}
586586

587587
@Override
588-
public ByteBuf writeInt(int value) {
588+
public ArrowBuf writeInt(int value) {
589589
ensure(4);
590590
PlatformDependent.putInt(addr(writerIndex), value);
591591
writerIndex += 4;
592592
return this;
593593
}
594594

595595
@Override
596-
public ByteBuf writeLong(long value) {
596+
public ArrowBuf writeLong(long value) {
597597
ensure(8);
598598
PlatformDependent.putLong(addr(writerIndex), value);
599599
writerIndex += 8;
600600
return this;
601601
}
602602

603603
@Override
604-
public ByteBuf writeChar(int value) {
604+
public ArrowBuf writeChar(int value) {
605605
ensure(2);
606606
PlatformDependent.putShort(addr(writerIndex), (short) value);
607607
writerIndex += 2;
608608
return this;
609609
}
610610

611611
@Override
612-
public ByteBuf writeFloat(float value) {
612+
public ArrowBuf writeFloat(float value) {
613613
ensure(4);
614614
PlatformDependent.putInt(addr(writerIndex), Float.floatToRawIntBits(value));
615615
writerIndex += 4;
616616
return this;
617617
}
618618

619619
@Override
620-
public ByteBuf writeDouble(double value) {
620+
public ArrowBuf writeDouble(double value) {
621621
ensure(8);
622622
PlatformDependent.putLong(addr(writerIndex), Double.doubleToRawLongBits(value));
623623
writerIndex += 8;
624624
return this;
625625
}
626626

627627
@Override
628-
public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
628+
public ArrowBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
629629
udle.getBytes(index + offset, dst, dstIndex, length);
630630
return this;
631631
}
632632

633633
@Override
634-
public ByteBuf getBytes(int index, ByteBuffer dst) {
634+
public ArrowBuf getBytes(int index, ByteBuffer dst) {
635635
udle.getBytes(index + offset, dst);
636636
return this;
637637
}
638638

639639
@Override
640-
public ByteBuf setByte(int index, int value) {
640+
public ArrowBuf setByte(int index, int value) {
641641
chk(index, 1);
642642
PlatformDependent.putByte(addr(index), (byte) value);
643643
return this;
@@ -699,13 +699,13 @@ protected void _setLong(int index, long value) {
699699
}
700700

701701
@Override
702-
public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
702+
public ArrowBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
703703
udle.getBytes(index + offset, dst, dstIndex, length);
704704
return this;
705705
}
706706

707707
@Override
708-
public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException {
708+
public ArrowBuf getBytes(int index, OutputStream out, int length) throws IOException {
709709
udle.getBytes(index + offset, out, length);
710710
return this;
711711
}
@@ -724,12 +724,12 @@ public int getBytes(int index, GatheringByteChannel out, int length) throws IOEx
724724
}
725725

726726
@Override
727-
public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
727+
public ArrowBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
728728
udle.setBytes(index + offset, src, srcIndex, length);
729729
return this;
730730
}
731731

732-
public ByteBuf setBytes(int index, ByteBuffer src, int srcIndex, int length) {
732+
public ArrowBuf setBytes(int index, ByteBuffer src, int srcIndex, int length) {
733733
if (src.isDirect()) {
734734
checkIndex(index, length);
735735
PlatformDependent.copyMemory(PlatformDependent.directBufferAddress(src) + srcIndex, this.memoryAddress() + index,
@@ -749,13 +749,13 @@ public ByteBuf setBytes(int index, ByteBuffer src, int srcIndex, int length) {
749749
}
750750

751751
@Override
752-
public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
752+
public ArrowBuf setBytes(int index, byte[] src, int srcIndex, int length) {
753753
udle.setBytes(index + offset, src, srcIndex, length);
754754
return this;
755755
}
756756

757757
@Override
758-
public ByteBuf setBytes(int index, ByteBuffer src) {
758+
public ArrowBuf setBytes(int index, ByteBuffer src) {
759759
udle.setBytes(index + offset, src);
760760
return this;
761761
}
@@ -860,4 +860,17 @@ public void print(StringBuilder sb, int indent, Verbosity verbosity) {
860860
}
861861
}
862862

863+
@Override
864+
public ArrowBuf readerIndex(int readerIndex) {
865+
super.readerIndex(readerIndex);
866+
return this;
867+
}
868+
869+
@Override
870+
public ArrowBuf writerIndex(int writerIndex) {
871+
super.writerIndex(writerIndex);
872+
return this;
873+
}
874+
875+
863876
}

‎vector/src/main/codegen/data/ArrowTypes.tdd‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,15 +30,15 @@
3030
},
3131
{
3232
name: "Union",
33-
fields: []
33+
fields: [{name: "mode", type: short}]
3434
},
3535
{
3636
name: "Int",
3737
fields: [{name: "bitWidth", type: int}, {name: "isSigned", type: boolean}]
3838
},
3939
{
4040
name: "FloatingPoint",
41-
fields: [{name: precision, type: int}]
41+
fields: [{name: precision, type: short}]
4242
},
4343
{
4444
name: "Utf8",

‎vector/src/main/codegen/templates/ArrowType.java‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,8 @@
2424

2525
<@pp.dropOutputFile />
2626
<@pp.changeOutputFile name="/org/apache/arrow/vector/types/pojo/ArrowType.java" />
27-
28-
2927
<#include "/@includes/license.ftl" />
28+
3029
package org.apache.arrow.vector.types.pojo;
3130

3231
import com.google.flatbuffers.FlatBufferBuilder;
@@ -38,7 +37,13 @@ public abstract class ArrowType {
3837

3938
public abstract byte getTypeType();
4039
public abstract int getType(FlatBufferBuilder builder);
40+
public abstract <T> T accept(ArrowTypeVisitor<T> visitor);
4141

42+
public static interface ArrowTypeVisitor<T> {
43+
<#list arrowTypes.types as type>
44+
T visit(${type.name} type);
45+
</#list>
46+
}
4247

4348
<#list arrowTypes.types as type>
4449
<#assign name = type.name>
@@ -70,9 +75,14 @@ public byte getTypeType() {
7075

7176
@Override
7277
public int getType(FlatBufferBuilder builder) {
78+
<#list type.fields as field>
79+
<#if field.type == "String">
80+
int ${field.name} = builder.createString(this.${field.name});
81+
</#if>
82+
</#list>
7383
org.apache.arrow.flatbuf.${type.name}.start${type.name}(builder);
7484
<#list type.fields as field>
75-
org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, <#if field.type == "String">builder.createString(${field.name})<#else>${field.name}</#if>);
85+
org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, ${field.name});
7686
</#list>
7787
return org.apache.arrow.flatbuf.${type.name}.end${type.name}(builder);
7888
}
@@ -83,6 +93,14 @@ public int getType(FlatBufferBuilder builder) {
8393
}
8494
</#list>
8595

96+
public String toString() {
97+
return "${name}{"
98+
<#list fields as field>
99+
+ ", " + ${field.name}
100+
</#list>
101+
+ "}";
102+
}
103+
86104
@Override
87105
public int hashCode() {
88106
return Objects.hash(<#list type.fields as field>${field.name}<#if field_has_next>, </#if></#list>);
@@ -102,6 +120,11 @@ public boolean equals(Object obj) {
102120
</#list>
103121
</#if>
104122
}
123+
124+
@Override
125+
public <T> T accept(ArrowTypeVisitor<T> visitor) {
126+
return visitor.visit(this);
127+
}
105128
}
106129
</#list>
107130

0 commit comments

Comments
 (0)