Skip to content

Commit e7e399d

Browse files
StevenMPhillipsStevenMPhillips
committed
ARROW-259: Use Flatbuffer Field type instead of MaterializedField
Remove MaterializedField, MajorType, RepeatedTypes Add code to convert from FlatBuf representation to Pojo also adds tests to test the conversion
1 parent 246a126 commit e7e399d

File tree

77 files changed

+2464
-5180
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

77 files changed

+2464
-5180
lines changed

‎format/Message.fbs

+21-1
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
1-
namespace apache.arrow.flatbuf;
1+
namespace org.apache.arrow.flatbuf;
22

33
/// ----------------------------------------------------------------------
44
/// Logical types and their metadata (if any)
55
///
66
/// These are stored in the flatbuffer in the Type union below
77

8+
table Null {
9+
}
10+
811
/// A Tuple in the flatbuffer metadata is the same as an Arrow Struct
912
/// (according to the physical memory layout). We used Tuple here as Struct is
1013
/// a reserved word in Flatbuffers
@@ -45,10 +48,22 @@ table Decimal {
4548
scale: int;
4649
}
4750

51+
table Date {
52+
}
53+
54+
table Time {
55+
}
56+
4857
table Timestamp {
4958
timezone: string;
5059
}
5160

61+
table IntervalDay {
62+
}
63+
64+
table IntervalYear {
65+
}
66+
5267
table JSONScalar {
5368
dense:bool=true;
5469
}
@@ -58,13 +73,18 @@ table JSONScalar {
5873
/// add new logical types to Type without breaking backwards compatibility
5974

6075
union Type {
76+
Null,
6177
Int,
6278
FloatingPoint,
6379
Binary,
6480
Utf8,
6581
Bool,
6682
Decimal,
83+
Date,
84+
Time,
6785
Timestamp,
86+
IntervalDay,
87+
IntervalYear,
6888
List,
6989
Tuple,
7090
Union,

‎header

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
Licensed to the Apache Software Foundation (ASF) under one
2+
or more contributor license agreements. See the NOTICE file
3+
distributed with this work for additional information
4+
regarding copyright ownership. The ASF licenses this file
5+
to you under the Apache License, Version 2.0 (the
6+
"License"); you may not use this file except in compliance
7+
with the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+

‎java/format/pom.xml

+163
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
<?xml version="1.0"?>
2+
<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
3+
license agreements. See the NOTICE file distributed with this work for additional
4+
information regarding copyright ownership. The ASF licenses this file to
5+
You under the Apache License, Version 2.0 (the "License"); you may not use
6+
this file except in compliance with the License. You may obtain a copy of
7+
the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
8+
by applicable law or agreed to in writing, software distributed under the
9+
License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
10+
OF ANY KIND, either express or implied. See the License for the specific
11+
language governing permissions and limitations under the License. -->
12+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
13+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
14+
<modelVersion>4.0.0</modelVersion>
15+
16+
<parent>
17+
<artifactId>arrow-java-root</artifactId>
18+
<groupId>org.apache.arrow</groupId>
19+
<version>0.1-decimal</version>
20+
</parent>
21+
22+
<artifactId>arrow-format</artifactId>
23+
<packaging>jar</packaging>
24+
<name>Arrow Format</name>
25+
26+
<properties>
27+
<fbs.version>1.2.0-3f79e055</fbs.version>
28+
<maven-compiler-plugin.version>3.3</maven-compiler-plugin.version>
29+
<maven-dependency-plugin.version>2.10</maven-dependency-plugin.version>
30+
<os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
31+
</properties>
32+
33+
<dependencies>
34+
<dependency>
35+
<groupId>com.vlkan</groupId>
36+
<artifactId>flatbuffers</artifactId>
37+
<version>${fbs.version}</version>
38+
</dependency>
39+
</dependencies>
40+
41+
<build>
42+
<extensions>
43+
<!-- provides os.detected.classifier (i.e. linux-x86_64, osx-x86_64) property -->
44+
<extension>
45+
<groupId>kr.motd.maven</groupId>
46+
<artifactId>os-maven-plugin</artifactId>
47+
<version>${os-maven-plugin.version}</version>
48+
</extension>
49+
</extensions>
50+
51+
<plugins>
52+
<plugin>
53+
<groupId>org.apache.maven.plugins</groupId>
54+
<artifactId>maven-dependency-plugin</artifactId>
55+
<version>${maven-dependency-plugin.version}</version>
56+
<executions>
57+
<execution>
58+
<id>copy-flatc</id>
59+
<phase>generate-sources</phase>
60+
<goals>
61+
<goal>copy</goal>
62+
</goals>
63+
<configuration>
64+
<artifactItems>
65+
<artifactItem>
66+
<groupId>com.vlkan</groupId>
67+
<artifactId>flatc-${os.detected.classifier}</artifactId>
68+
<version>${fbs.version}</version>
69+
<type>exe</type>
70+
<overWrite>true</overWrite>
71+
<outputDirectory>${project.build.directory}</outputDirectory>
72+
</artifactItem>
73+
</artifactItems>
74+
</configuration>
75+
</execution>
76+
</executions>
77+
</plugin>
78+
<plugin>
79+
<groupId>org.codehaus.mojo</groupId>
80+
<artifactId>exec-maven-plugin</artifactId>
81+
<version>1.4.0</version>
82+
<executions>
83+
<execution>
84+
<id>script-chmod</id>
85+
<goals>
86+
<goal>exec</goal>
87+
</goals>
88+
<phase>generate-sources</phase>
89+
<configuration>
90+
<executable>chmod</executable>
91+
<arguments>
92+
<argument>+x</argument>
93+
<argument>${project.build.directory}/flatc-${os.detected.classifier}-${fbs.version}.exe</argument>
94+
</arguments>
95+
</configuration>
96+
</execution>
97+
<execution>
98+
<goals>
99+
<goal>exec</goal>
100+
</goals>
101+
<phase>generate-sources</phase>
102+
<configuration>
103+
<executable>${project.build.directory}/flatc-${os.detected.classifier}-${fbs.version}.exe</executable>
104+
<arguments>
105+
<argument>-j</argument>
106+
<argument>-o</argument>
107+
<argument>target/generated-sources/</argument>
108+
<argument>../../format/Message.fbs</argument>
109+
</arguments>
110+
</configuration>
111+
</execution>
112+
</executions>
113+
</plugin>
114+
<plugin>
115+
<groupId>com.mycila</groupId>
116+
<artifactId>license-maven-plugin</artifactId>
117+
<version>2.3</version>
118+
<configuration>
119+
<header>${basedir}/../../header</header>
120+
<includes>
121+
<include>**/*.java</include>
122+
</includes>
123+
</configuration>
124+
<executions>
125+
<execution>
126+
<phase>process-sources</phase>
127+
<goals>
128+
<goal>format</goal>
129+
</goals>
130+
</execution>
131+
</executions>
132+
</plugin>
133+
<plugin>
134+
<groupId>org.codehaus.mojo</groupId>
135+
<artifactId>build-helper-maven-plugin</artifactId>
136+
<version>1.9.1</version>
137+
<executions>
138+
<execution>
139+
<id>add-sources-as-resources</id>
140+
<phase>generate-sources</phase>
141+
<goals>
142+
<goal>add-source</goal>
143+
</goals>
144+
<configuration>
145+
<sources>
146+
<source>${project.build.directory}/generated-sources</source>
147+
</sources>
148+
</configuration>
149+
</execution>
150+
</executions>
151+
</plugin>
152+
<plugin>
153+
<groupId>org.apache.maven.plugins</groupId>
154+
<artifactId>maven-checkstyle-plugin</artifactId>
155+
<configuration>
156+
<skip>true</skip>
157+
</configuration>
158+
</plugin>
159+
</plugins>
160+
161+
</build>
162+
</project>
163+

‎java/memory/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
<parent>
1616
<groupId>org.apache.arrow</groupId>
1717
<artifactId>arrow-java-root</artifactId>
18-
<version>0.1-SNAPSHOT</version>
18+
<version>0.1-decimal</version>
1919
</parent>
2020
<artifactId>arrow-memory</artifactId>
2121
<name>arrow-memory</name>

‎java/pom.xml

+2-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
<groupId>org.apache.arrow</groupId>
2323
<artifactId>arrow-java-root</artifactId>
24-
<version>0.1-SNAPSHOT</version>
24+
<version>0.1-decimal</version>
2525
<packaging>pom</packaging>
2626

2727
<name>Apache Arrow Java Root POM</name>
@@ -465,6 +465,7 @@
465465
</dependencies>
466466

467467
<modules>
468+
<module>format</module>
468469
<module>memory</module>
469470
<module>vector</module>
470471
</modules>

‎java/vector/pom.xml

+6-1
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,18 @@
1515
<parent>
1616
<groupId>org.apache.arrow</groupId>
1717
<artifactId>arrow-java-root</artifactId>
18-
<version>0.1-SNAPSHOT</version>
18+
<version>0.1-decimal</version>
1919
</parent>
2020
<artifactId>vector</artifactId>
2121
<name>vectors</name>
2222

2323
<dependencies>
2424

25+
<dependency>
26+
<groupId>org.apache.arrow</groupId>
27+
<artifactId>arrow-format</artifactId>
28+
<version>0.1-decimal</version>
29+
</dependency>
2530
<dependency>
2631
<groupId>org.apache.arrow</groupId>
2732
<artifactId>arrow-memory</artifactId>

‎java/vector/src/main/codegen/config.fmpp

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
data: {
1818
# TODO: Rename to ~valueVectorModesAndTypes for clarity.
1919
vv: tdd(../data/ValueVectorTypes.tdd),
20+
arrowTypes: tdd(../data/ArrowTypes.tdd)
2021

2122
}
2223
freemarkerLinks: {

‎java/vector/src/main/codegen/data/ArrowTypes.tdd

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http:# www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
{
18+
types: [
19+
{
20+
name: "Null",
21+
fields: []
22+
},
23+
{
24+
name: "Tuple",
25+
fields: []
26+
},
27+
{
28+
name: "List",
29+
fields: []
30+
},
31+
{
32+
name: "Union",
33+
fields: []
34+
},
35+
{
36+
name: "Int",
37+
fields: [{name: "bitWidth", type: int}, {name: "isSigned", type: boolean}]
38+
},
39+
{
40+
name: "FloatingPoint",
41+
fields: [{name: precision, type: int}]
42+
},
43+
{
44+
name: "Utf8",
45+
fields: []
46+
},
47+
{
48+
name: "Binary",
49+
fields: []
50+
},
51+
{
52+
name: "Bool",
53+
fields: []
54+
},
55+
{
56+
name: "Decimal",
57+
fields: [{name: "precision", type: int}, {name: "scale", type: int}]
58+
},
59+
{
60+
name: "Date",
61+
fields: []
62+
},
63+
{
64+
name: "Time",
65+
fields: []
66+
},
67+
{
68+
name: "Timestamp",
69+
fields: [{name: "timezone", type: "String"}]
70+
},
71+
{
72+
name: "IntervalDay",
73+
fields: []
74+
},
75+
{
76+
name: "IntervalYear",
77+
fields: []
78+
}
79+
]
80+
}

0 commit comments

Comments
 (0)