From 6fff33fbde1da3a8ce245dcb6b70fe2b88c5a7da Mon Sep 17 00:00:00 2001 From: Xu Wenhao Date: Wed, 2 Jul 2014 11:03:21 +0800 Subject: [PATCH 1/4] Add BooleanWritableConverter to use them in SequenceFileLoader in pig --- .../pig/util/BooleanWritableConverter.java | 96 +++++++++++++++++++ ...tegrationTestBooleanWritableConverter.java | 16 ++++ 2 files changed, 112 insertions(+) create mode 100644 pig/src/main/java/com/twitter/elephantbird/pig/util/BooleanWritableConverter.java create mode 100644 pig/src/test/java/com/twitter/elephantbird/pig/util/IntegrationTestBooleanWritableConverter.java diff --git a/pig/src/main/java/com/twitter/elephantbird/pig/util/BooleanWritableConverter.java b/pig/src/main/java/com/twitter/elephantbird/pig/util/BooleanWritableConverter.java new file mode 100644 index 000000000..fd4ec6a31 --- /dev/null +++ b/pig/src/main/java/com/twitter/elephantbird/pig/util/BooleanWritableConverter.java @@ -0,0 +1,96 @@ +package com.twitter.elephantbird.pig.util; + +import com.twitter.elephantbird.pig.util.AbstractWritableConverter; +import org.apache.hadoop.io.BooleanWritable; +import org.apache.pig.ResourceSchema; +import org.apache.pig.data.DataByteArray; +import org.apache.pig.data.DataType; + +import java.io.IOException; + +/** + * Supports conversion between Pig types and {@link org.apache.hadoop.io.BooleanWritable}. + * + * @author Xu Wenhao + */ +public class BooleanWritableConverter extends AbstractWritableConverter { + public BooleanWritableConverter() { + super(new BooleanWritable()); + } + + @Override + public ResourceSchema.ResourceFieldSchema getLoadSchema() throws IOException { + ResourceSchema.ResourceFieldSchema schema = new ResourceSchema.ResourceFieldSchema(); + schema.setType(DataType.INTEGER); + return schema; + } + + @Override + public Object bytesToObject(DataByteArray dataByteArray) throws IOException { + return bytesToInteger(dataByteArray.get()); + } + + @Override + protected String toCharArray(BooleanWritable writable) throws IOException { + return String.valueOf(writable.get()); + } + + @Override + protected Integer toInteger(BooleanWritable writable) throws IOException { + return writable.get() ? 1 : 0; + } + + @Override + protected Long toLong(BooleanWritable writable) throws IOException { + return (long)( writable.get() ? 1:0); + } + + @Override + protected Float toFloat(BooleanWritable writable) throws IOException { + return (float)( writable.get() ? 1:0); + } + + @Override + protected Double toDouble(BooleanWritable writable) throws IOException { + return (double)( writable.get() ? 1: 0); + } + + @Override + public void checkStoreSchema(ResourceSchema.ResourceFieldSchema schema) throws IOException { + switch (schema.getType()) { + case DataType.CHARARRAY: + case DataType.INTEGER: + case DataType.LONG: + case DataType.FLOAT: + case DataType.DOUBLE: + return; + } + throw new IOException("Pig type '" + DataType.findTypeName(schema.getType()) + "' unsupported"); + } + + @Override + protected BooleanWritable toWritable(String value) throws IOException { + return toWritable(Integer.parseInt(value)); + } + + @Override + protected BooleanWritable toWritable(Integer value) throws IOException { + writable.set(value.intValue() == 1 ? true : false); + return writable; + } + + @Override + protected BooleanWritable toWritable(Long value) throws IOException { + return toWritable(value.intValue()); + } + + @Override + protected BooleanWritable toWritable(Float value) throws IOException { + return toWritable(value.intValue()); + } + + @Override + protected BooleanWritable toWritable(Double value) throws IOException { + return toWritable(value.intValue()); + } +} diff --git a/pig/src/test/java/com/twitter/elephantbird/pig/util/IntegrationTestBooleanWritableConverter.java b/pig/src/test/java/com/twitter/elephantbird/pig/util/IntegrationTestBooleanWritableConverter.java new file mode 100644 index 000000000..0aa82d2d9 --- /dev/null +++ b/pig/src/test/java/com/twitter/elephantbird/pig/util/IntegrationTestBooleanWritableConverter.java @@ -0,0 +1,16 @@ +package com.twitter.elephantbird.pig.util; + +import org.apache.hadoop.io.BooleanWritable; + +/** + * @author Xu Wenhao + */ +public class IntegrationTestBooleanWritableConverter extends + AbstractTestWritableConverter { + private static final BooleanWritable[] DATA = { new BooleanWritable(true), new BooleanWritable(false)}; + private static final String[] EXPECTED = { "true", "false" }; + + public IntegrationTestBooleanWritableConverter() { + super(BooleanWritable.class, BooleanWritableConverter.class, "", DATA, EXPECTED, "boolean"); + } +} From b5885940ff6ea0e7f63b1691cb164a8d85758976 Mon Sep 17 00:00:00 2001 From: Xu Wenhao Date: Sat, 5 Jul 2014 16:52:14 +0800 Subject: [PATCH 2/4] Fix the functional test since boolean in pig is represented in 1/0 integer --- .../pig/util/IntegrationTestBooleanWritableConverter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pig/src/test/java/com/twitter/elephantbird/pig/util/IntegrationTestBooleanWritableConverter.java b/pig/src/test/java/com/twitter/elephantbird/pig/util/IntegrationTestBooleanWritableConverter.java index 0aa82d2d9..42e832760 100644 --- a/pig/src/test/java/com/twitter/elephantbird/pig/util/IntegrationTestBooleanWritableConverter.java +++ b/pig/src/test/java/com/twitter/elephantbird/pig/util/IntegrationTestBooleanWritableConverter.java @@ -8,9 +8,9 @@ public class IntegrationTestBooleanWritableConverter extends AbstractTestWritableConverter { private static final BooleanWritable[] DATA = { new BooleanWritable(true), new BooleanWritable(false)}; - private static final String[] EXPECTED = { "true", "false" }; + private static final String[] EXPECTED = { "1", "0" }; public IntegrationTestBooleanWritableConverter() { - super(BooleanWritable.class, BooleanWritableConverter.class, "", DATA, EXPECTED, "boolean"); + super(BooleanWritable.class, BooleanWritableConverter.class, "", DATA, EXPECTED, "int"); } } From 417b105a1512b0f9d56690735d696dbd9acb4932 Mon Sep 17 00:00:00 2001 From: Xu Wenhao Date: Sun, 13 Jul 2014 12:15:40 +0800 Subject: [PATCH 3/4] throw exception if user try to convert numbers other than 1 or 0 to BooleanWritable --- .../pig/util/BooleanWritableConverter.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pig/src/main/java/com/twitter/elephantbird/pig/util/BooleanWritableConverter.java b/pig/src/main/java/com/twitter/elephantbird/pig/util/BooleanWritableConverter.java index fd4ec6a31..2b9739a24 100644 --- a/pig/src/main/java/com/twitter/elephantbird/pig/util/BooleanWritableConverter.java +++ b/pig/src/main/java/com/twitter/elephantbird/pig/util/BooleanWritableConverter.java @@ -42,17 +42,17 @@ protected Integer toInteger(BooleanWritable writable) throws IOException { @Override protected Long toLong(BooleanWritable writable) throws IOException { - return (long)( writable.get() ? 1:0); + return writable.get() ? 1L : 0L; } @Override protected Float toFloat(BooleanWritable writable) throws IOException { - return (float)( writable.get() ? 1:0); + return writable.get() ? 1f : 0f; } @Override protected Double toDouble(BooleanWritable writable) throws IOException { - return (double)( writable.get() ? 1: 0); + return writable.get() ? 1.0 : 0.0; } @Override @@ -75,7 +75,14 @@ protected BooleanWritable toWritable(String value) throws IOException { @Override protected BooleanWritable toWritable(Integer value) throws IOException { - writable.set(value.intValue() == 1 ? true : false); + int valueInt = value.intValue(); + if (valueInt == 1 ) { + writable.set(true); + } else if (valueInt == 0) { + writable.set(false); + } else { + throw new IllegalArgumentException("Only 1 and 0 could be convert to BooleanWritable!"); + } return writable; } From 2f93ef17fea2cd4a14df7f5a28987acac560fceb Mon Sep 17 00:00:00 2001 From: Xu Wenhao Date: Thu, 30 Apr 2015 14:25:32 +0800 Subject: [PATCH 4/4] Add toWritable(Boolean value) metghod --- .../elephantbird/pig/util/BooleanWritableConverter.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pig/src/main/java/com/twitter/elephantbird/pig/util/BooleanWritableConverter.java b/pig/src/main/java/com/twitter/elephantbird/pig/util/BooleanWritableConverter.java index 2b9739a24..e8e635094 100644 --- a/pig/src/main/java/com/twitter/elephantbird/pig/util/BooleanWritableConverter.java +++ b/pig/src/main/java/com/twitter/elephantbird/pig/util/BooleanWritableConverter.java @@ -1,6 +1,5 @@ package com.twitter.elephantbird.pig.util; -import com.twitter.elephantbird.pig.util.AbstractWritableConverter; import org.apache.hadoop.io.BooleanWritable; import org.apache.pig.ResourceSchema; import org.apache.pig.data.DataByteArray; @@ -63,6 +62,7 @@ public void checkStoreSchema(ResourceSchema.ResourceFieldSchema schema) throws I case DataType.LONG: case DataType.FLOAT: case DataType.DOUBLE: + case DataType.BOOLEAN: return; } throw new IOException("Pig type '" + DataType.findTypeName(schema.getType()) + "' unsupported"); @@ -100,4 +100,9 @@ protected BooleanWritable toWritable(Float value) throws IOException { protected BooleanWritable toWritable(Double value) throws IOException { return toWritable(value.intValue()); } + + @Override + protected BooleanWritable toWritable(Boolean value) throws IOException { + return toWritable(value.booleanValue() ? 1 : 0); + } }