-
Notifications
You must be signed in to change notification settings - Fork 387
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add BooleanWritableConverter to use them in SequenceFileLoader in pig #404
base: master
Are you sure you want to change the base?
Changes from all commits
6fff33f
b588594
417b105
2f93ef1
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,108 @@ | ||
package com.twitter.elephantbird.pig.util; | ||
|
||
import org.apache.hadoop.io.BooleanWritable; | ||
import org.apache.pig.ResourceSchema; | ||
import org.apache.pig.data.DataByteArray; | ||
import org.apache.pig.data.DataType; | ||
|
||
import java.io.IOException; | ||
|
||
/** | ||
* Supports conversion between Pig types and {@link org.apache.hadoop.io.BooleanWritable}. | ||
* | ||
* @author Xu Wenhao | ||
*/ | ||
public class BooleanWritableConverter extends AbstractWritableConverter<BooleanWritable> { | ||
public BooleanWritableConverter() { | ||
super(new BooleanWritable()); | ||
} | ||
|
||
@Override | ||
public ResourceSchema.ResourceFieldSchema getLoadSchema() throws IOException { | ||
ResourceSchema.ResourceFieldSchema schema = new ResourceSchema.ResourceFieldSchema(); | ||
schema.setType(DataType.INTEGER); | ||
return schema; | ||
} | ||
|
||
@Override | ||
public Object bytesToObject(DataByteArray dataByteArray) throws IOException { | ||
return bytesToInteger(dataByteArray.get()); | ||
} | ||
|
||
@Override | ||
protected String toCharArray(BooleanWritable writable) throws IOException { | ||
return String.valueOf(writable.get()); | ||
} | ||
|
||
@Override | ||
protected Integer toInteger(BooleanWritable writable) throws IOException { | ||
return writable.get() ? 1 : 0; | ||
} | ||
|
||
@Override | ||
protected Long toLong(BooleanWritable writable) throws IOException { | ||
return writable.get() ? 1L : 0L; | ||
} | ||
|
||
@Override | ||
protected Float toFloat(BooleanWritable writable) throws IOException { | ||
return writable.get() ? 1f : 0f; | ||
} | ||
|
||
@Override | ||
protected Double toDouble(BooleanWritable writable) throws IOException { | ||
return writable.get() ? 1.0 : 0.0; | ||
} | ||
|
||
@Override | ||
public void checkStoreSchema(ResourceSchema.ResourceFieldSchema schema) throws IOException { | ||
switch (schema.getType()) { | ||
case DataType.CHARARRAY: | ||
case DataType.INTEGER: | ||
case DataType.LONG: | ||
case DataType.FLOAT: | ||
case DataType.DOUBLE: | ||
case DataType.BOOLEAN: | ||
return; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why not support string There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ah, I see chararray up top, though looks like it's stringifying the integer zero / one? |
||
} | ||
throw new IOException("Pig type '" + DataType.findTypeName(schema.getType()) + "' unsupported"); | ||
} | ||
|
||
@Override | ||
protected BooleanWritable toWritable(String value) throws IOException { | ||
return toWritable(Integer.parseInt(value)); | ||
} | ||
|
||
@Override | ||
protected BooleanWritable toWritable(Integer value) throws IOException { | ||
int valueInt = value.intValue(); | ||
if (valueInt == 1 ) { | ||
writable.set(true); | ||
} else if (valueInt == 0) { | ||
writable.set(false); | ||
} else { | ||
throw new IllegalArgumentException("Only 1 and 0 could be convert to BooleanWritable!"); | ||
} | ||
return writable; | ||
} | ||
|
||
@Override | ||
protected BooleanWritable toWritable(Long value) throws IOException { | ||
return toWritable(value.intValue()); | ||
} | ||
|
||
@Override | ||
protected BooleanWritable toWritable(Float value) throws IOException { | ||
return toWritable(value.intValue()); | ||
} | ||
|
||
@Override | ||
protected BooleanWritable toWritable(Double value) throws IOException { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Missing an override for There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed the issue mentioned. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Dear Andy, would you mind taking some time to merge it to master? |
||
return toWritable(value.intValue()); | ||
} | ||
|
||
@Override | ||
protected BooleanWritable toWritable(Boolean value) throws IOException { | ||
return toWritable(value.booleanValue() ? 1 : 0); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
package com.twitter.elephantbird.pig.util; | ||
|
||
import org.apache.hadoop.io.BooleanWritable; | ||
|
||
/** | ||
* @author Xu Wenhao | ||
*/ | ||
public class IntegrationTestBooleanWritableConverter extends | ||
AbstractTestWritableConverter<BooleanWritable, BooleanWritableConverter> { | ||
private static final BooleanWritable[] DATA = { new BooleanWritable(true), new BooleanWritable(false)}; | ||
private static final String[] EXPECTED = { "1", "0" }; | ||
|
||
public IntegrationTestBooleanWritableConverter() { | ||
super(BooleanWritable.class, BooleanWritableConverter.class, "", DATA, EXPECTED, "int"); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
DataType.BOOLEAN
should likely be in here:https://github.com/apache/pig/blob/trunk/src/org/apache/pig/data/DataType.java