diff --git a/data-avro/src/main/java/com/linkedin/data/avro/DataTranslator.java b/data-avro/src/main/java/com/linkedin/data/avro/DataTranslator.java index 21b2697c78..61acbc175a 100644 --- a/data-avro/src/main/java/com/linkedin/data/avro/DataTranslator.java +++ b/data-avro/src/main/java/com/linkedin/data/avro/DataTranslator.java @@ -357,6 +357,7 @@ private Object translate(Object value, DataSchema dataSchema, Schema avroSchema) Schema recordAvroSchema = record.getSchema(); RecordDataSchema recordDataSchema = (RecordDataSchema) dereferencedDataSchema; dataMap = new DataMap(avroSchema.getFields().size()); + String setFieldName = ""; for (RecordDataSchema.Field field : recordDataSchema.getFields()) { String fieldName = field.getName(); @@ -370,16 +371,30 @@ private Object translate(Object value, DataSchema dataSchema, Schema avroSchema) //explicitly check the avro record schema has this field as accessing a non-existent field throws //under avro 1.10+ Schema.Field avroSchemaField = recordAvroSchema.getField(fieldName); - if (avroSchemaField == null) - { - continue; + Object fieldValue = null; + + // If the field is field discriminator enum, but not present in the recordAvroSchema, + // then the recordAvroSchema is the schema generated from proto + // To make the avro object compatible with the one genarated from pegasus translated + // avro schema, we need to set the field discriminator enum value to the field name + // that was set in the record. + if (fieldName.equals(DataSchemaConstants.DISCRIMINATOR_FIELD) && avroSchemaField == null) { + fieldValue = setFieldName; } + else { + if (avroSchemaField == null) { + continue; + } - Object fieldValue = record.get(avroSchemaField.pos()); + fieldValue = record.get(avroSchemaField.pos()); + } if (fieldValue == null) { continue; } + if (!fieldName.equals(DataSchemaConstants.DISCRIMINATOR_FIELD)) { + setFieldName = fieldName; + } DataSchema fieldDataSchema = field.getType(); Schema fieldAvroSchema = avroSchema.getField(fieldName).schema(); if (fieldDataSchema.getDereferencedType() != DataSchema.Type.UNION && fieldAvroSchema.getType() == Schema.Type.UNION) @@ -440,10 +455,47 @@ private Object translate(Object value, DataSchema dataSchema, Schema avroSchema) return result; } + private Schema findRecordSchemaWithoutFieldDiscriminator(Object value, Schema avroSchema) { + for (Schema schema : avroSchema.getTypes()) { + if (schema.getType() == Schema.Type.RECORD) { + Schema.Field fieldDiscriminator = schema.getField(DataSchemaConstants.DISCRIMINATOR_FIELD); + + if (isEnumField(fieldDiscriminator)) { + GenericRecord record = (GenericRecord) value; + + if (recordHasNonNullField(record) && record.get(DataSchemaConstants.DISCRIMINATOR_FIELD) == null) { + return schema; + } + } + } + } + + return null; + } + + private boolean isEnumField(Schema.Field field) { + return field != null && field.schema().getType() == Schema.Type.ENUM; + } + + private boolean recordHasNonNullField(GenericRecord record) { + for (Schema.Field recordField : record.getSchema().getFields()) { + if (record.get(recordField.name()) != null) { + return true; + } + } + + return false; + } + private final Map.Entry findUnionMemberSchema(Object value, UnionDataSchema unionDataSchema, Schema avroSchema) { - int index = _genericData.resolveUnion(avroSchema, value); - Schema memberAvroSchema = avroSchema.getTypes().get(index); + Schema memberAvroSchema = findRecordSchemaWithoutFieldDiscriminator(value, avroSchema); + + if (memberAvroSchema == null) { + int index = _genericData.resolveUnion(avroSchema, value); + memberAvroSchema = avroSchema.getTypes().get(index); + } + String key; switch (memberAvroSchema.getType()) { @@ -479,6 +531,31 @@ private final Map.Entry findUnionMemberSchema(Object value, return new AbstractMap.SimpleEntry<>(memberDataSchema, memberAvroSchema); } + private Object getDataMapForUnionField(String fieldName, Object fieldValue, DataSchema fieldDataSchema, Schema fieldAvroSchema) + { + DataMap dataMap = new DataMap(1); + _path.addLast(fieldName); + dataMap.put(fieldName, translate(fieldValue, fieldDataSchema, extractNonnullSchema(fieldAvroSchema))); + _path.removeLast(); + return dataMap; + + } + + private Object translateAvroRecordToPegasusUnionWithAliasesWithoutDescriminator(GenericRecord record, UnionDataSchema unionDataSchema) { + for (Schema.Field field : record.getSchema().getFields()) + { + Object fieldValue = record.get(field.name()); + if (fieldValue != null) + { + DataSchema memberDataSchema = unionDataSchema.getTypeByMemberKey(field.name()); + Schema memberAvroSchema = field.schema(); + return getDataMapForUnionField(field.name(), fieldValue, memberDataSchema, memberAvroSchema); + } + } + appendMessage("cannot find required field in record %1$s", record); + return BAD_RESULT; + } + private Object translateAvroRecordToPegasusUnionWithAliases(Object value, UnionDataSchema unionDataSchema, Schema avroSchema) { Schema recordAvroSchema = extractNonnullSchema(avroSchema); @@ -489,6 +566,7 @@ private Object translateAvroRecordToPegasusUnionWithAliases(Object value, UnionD { appendMessage("cannot find required field %1$s in record %2$s", DataSchemaConstants.DISCRIMINATOR_FIELD, record); return BAD_RESULT; +// return translateAvroRecordToPegasusUnionWithAliasesWithoutDescriminator(record, unionDataSchema); } String fieldDiscriminator = fieldDiscriminatorValue.toString(); @@ -502,11 +580,12 @@ private Object translateAvroRecordToPegasusUnionWithAliases(Object value, UnionD Schema fieldAvroSchema = recordAvroSchema.getField(fieldDiscriminator).schema(); DataSchema memberDataSchema = unionDataSchema.getTypeByMemberKey(fieldDiscriminator); - DataMap result = new DataMap(1); - _path.add(fieldDiscriminator); - result.put(fieldDiscriminator, translate(fieldValue, memberDataSchema, extractNonnullSchema(fieldAvroSchema))); - _path.removeLast(); - return result; +// DataMap result = new DataMap(1); +// _path.add(fieldDiscriminator); +// result.put(fieldDiscriminator, translate(fieldValue, memberDataSchema, extractNonnullSchema(fieldAvroSchema))); +// _path.removeLast(); +// return result; + return getDataMapForUnionField(fieldDiscriminator, fieldValue, memberDataSchema, fieldAvroSchema); } } } diff --git a/data-avro/src/test/java/com/linkedin/data/avro/TestDataTranslator.java b/data-avro/src/test/java/com/linkedin/data/avro/TestDataTranslator.java index 3abd677853..43d61b578b 100644 --- a/data-avro/src/test/java/com/linkedin/data/avro/TestDataTranslator.java +++ b/data-avro/src/test/java/com/linkedin/data/avro/TestDataTranslator.java @@ -17,6 +17,7 @@ package com.linkedin.data.avro; import com.google.common.collect.ImmutableMap; +import com.linkedin.data.ByteString; import com.linkedin.data.DataList; import com.linkedin.data.DataMap; import com.linkedin.data.TestUtil; @@ -54,1021 +55,309 @@ import org.apache.avro.generic.GenericArray; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; +import org.apache.avro.specific.SpecificRecord; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import static org.testng.Assert.*; -public class TestDataTranslator -{ +public class TestDataTranslator { public static final PrintStream out = new PrintStream(new FileOutputStream(FileDescriptor.out)); public static final String ONE_WAY = "ONE_WAY"; @Test - public void testDataTranslator() throws IOException - { + public void testDataTranslator() throws IOException { boolean debug = false; - String[][][] inputs = - { - // { - // { - // 1 string holding the Pegasus schema in JSON. - // The string may be marked with ##T_START and ##T_END markers. The markers are used for typeref testing. - // If the string these markers, then two schemas will be constructed and tested. - // The first schema replaces these markers with two empty strings. - // The second schema replaces these markers with a typeref enclosing the type between these markers. - // }, - // { - // 1st string is input DataMap, JSON will be deserialized into DataMap. - // 2nd string is expected output after translating from DataMap to Avro GenericRecord - // if translation is successful, this string should be Avro GenericRecord serialized to JSON, - // else the output should be a string providing diagnostic messages regarding the translation - // failure. In this case, the 2nd string provides a string that will be checked against the - // diagnostic messages. The diagnostic message should contain this string. - // } - // } - { - // record with int field - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " { \"name\" : \"intRequired\", \"type\" : ##T_START \"int\" ##T_END }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"intRequired\" : 42 }", - "{\"intRequired\":42}" - }, - { - "{ }", - "Error processing /intRequired" - }, - { - "{ \"intRequired\" : null }", - "Error processing /intRequired" - }, - { - "{ \"intRequired\" : \"string\" }", - "Error processing /intRequired" - } - }, - // record with long field - { - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " { \"name\" : \"longRequired\", \"type\" : ##T_START \"long\" ##T_END }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"longRequired\" : 42 }", - "{\"longRequired\":42}" - }, - { - "{ }", - "Error processing /longRequired" - }, - { - "{ \"longRequired\" : null }", - "Error processing /longRequired" - }, - { - "{ \"longRequired\" : \"string\" }", - "Error processing /longRequired" - } - }, - // record with float field - { - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " { \"name\" : \"floatRequired\", \"type\" : ##T_START \"float\" ##T_END }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"floatRequired\" : 42 }", - "{\"floatRequired\":42.0}" - }, - { - "{ }", - "Error processing /floatRequired" - }, - { - "{ \"floatRequired\" : null }", - "Error processing /floatRequired" - }, - { - "{ \"floatRequired\" : \"string\" }", - "Error processing /floatRequired" - } - }, - // record with double field - { - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " { \"name\" : \"doubleRequired\", \"type\" : ##T_START \"double\" ##T_END }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"doubleRequired\" : 42 }", - "{\"doubleRequired\":42.0}" - }, - { - "{ }", - "Error processing /doubleRequired" - }, - { - "{ \"doubleRequired\" : null }", - "Error processing /doubleRequired" - }, - { - "{ \"doubleRequired\" : \"string\" }", - "Error processing /doubleRequired" - } - }, - { + String[][][] inputs = { + // { + // { + // 1 string holding the Pegasus schema in JSON. + // The string may be marked with ##T_START and ##T_END markers. The markers are used for typeref testing. + // If the string these markers, then two schemas will be constructed and tested. + // The first schema replaces these markers with two empty strings. + // The second schema replaces these markers with a typeref enclosing the type between these markers. + // }, + // { + // 1st string is input DataMap, JSON will be deserialized into DataMap. + // 2nd string is expected output after translating from DataMap to Avro GenericRecord + // if translation is successful, this string should be Avro GenericRecord serialized to JSON, + // else the output should be a string providing diagnostic messages regarding the translation + // failure. In this case, the 2nd string provides a string that will be checked against the + // diagnostic messages. The diagnostic message should contain this string. + // } + // } + { + // record with int field + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + + " { \"name\" : \"intRequired\", \"type\" : ##T_START \"int\" ##T_END }\n" + " ]\n" + "}\n"}, + {"{ \"intRequired\" : 42 }", "{\"intRequired\":42}"}, {"{ }", "Error processing /intRequired"}, + {"{ \"intRequired\" : null }", "Error processing /intRequired"}, + {"{ \"intRequired\" : \"string\" }", "Error processing /intRequired"}}, + // record with long field + {{"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + + " { \"name\" : \"longRequired\", \"type\" : ##T_START \"long\" ##T_END }\n" + " ]\n" + "}\n"}, + {"{ \"longRequired\" : 42 }", "{\"longRequired\":42}"}, {"{ }", "Error processing /longRequired"}, + {"{ \"longRequired\" : null }", "Error processing /longRequired"}, + {"{ \"longRequired\" : \"string\" }", "Error processing /longRequired"}}, + // record with float field + {{"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + + " { \"name\" : \"floatRequired\", \"type\" : ##T_START \"float\" ##T_END }\n" + " ]\n" + "}\n"}, + {"{ \"floatRequired\" : 42 }", "{\"floatRequired\":42.0}"}, {"{ }", "Error processing /floatRequired"}, + {"{ \"floatRequired\" : null }", "Error processing /floatRequired"}, + {"{ \"floatRequired\" : \"string\" }", "Error processing /floatRequired"}}, + // record with double field + {{"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + + " { \"name\" : \"doubleRequired\", \"type\" : ##T_START \"double\" ##T_END }\n" + " ]\n" + "}\n"}, + {"{ \"doubleRequired\" : 42 }", "{\"doubleRequired\":42.0}"}, {"{ }", "Error processing /doubleRequired"}, + {"{ \"doubleRequired\" : null }", "Error processing /doubleRequired"}, + {"{ \"doubleRequired\" : \"string\" }", "Error processing /doubleRequired"}}, { // record with boolean field - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " { \"name\" : \"booleanRequired\", \"type\" : ##T_START \"boolean\" ##T_END }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"booleanRequired\" : true }", - "{\"booleanRequired\":true}" - }, - { - "{ \"booleanRequired\" : false }", - "{\"booleanRequired\":false}" - }, - { - "{ }", - "Error processing /booleanRequired" - }, - { - "{ \"booleanRequired\" : null }", - "Error processing /booleanRequired" - }, - { - "{ \"booleanRequired\" : \"string\" }", - "Error processing /booleanRequired" - } - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + + " { \"name\" : \"booleanRequired\", \"type\" : ##T_START \"boolean\" ##T_END }\n" + " ]\n" + "}\n"}, + {"{ \"booleanRequired\" : true }", "{\"booleanRequired\":true}"}, + {"{ \"booleanRequired\" : false }", "{\"booleanRequired\":false}"}, + {"{ }", "Error processing /booleanRequired"}, + {"{ \"booleanRequired\" : null }", "Error processing /booleanRequired"}, + {"{ \"booleanRequired\" : \"string\" }", "Error processing /booleanRequired"}}, { // record with string field - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " { \"name\" : \"stringRequired\", \"type\" : ##T_START \"string\" ##T_END }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"stringRequired\" : \"bar\" }", - "{\"stringRequired\":\"bar\"}" - }, - { - "{ }", - "Error processing /stringRequired" - }, - { - "{ \"stringRequired\" : null }", - "Error processing /stringRequired" - }, - { - "{ \"stringRequired\" : false }", - "Error processing /stringRequired" - } - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + + " { \"name\" : \"stringRequired\", \"type\" : ##T_START \"string\" ##T_END }\n" + " ]\n" + "}\n"}, + {"{ \"stringRequired\" : \"bar\" }", "{\"stringRequired\":\"bar\"}"}, + {"{ }", "Error processing /stringRequired"}, + {"{ \"stringRequired\" : null }", "Error processing /stringRequired"}, + {"{ \"stringRequired\" : false }", "Error processing /stringRequired"}}, { // record with bytes field - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " { \"name\" : \"bytesRequired\", \"type\" : ##T_START \"bytes\" ##T_END }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"bytesRequired\" : \"12345\\u0001\" }", - "{\"bytesRequired\":\"12345\\u0001\"}" - }, - { - "{ }", - "Error processing /bytesRequired" - }, - { - "{ \"bytesRequired\" : null }", - "Error processing /bytesRequired" - }, - { - "{ \"bytesRequired\" : false }", - "Error processing /bytesRequired" - }, - { - "{ \"bytesRequired\" : \"1234\\u0101\" }", - "Error processing /bytesRequired" - } - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + + " { \"name\" : \"bytesRequired\", \"type\" : ##T_START \"bytes\" ##T_END }\n" + " ]\n" + "}\n"}, + {"{ \"bytesRequired\" : \"12345\\u0001\" }", "{\"bytesRequired\":\"12345\\u0001\"}"}, + {"{ }", "Error processing /bytesRequired"}, {"{ \"bytesRequired\" : null }", "Error processing /bytesRequired"}, + {"{ \"bytesRequired\" : false }", "Error processing /bytesRequired"}, + {"{ \"bytesRequired\" : \"1234\\u0101\" }", "Error processing /bytesRequired"}}, { // record with fixed field - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"fixedRequired\",\n" + - " \"type\" : ##T_START { \"type\" : \"fixed\", \"name\" : \"Fixed5\", \"size\" : 5 } ##T_END\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"fixedRequired\" : \"12345\" }", - "{\"fixedRequired\":\"12345\"}" - }, - { - "{ }", - "Error processing /fixedRequired" - }, - { - "{ \"fixedRequired\" : null }", - "Error processing /fixedRequired" - }, - { - "{ \"fixedRequired\" : false }", - "Error processing /fixedRequired" - }, - { - "{ \"fixedRequired\" : \"1234\" }", - "Error processing /fixedRequired" - }, - { - "{ \"fixedRequired\" : \"123456\" }", - "Error processing /fixedRequired" - } - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"fixedRequired\",\n" + + " \"type\" : ##T_START { \"type\" : \"fixed\", \"name\" : \"Fixed5\", \"size\" : 5 } ##T_END\n" + + " }\n" + " ]\n" + "}\n"}, {"{ \"fixedRequired\" : \"12345\" }", "{\"fixedRequired\":\"12345\"}"}, + {"{ }", "Error processing /fixedRequired"}, {"{ \"fixedRequired\" : null }", "Error processing /fixedRequired"}, + {"{ \"fixedRequired\" : false }", "Error processing /fixedRequired"}, + {"{ \"fixedRequired\" : \"1234\" }", "Error processing /fixedRequired"}, + {"{ \"fixedRequired\" : \"123456\" }", "Error processing /fixedRequired"}}, { // record with enum field - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"enumRequired\",\n" + - " \"type\" : ##T_START {\n" + - " \"name\" : \"Fruits\",\n" + - " \"type\" : \"enum\",\n" + - " \"symbols\" : [ \"APPLE\", \"ORANGE\" ]\n" + - " } ##T_END\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"enumRequired\" : \"APPLE\" }", - "{\"enumRequired\":\"APPLE\"}" - }, - { - "{ \"enumRequired\" : \"ORANGE\" }", - "{\"enumRequired\":\"ORANGE\"}" - }, - { - "{ }", - "Error processing /enumRequired" - }, - { - "{ \"enumRequired\" : null }", - "Error processing /enumRequired" - }, - { - "{ \"enumRequired\" : false }", - "Error processing /enumRequired" - } - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"enumRequired\",\n" + " \"type\" : ##T_START {\n" + + " \"name\" : \"Fruits\",\n" + " \"type\" : \"enum\",\n" + + " \"symbols\" : [ \"APPLE\", \"ORANGE\" ]\n" + " } ##T_END\n" + " }\n" + " ]\n" + "}\n"}, + {"{ \"enumRequired\" : \"APPLE\" }", "{\"enumRequired\":\"APPLE\"}"}, + {"{ \"enumRequired\" : \"ORANGE\" }", "{\"enumRequired\":\"ORANGE\"}"}, + {"{ }", "Error processing /enumRequired"}, {"{ \"enumRequired\" : null }", "Error processing /enumRequired"}, + {"{ \"enumRequired\" : false }", "Error processing /enumRequired"}}, { // record with array field - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"arrayRequired\",\n" + - " \"type\" : ##T_START {\n" + - " \"type\" : \"array\",\n" + - " \"items\" : {\n" + - " \"name\" : \"Fruits\",\n" + - " \"type\" : \"enum\",\n" + - " \"symbols\" : [ \"APPLE\", \"ORANGE\" ]\n" + - " }\n" + - " } ##T_END\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"arrayRequired\" : [] }", - "{\"arrayRequired\":[]}" - }, - { - "{ \"arrayRequired\" : [ \"APPLE\", \"ORANGE\" ] }", - "{\"arrayRequired\":[\"APPLE\",\"ORANGE\"]}" - }, - { - "{ }", - "Error processing /arrayRequired" - }, - { - "{ \"arrayRequired\" : null }", - "Error processing /arrayRequired" - }, - { - "{ \"arrayRequired\" : {} }", - "Error processing /arrayRequired" - }, - { - "{ \"arrayRequired\" : [ null ] }", - "Error processing /arrayRequired/0" - }, - { - "{ \"arrayRequired\" : [ \"APPLE\", \"PINEAPPLE\" ] }", - "Error processing /arrayRequired/1" - } - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"arrayRequired\",\n" + " \"type\" : ##T_START {\n" + + " \"type\" : \"array\",\n" + " \"items\" : {\n" + " \"name\" : \"Fruits\",\n" + + " \"type\" : \"enum\",\n" + " \"symbols\" : [ \"APPLE\", \"ORANGE\" ]\n" + " }\n" + + " } ##T_END\n" + " }\n" + " ]\n" + "}\n"}, + {"{ \"arrayRequired\" : [] }", "{\"arrayRequired\":[]}"}, + {"{ \"arrayRequired\" : [ \"APPLE\", \"ORANGE\" ] }", "{\"arrayRequired\":[\"APPLE\",\"ORANGE\"]}"}, + {"{ }", "Error processing /arrayRequired"}, {"{ \"arrayRequired\" : null }", "Error processing /arrayRequired"}, + {"{ \"arrayRequired\" : {} }", "Error processing /arrayRequired"}, + {"{ \"arrayRequired\" : [ null ] }", "Error processing /arrayRequired/0"}, + {"{ \"arrayRequired\" : [ \"APPLE\", \"PINEAPPLE\" ] }", "Error processing /arrayRequired/1"}}, { // record with map field - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"mapRequired\",\n" + - " \"type\" : ##T_START {\n" + - " \"type\" : \"map\",\n" + - " \"values\" : \"int\" " + - " } ##T_END\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"mapRequired\" : {} }", - "{\"mapRequired\":{}}" - }, - { - "{ \"mapRequired\" : { \"x\" : 1} }", - "{\"mapRequired\":{\"x\":1}}" - }, - { - "{ }", - "Error processing /mapRequired" - }, - { - "{ \"mapRequired\" : null }", - "Error processing /mapRequired" - }, - { - "{ \"mapRequired\" : [] }", - "Error processing /mapRequired" - }, - { - "{ \"mapRequired\" : { \"x\" : null } }", - "Error processing /mapRequired/x" - }, - { - "{ \"mapRequired\" : { \"x\" : \"PINEAPPLE\" } }", - "Error processing /mapRequired/x" - } - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"mapRequired\",\n" + " \"type\" : ##T_START {\n" + " \"type\" : \"map\",\n" + + " \"values\" : \"int\" " + " } ##T_END\n" + " }\n" + " ]\n" + "}\n"}, + {"{ \"mapRequired\" : {} }", "{\"mapRequired\":{}}"}, + {"{ \"mapRequired\" : { \"x\" : 1} }", "{\"mapRequired\":{\"x\":1}}"}, {"{ }", "Error processing /mapRequired"}, + {"{ \"mapRequired\" : null }", "Error processing /mapRequired"}, + {"{ \"mapRequired\" : [] }", "Error processing /mapRequired"}, + {"{ \"mapRequired\" : { \"x\" : null } }", "Error processing /mapRequired/x"}, + {"{ \"mapRequired\" : { \"x\" : \"PINEAPPLE\" } }", "Error processing /mapRequired/x"}}, { // record with union field - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"foo.Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"unionRequired\",\n" + - " \"type\" : ##T_START [ \"int\", \"string\", \"foo.Foo\" ] ##T_END\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"unionRequired\" : { \"int\" : 5 } }", - "{\"unionRequired\":{\"int\":5}}" - }, - { - "{ \"unionRequired\" : { \"string\" : \"s1\" } }", - "{\"unionRequired\":{\"string\":\"s1\"}}" - }, - { - "{ \"unionRequired\" : { \"foo.Foo\" : { \"unionRequired\" : { \"int\" : 5 } } } }", - "{\"unionRequired\":{\"##NS(foo.)Foo\":{\"unionRequired\":{\"int\":5}}}}" - }, - { - "{ }", - "Error processing /unionRequired" - }, - { - "{ \"unionRequired\" : null }", - "Error processing /unionRequired" - }, - { - "{ \"unionRequired\" : {} }", - "Error processing /unionRequired" - }, - { - "{ \"unionRequired\" : { \"bad\" : 0 } }", - "Error processing /unionRequired" - } - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"foo.Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"unionRequired\",\n" + + " \"type\" : ##T_START [ \"int\", \"string\", \"foo.Foo\" ] ##T_END\n" + " }\n" + " ]\n" + + "}\n"}, {"{ \"unionRequired\" : { \"int\" : 5 } }", "{\"unionRequired\":{\"int\":5}}"}, + {"{ \"unionRequired\" : { \"string\" : \"s1\" } }", "{\"unionRequired\":{\"string\":\"s1\"}}"}, + {"{ \"unionRequired\" : { \"foo.Foo\" : { \"unionRequired\" : { \"int\" : 5 } } } }", + "{\"unionRequired\":{\"##NS(foo.)Foo\":{\"unionRequired\":{\"int\":5}}}}"}, + {"{ }", "Error processing /unionRequired"}, {"{ \"unionRequired\" : null }", "Error processing /unionRequired"}, + {"{ \"unionRequired\" : {} }", "Error processing /unionRequired"}, + {"{ \"unionRequired\" : { \"bad\" : 0 } }", "Error processing /unionRequired"}}, { // record with a required "union with aliases" field - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"foo.Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"uwaRequiredNoNull\",\n" + - " \"type\" : ##T_START [\n" + - " { \"alias\": \"success\", \"type\": \"string\" },\n" + - " { \"alias\": \"failure\", \"type\": \"string\" }\n" + - " ] ##T_END\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"uwaRequiredNoNull\" : { \"success\" : \"Union with aliases!\" } }", - "{\"uwaRequiredNoNull\":{\"success\":{\"string\":\"Union with aliases!\"},\"failure\":null,\"fieldDiscriminator\":\"success\"}}" - }, - { - "{ \"uwaRequiredNoNull\" : { \"failure\" : \"Union with aliases!\" } }", - "{\"uwaRequiredNoNull\":{\"success\":null,\"failure\":{\"string\":\"Union with aliases!\"},\"fieldDiscriminator\":\"failure\"}}" - }, - { - "{ \"uwaRequiredNoNull\" : null }", - "Error processing /uwaRequiredNoNull" - }, - { - "{}", - "Error processing /uwaRequiredNoNull" - }, - { - "{ \"uwaRequiredNoNull\" : {} }", - "Error processing /uwaRequiredNoNull" - }, - { - "{ \"uwaRequiredNoNull\" : \"Union with aliases!\" }", - "Error processing /uwaRequiredNoNull" - }, - { - "{ \"uwaRequiredNoNull\" : { \"string\" : \"Union with aliases!\" } }", - "Error processing /uwaRequiredNoNull" - }, - { - "{ \"uwaRequiredNoNull\" : { \"success\" : 123 } }", - "Error processing /uwaRequiredNoNull/success" - } - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"foo.Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"uwaRequiredNoNull\",\n" + " \"type\" : ##T_START [\n" + + " { \"alias\": \"success\", \"type\": \"string\" },\n" + + " { \"alias\": \"failure\", \"type\": \"string\" }\n" + " ] ##T_END\n" + " }\n" + " ]\n" + + "}\n"}, {"{ \"uwaRequiredNoNull\" : { \"success\" : \"Union with aliases!\" } }", + "{\"uwaRequiredNoNull\":{\"success\":{\"string\":\"Union with aliases!\"},\"failure\":null,\"fieldDiscriminator\":\"success\"}}"}, + {"{ \"uwaRequiredNoNull\" : { \"failure\" : \"Union with aliases!\" } }", + "{\"uwaRequiredNoNull\":{\"success\":null,\"failure\":{\"string\":\"Union with aliases!\"},\"fieldDiscriminator\":\"failure\"}}"}, + {"{ \"uwaRequiredNoNull\" : null }", "Error processing /uwaRequiredNoNull"}, + {"{}", "Error processing /uwaRequiredNoNull"}, + {"{ \"uwaRequiredNoNull\" : {} }", "Error processing /uwaRequiredNoNull"}, + {"{ \"uwaRequiredNoNull\" : \"Union with aliases!\" }", "Error processing /uwaRequiredNoNull"}, + {"{ \"uwaRequiredNoNull\" : { \"string\" : \"Union with aliases!\" } }", "Error processing /uwaRequiredNoNull"}, + {"{ \"uwaRequiredNoNull\" : { \"success\" : 123 } }", "Error processing /uwaRequiredNoNull/success"}}, { // record with a required "union with aliases" field with null member - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"foo.Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"uwaRequiredWithNull\",\n" + - " \"type\" : ##T_START [\n" + - " \"null\",\n" + - " { \"alias\": \"success\", \"type\": \"string\" },\n" + - " { \"alias\": \"failure\", \"type\": \"string\" }\n" + - " ] ##T_END\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"uwaRequiredWithNull\" : { \"success\" : \"Union with aliases!\" } }", - "{\"uwaRequiredWithNull\":{\"success\":{\"string\":\"Union with aliases!\"},\"failure\":null,\"fieldDiscriminator\":\"success\"}}" - }, - { - "{ \"uwaRequiredWithNull\" : { \"failure\" : \"Union with aliases!\" } }", - "{\"uwaRequiredWithNull\":{\"success\":null,\"failure\":{\"string\":\"Union with aliases!\"},\"fieldDiscriminator\":\"failure\"}}" - }, - { - "{ \"uwaRequiredWithNull\" : null }", - "{\"uwaRequiredWithNull\":{\"success\":null,\"failure\":null,\"fieldDiscriminator\":\"null\"}}" - }, - { - "{}", - "Error processing /uwaRequiredWithNull" - }, - { - "{ \"uwaRequiredWithNull\" : {} }", - "Error processing /uwaRequiredWithNull" - }, - { - "{ \"uwaRequiredWithNull\" : \"Union with aliases!\" }", - "Error processing /uwaRequiredWithNull" - }, - { - "{ \"uwaRequiredWithNull\" : { \"string\" : \"Union with aliases!\" } }", - "Error processing /uwaRequiredWithNull" - }, - { - "{ \"uwaRequiredWithNull\" : { \"success\" : 123 } }", - "Error processing /uwaRequiredWithNull/success" - } - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"foo.Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"uwaRequiredWithNull\",\n" + " \"type\" : ##T_START [\n" + " \"null\",\n" + + " { \"alias\": \"success\", \"type\": \"string\" },\n" + + " { \"alias\": \"failure\", \"type\": \"string\" }\n" + " ] ##T_END\n" + " }\n" + " ]\n" + + "}\n"}, {"{ \"uwaRequiredWithNull\" : { \"success\" : \"Union with aliases!\" } }", + "{\"uwaRequiredWithNull\":{\"success\":{\"string\":\"Union with aliases!\"},\"failure\":null,\"fieldDiscriminator\":\"success\"}}"}, + {"{ \"uwaRequiredWithNull\" : { \"failure\" : \"Union with aliases!\" } }", + "{\"uwaRequiredWithNull\":{\"success\":null,\"failure\":{\"string\":\"Union with aliases!\"},\"fieldDiscriminator\":\"failure\"}}"}, + {"{ \"uwaRequiredWithNull\" : null }", + "{\"uwaRequiredWithNull\":{\"success\":null,\"failure\":null,\"fieldDiscriminator\":\"null\"}}"}, + {"{}", "Error processing /uwaRequiredWithNull"}, + {"{ \"uwaRequiredWithNull\" : {} }", "Error processing /uwaRequiredWithNull"}, + {"{ \"uwaRequiredWithNull\" : \"Union with aliases!\" }", "Error processing /uwaRequiredWithNull"}, + {"{ \"uwaRequiredWithNull\" : { \"string\" : \"Union with aliases!\" } }", + "Error processing /uwaRequiredWithNull"}, + {"{ \"uwaRequiredWithNull\" : { \"success\" : 123 } }", "Error processing /uwaRequiredWithNull/success"}}, { // record with array of union with null field // this is to check that translation of union with null that does not get converted to optional, // and that null union member translates correctly from Data to Avro and Avro to Data. - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"foo.Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"arrayOfUnionWitNull\",\n" + - " \"type\" : ##T_START { \"type\" : \"array\", \"items\" : [ \"int\", \"null\" ] } ##T_END\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"arrayOfUnionWitNull\" : [ { \"int\" : 5 } ] }", - "{\"arrayOfUnionWitNull\":[{\"int\":5}]}" - }, - { - "{ \"arrayOfUnionWitNull\" : [ null ] }", - "{\"arrayOfUnionWitNull\":[null]}" - }, - { - "{ }", - "Error processing /arrayOfUnionWitNull" - }, - { - "{ \"arrayOfUnionWitNull\" : [ {} ] }", - "Error processing /arrayOfUnionWitNull/0" - }, - { - "{ \"arrayOfUnionWitNull\" : [ { \"bad\" : 0 } ] }", - "Error processing /arrayOfUnionWitNull/0" - } - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"foo.Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"arrayOfUnionWitNull\",\n" + + " \"type\" : ##T_START { \"type\" : \"array\", \"items\" : [ \"int\", \"null\" ] } ##T_END\n" + + " }\n" + " ]\n" + "}\n"}, + {"{ \"arrayOfUnionWitNull\" : [ { \"int\" : 5 } ] }", "{\"arrayOfUnionWitNull\":[{\"int\":5}]}"}, + {"{ \"arrayOfUnionWitNull\" : [ null ] }", "{\"arrayOfUnionWitNull\":[null]}"}, + {"{ }", "Error processing /arrayOfUnionWitNull"}, + {"{ \"arrayOfUnionWitNull\" : [ {} ] }", "Error processing /arrayOfUnionWitNull/0"}, + {"{ \"arrayOfUnionWitNull\" : [ { \"bad\" : 0 } ] }", "Error processing /arrayOfUnionWitNull/0"}}, { // record with record field. - { - "{ \n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"bar\",\n" + - " \"type\" : ##T_START {\n" + - " \"name\" : \"Bar\",\n" + - " \"type\" : \"record\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"baz\",\n" + - " \"type\" : \"int\"\n" + - " }\n" + - " ]\n" + - " } ##T_END\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"bar\" : { \"baz\" : 1 } }", - "{\"bar\":{\"baz\":1}}" - }, - { - "{ \"bar\" : { \"baz\" : null } }", - "Error processing /bar/baz" - }, - }, - // - // Optional - // - { - // record with optional non-union field - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"intOptional\",\n" + - " \"type\" : ##T_START \"int\" ##T_END,\n" + - " \"optional\" : true\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ }", - "{\"intOptional\":null}" - }, - { - "{ \"intOptional\" : 42 }", - "{\"intOptional\":{\"int\":42}}" - }, - { - "{ \"intOptional\" : null }", - ONE_WAY, - "{\"intOptional\":null}" - }, - { - "{ \"intOptional\" : \"s1\" }", - "Error processing /intOptional" - }, - { - "{ \"intOptional\" : {} }", - "Error processing /intOptional" - }, - }, - { + {"{ \n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"bar\",\n" + " \"type\" : ##T_START {\n" + " \"name\" : \"Bar\",\n" + + " \"type\" : \"record\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"baz\",\n" + " \"type\" : \"int\"\n" + " }\n" + " ]\n" + + " } ##T_END\n" + " }\n" + " ]\n" + "}\n"}, + {"{ \"bar\" : { \"baz\" : 1 } }", "{\"bar\":{\"baz\":1}}"}, + {"{ \"bar\" : { \"baz\" : null } }", "Error processing /bar/baz"},}, + // + // Optional + // + { + // record with optional non-union field + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"intOptional\",\n" + " \"type\" : ##T_START \"int\" ##T_END,\n" + + " \"optional\" : true\n" + " }\n" + " ]\n" + "}\n"}, {"{ }", "{\"intOptional\":null}"}, + {"{ \"intOptional\" : 42 }", "{\"intOptional\":{\"int\":42}}"}, + {"{ \"intOptional\" : null }", ONE_WAY, "{\"intOptional\":null}"}, + {"{ \"intOptional\" : \"s1\" }", "Error processing /intOptional"}, + {"{ \"intOptional\" : {} }", "Error processing /intOptional"},}, { // record with optional union field that does not include null - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"unionOptional\",\n" + - " \"type\" : ##T_START [ \"int\", \"string\" ] ##T_END,\n" + - " \"optional\" : true\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ }", - "{\"unionOptional\":null}" - }, - { - "{ \"unionOptional\" : { \"int\" : 42 } }", - "{\"unionOptional\":{\"int\":42}}" - }, - { - "{ \"unionOptional\" : { \"string\" : \"s1\" } }", - "{\"unionOptional\":{\"string\":\"s1\"}}" - }, - { - "{ \"unionOptional\" : null }", - ONE_WAY, - "{\"unionOptional\":null}" - }, - { - "{ \"unionOptional\" : \"s1\" }", - "Error processing /unionOptional" - }, - { - "{ \"unionOptional\" : {} }", - "Error processing /unionOptional" - }, - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"unionOptional\",\n" + " \"type\" : ##T_START [ \"int\", \"string\" ] ##T_END,\n" + + " \"optional\" : true\n" + " }\n" + " ]\n" + "}\n"}, {"{ }", "{\"unionOptional\":null}"}, + {"{ \"unionOptional\" : { \"int\" : 42 } }", "{\"unionOptional\":{\"int\":42}}"}, + {"{ \"unionOptional\" : { \"string\" : \"s1\" } }", "{\"unionOptional\":{\"string\":\"s1\"}}"}, + {"{ \"unionOptional\" : null }", ONE_WAY, "{\"unionOptional\":null}"}, + {"{ \"unionOptional\" : \"s1\" }", "Error processing /unionOptional"}, + {"{ \"unionOptional\" : {} }", "Error processing /unionOptional"},}, { // record with optional union field that includes null - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"unionOptional\",\n" + - " \"type\" : ##T_START [ \"null\", \"string\" ] ##T_END,\n" + - " \"optional\" : true\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ }", - "{\"unionOptional\":null}" - }, - { - "{ \"unionOptional\" : { \"string\" : \"s1\" } }", - "{\"unionOptional\":{\"string\":\"s1\"}}" - }, - { - "{ \"unionOptional\" : null }", - // The round-trip result will drop the optional field. - // A null in the union is translated to an absent field. - ONE_WAY, - "{\"unionOptional\":null}" - }, - { - "{ \"unionOptional\" : \"s1\" }", - "Error processing /unionOptional" - }, - { - "{ \"unionOptional\" : {} }", - "Error processing /unionOptional" - }, - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"unionOptional\",\n" + " \"type\" : ##T_START [ \"null\", \"string\" ] ##T_END,\n" + + " \"optional\" : true\n" + " }\n" + " ]\n" + "}\n"}, {"{ }", "{\"unionOptional\":null}"}, + {"{ \"unionOptional\" : { \"string\" : \"s1\" } }", "{\"unionOptional\":{\"string\":\"s1\"}}"}, + {"{ \"unionOptional\" : null }", + // The round-trip result will drop the optional field. + // A null in the union is translated to an absent field. + ONE_WAY, "{\"unionOptional\":null}"}, {"{ \"unionOptional\" : \"s1\" }", "Error processing /unionOptional"}, + {"{ \"unionOptional\" : {} }", "Error processing /unionOptional"},}, { // record with an optional "union with aliases" field - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"foo.Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"uwaOptionalNoNull\",\n" + - " \"type\" : ##T_START [\n" + - " { \"alias\": \"success\", \"type\": \"string\" },\n" + - " { \"alias\": \"failure\", \"type\": \"string\" }\n" + - " ] ##T_END,\n" + - " \"optional\": true\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"uwaOptionalNoNull\" : { \"success\" : \"Union with aliases!\" } }", - "{\"uwaOptionalNoNull\":{\"##NS(foo.)FooUwaOptionalNoNull\":{\"success\":{\"string\":\"Union with aliases!\"},\"failure\":null,\"fieldDiscriminator\":\"success\"}}}" - }, - { - "{}", - "{\"uwaOptionalNoNull\":null}" - }, - { - "{ \"uwaOptionalNoNull\" : null }", - ONE_WAY, - "{\"uwaOptionalNoNull\":null}" - }, - { - "{ \"uwaOptionalNoNull\" : {} }", - "Error processing /uwaOptionalNoNull" - }, - { - "{ \"uwaOptionalNoNull\" : \"Union with aliases!\" }", - "Error processing /uwaOptionalNoNull" - }, - { - "{ \"uwaOptionalNoNull\" : { \"string\" : \"Union with aliases!\" } }", - "Error processing /uwaOptionalNoNull" - }, - { - "{ \"uwaOptionalNoNull\" : { \"success\" : 123 } }", - "Error processing /uwaOptionalNoNull/success" - } - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"foo.Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"uwaOptionalNoNull\",\n" + " \"type\" : ##T_START [\n" + + " { \"alias\": \"success\", \"type\": \"string\" },\n" + + " { \"alias\": \"failure\", \"type\": \"string\" }\n" + " ] ##T_END,\n" + + " \"optional\": true\n" + " }\n" + " ]\n" + "}\n"}, + {"{ \"uwaOptionalNoNull\" : { \"success\" : \"Union with aliases!\" } }", + "{\"uwaOptionalNoNull\":{\"##NS(foo.)FooUwaOptionalNoNull\":{\"success\":{\"string\":\"Union with aliases!\"},\"failure\":null,\"fieldDiscriminator\":\"success\"}}}"}, + {"{}", "{\"uwaOptionalNoNull\":null}"}, + {"{ \"uwaOptionalNoNull\" : null }", ONE_WAY, "{\"uwaOptionalNoNull\":null}"}, + {"{ \"uwaOptionalNoNull\" : {} }", "Error processing /uwaOptionalNoNull"}, + {"{ \"uwaOptionalNoNull\" : \"Union with aliases!\" }", "Error processing /uwaOptionalNoNull"}, + {"{ \"uwaOptionalNoNull\" : { \"string\" : \"Union with aliases!\" } }", "Error processing /uwaOptionalNoNull"}, + {"{ \"uwaOptionalNoNull\" : { \"success\" : 123 } }", "Error processing /uwaOptionalNoNull/success"}}, { // record with an optional "union with aliases" field with null member - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"foo.Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"uwaOptionalWithNull\",\n" + - " \"type\" : ##T_START [\n" + - " \"null\",\n" + - " { \"alias\": \"success\", \"type\": \"string\" },\n" + - " { \"alias\": \"failure\", \"type\": \"string\" }\n" + - " ] ##T_END,\n" + - " \"optional\": true\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"uwaOptionalWithNull\" : { \"success\" : \"Union with aliases!\" } }", - "{\"uwaOptionalWithNull\":{\"##NS(foo.)FooUwaOptionalWithNull\":{\"success\":{\"string\":\"Union with aliases!\"},\"failure\":null,\"fieldDiscriminator\":\"success\"}}}" - }, - { - "{}", - "{\"uwaOptionalWithNull\":null}" - }, - { - "{ \"uwaOptionalWithNull\" : null }", - "{\"uwaOptionalWithNull\":null}" - }, - { - "{ \"uwaOptionalWithNull\" : {} }", - "Error processing /uwaOptionalWithNull" - }, - { - "{ \"uwaOptionalWithNull\" : \"Union with aliases!\" }", - "Error processing /uwaOptionalWithNull" - }, - { - "{ \"uwaOptionalWithNull\" : { \"string\" : \"Union with aliases!\" } }", - "Error processing /uwaOptionalWithNull" - }, - { - "{ \"uwaOptionalWithNull\" : { \"success\" : 123 } }", - "Error processing /uwaOptionalWithNull/success" - } - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"foo.Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"uwaOptionalWithNull\",\n" + " \"type\" : ##T_START [\n" + " \"null\",\n" + + " { \"alias\": \"success\", \"type\": \"string\" },\n" + + " { \"alias\": \"failure\", \"type\": \"string\" }\n" + " ] ##T_END,\n" + + " \"optional\": true\n" + " }\n" + " ]\n" + "}\n"}, + {"{ \"uwaOptionalWithNull\" : { \"success\" : \"Union with aliases!\" } }", + "{\"uwaOptionalWithNull\":{\"##NS(foo.)FooUwaOptionalWithNull\":{\"success\":{\"string\":\"Union with aliases!\"},\"failure\":null,\"fieldDiscriminator\":\"success\"}}}"}, + {"{}", "{\"uwaOptionalWithNull\":null}"}, + {"{ \"uwaOptionalWithNull\" : null }", "{\"uwaOptionalWithNull\":null}"}, + {"{ \"uwaOptionalWithNull\" : {} }", "Error processing /uwaOptionalWithNull"}, + {"{ \"uwaOptionalWithNull\" : \"Union with aliases!\" }", "Error processing /uwaOptionalWithNull"}, + {"{ \"uwaOptionalWithNull\" : { \"string\" : \"Union with aliases!\" } }", + "Error processing /uwaOptionalWithNull"}, + {"{ \"uwaOptionalWithNull\" : { \"success\" : 123 } }", "Error processing /uwaOptionalWithNull/success"}}, { // record with optional enum field - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"enumOptional\",\n" + - " \"type\" : ##T_START { \"type\" : \"enum\", \"name\" : \"foo.bar\", \"symbols\" : [ \"A\", \"B\" ] } ##T_END,\n" + - " \"optional\" : true\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ }", - "{\"enumOptional\":null}" - }, - { - "{ \"enumOptional\" : \"A\" } }", - "{\"enumOptional\":{\"##NS(foo.)bar\":\"A\"}}" - }, - { - "{ \"enumOptional\" : \"B\" } }", - "{\"enumOptional\":{\"##NS(foo.)bar\":\"B\"}}" - }, - { - "{ \"enumOptional\" : {} }", - "Error processing /enumOptional" - }, - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"enumOptional\",\n" + + " \"type\" : ##T_START { \"type\" : \"enum\", \"name\" : \"foo.bar\", \"symbols\" : [ \"A\", \"B\" ] } ##T_END,\n" + + " \"optional\" : true\n" + " }\n" + " ]\n" + "}\n"}, {"{ }", "{\"enumOptional\":null}"}, + {"{ \"enumOptional\" : \"A\" } }", "{\"enumOptional\":{\"##NS(foo.)bar\":\"A\"}}"}, + {"{ \"enumOptional\" : \"B\" } }", "{\"enumOptional\":{\"##NS(foo.)bar\":\"B\"}}"}, + {"{ \"enumOptional\" : {} }", "Error processing /enumOptional"},}, { // record with optional enum field - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"enumOptional\",\n" + - " \"type\" : ##T_START { \"type\" : \"enum\", \"name\" : \"foo.bar\", \"symbols\" : [ \"A\", \"B\" ] } ##T_END,\n" + - " \"optional\" : true\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ }", - "{\"enumOptional\":null}" - }, - { - "{ \"enumOptional\" : \"A\" } }", - "{\"enumOptional\":{\"##NS(foo.)bar\":\"A\"}}" - }, - { - "{ \"enumOptional\" : \"B\" } }", - "{\"enumOptional\":{\"##NS(foo.)bar\":\"B\"}}" - }, - { - "{ \"enumOptional\" : {} }", - "Error processing /enumOptional" - }, - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"enumOptional\",\n" + + " \"type\" : ##T_START { \"type\" : \"enum\", \"name\" : \"foo.bar\", \"symbols\" : [ \"A\", \"B\" ] } ##T_END,\n" + + " \"optional\" : true\n" + " }\n" + " ]\n" + "}\n"}, {"{ }", "{\"enumOptional\":null}"}, + {"{ \"enumOptional\" : \"A\" } }", "{\"enumOptional\":{\"##NS(foo.)bar\":\"A\"}}"}, + {"{ \"enumOptional\" : \"B\" } }", "{\"enumOptional\":{\"##NS(foo.)bar\":\"B\"}}"}, + {"{ \"enumOptional\" : {} }", "Error processing /enumOptional"},}, { // record with optional union field of records - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"unionOptional\",\n" + - " \"type\" : ##T_START [\n" + - " { \"type\" : \"record\", \"name\" : \"R1\", \"fields\" : [ { \"name\" : \"r1\", \"type\" : \"string\" } ] },\n" + - " { \"type\" : \"record\", \"name\" : \"R2\", \"fields\" : [ { \"name\" : \"r2\", \"type\" : \"int\" } ] },\n" + - " \"int\",\n" + - " \"string\"\n" + - " ] ##T_END,\n" + - " \"optional\" : true\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ }", - "{\"unionOptional\":null}" - }, - { - "{ \"unionOptional\" : { \"R1\" : { \"r1\" : \"value\" } } }", - "{\"unionOptional\":{\"R1\":{\"r1\":\"value\"}}}" - }, - { - "{ \"unionOptional\" : { \"R2\" : { \"r2\" : 52 } } }", - "{\"unionOptional\":{\"R2\":{\"r2\":52}}}" - }, - { - "{ \"unionOptional\" : { \"int\" : 52 } }", - "{\"unionOptional\":{\"int\":52}}" - }, - { - "{ \"unionOptional\" : { \"string\" : \"value\" } }", - "{\"unionOptional\":{\"string\":\"value\"}}" - }, - { - "{ \"unionOptional\" : {} }", - "Error processing /unionOptional" - }, - }, - { + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"unionOptional\",\n" + " \"type\" : ##T_START [\n" + + " { \"type\" : \"record\", \"name\" : \"R1\", \"fields\" : [ { \"name\" : \"r1\", \"type\" : \"string\" } ] },\n" + + " { \"type\" : \"record\", \"name\" : \"R2\", \"fields\" : [ { \"name\" : \"r2\", \"type\" : \"int\" } ] },\n" + + " \"int\",\n" + " \"string\"\n" + " ] ##T_END,\n" + " \"optional\" : true\n" + + " }\n" + " ]\n" + "}\n"}, {"{ }", "{\"unionOptional\":null}"}, + {"{ \"unionOptional\" : { \"R1\" : { \"r1\" : \"value\" } } }", + "{\"unionOptional\":{\"R1\":{\"r1\":\"value\"}}}"}, + {"{ \"unionOptional\" : { \"R2\" : { \"r2\" : 52 } } }", "{\"unionOptional\":{\"R2\":{\"r2\":52}}}"}, + {"{ \"unionOptional\" : { \"int\" : 52 } }", "{\"unionOptional\":{\"int\":52}}"}, + {"{ \"unionOptional\" : { \"string\" : \"value\" } }", "{\"unionOptional\":{\"string\":\"value\"}}"}, + {"{ \"unionOptional\" : {} }", "Error processing /unionOptional"},}, { // record with optional union field with alias, union types are RECORD - { - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " {\n" + - " \"name\" : \"unionOptionalAlias\",\n" + - " \"type\" : ##T_START [\n" + - " { " + - " \"type\" : { \"type\" : \"record\", \"name\" : \"R1\", \"fields\" : [ { \"name\" : \"r1\", \"type\" : \"string\" } ] }, " + - " \"alias\": \"success\"" + - " },\n" + - " { " + - " \"type\": { \"type\" : \"record\", \"name\" : \"R2\", \"fields\" : [ { \"name\" : \"r2\", \"type\" : \"int\" } ] }, " + - " \"alias\": \"failure\"" + - " }\n" + - " ] ##T_END,\n" + - " \"optional\" : true\n" + - " }\n" + - " ]\n" + - "}\n" - }, - { - "{ \"unionOptionalAlias\" : { \"success\" : { \"r1\" : \"value\" } } }", - "{\"unionOptionalAlias\":{\"FooUnionOptionalAlias\":{\"success\":{\"R1\":{\"r1\":\"value\"}},\"failure\":null,\"fieldDiscriminator\":\"success\"}}}" - }, - { - "{}", - "{\"unionOptionalAlias\":null}" - }, - { - "{ \"unionOptionalAlias\" : {} }", - "Error processing /unionOptionalAlias" - }, - { - "{ \"unionOptionalAlias\" : { \"success\" : { \"r1\" : 123 } } }", - "Error processing /unionOptionalAlias/success" - } - } - }; + {"{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + " {\n" + + " \"name\" : \"unionOptionalAlias\",\n" + " \"type\" : ##T_START [\n" + " { " + + " \"type\" : { \"type\" : \"record\", \"name\" : \"R1\", \"fields\" : [ { \"name\" : \"r1\", \"type\" : \"string\" } ] }, " + + " \"alias\": \"success\"" + " },\n" + " { " + + " \"type\": { \"type\" : \"record\", \"name\" : \"R2\", \"fields\" : [ { \"name\" : \"r2\", \"type\" : \"int\" } ] }, " + + " \"alias\": \"failure\"" + " }\n" + " ] ##T_END,\n" + " \"optional\" : true\n" + + " }\n" + " ]\n" + "}\n"}, {"{ \"unionOptionalAlias\" : { \"success\" : { \"r1\" : \"value\" } } }", + "{\"unionOptionalAlias\":{\"FooUnionOptionalAlias\":{\"success\":{\"R1\":{\"r1\":\"value\"}},\"failure\":null,\"fieldDiscriminator\":\"success\"}}}"}, + {"{}", "{\"unionOptionalAlias\":null}"}, + {"{ \"unionOptionalAlias\" : {} }", "Error processing /unionOptionalAlias"}, + {"{ \"unionOptionalAlias\" : { \"success\" : { \"r1\" : 123 } } }", + "Error processing /unionOptionalAlias/success"}}}; // test translation of Pegasus DataMap to Avro GenericRecord. - for (String[][] row : inputs) - { + for (String[][] row : inputs) { String schemaText = row[0][0]; - if (schemaText.contains("##T_START")) - { + if (schemaText.contains("##T_START")) { assertTrue(schemaText.contains("##T_END")); String noTyperefSchemaText = schemaText.replace("##T_START", "").replace("##T_END", ""); assertFalse(noTyperefSchemaText.contains("##T_")); assertFalse(noTyperefSchemaText.contains("typeref")); - String typerefSchemaText = schemaText - .replace("##T_START", "{ \"type\" : \"typeref\", \"name\" : \"Ref\", \"ref\" : ") - .replace("##T_END", "}"); + String typerefSchemaText = + schemaText.replace("##T_START", "{ \"type\" : \"typeref\", \"name\" : \"Ref\", \"ref\" : ") + .replace("##T_END", "}"); assertFalse(typerefSchemaText.contains("##T_")); assertTrue(typerefSchemaText.contains("typeref")); testDataTranslation(noTyperefSchemaText, row); testDataTranslation(typerefSchemaText, row); - } - else - { + } else { assertFalse(schemaText.contains("##")); testDataTranslation(schemaText, row); } @@ -1077,31 +366,16 @@ public void testDataTranslator() throws IOException @Test public void testInfinityAndNan() throws IOException { - String schemaText = - "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - " { \"name\" : \"doubleRequired\", \"type\" : \"double\" },\n" + - " { \"name\" : \"floatRequired\", \"type\" : \"float\" }\n" + - " ]\n" + - "}\n"; + String schemaText = "{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + + " { \"name\" : \"doubleRequired\", \"type\" : \"double\" },\n" + + " { \"name\" : \"floatRequired\", \"type\" : \"float\" }\n" + " ]\n" + "}\n"; // First element is the input, second element is the expected output - Object[][] inputs = { - { - "{ \"doubleRequired\" : \"Infinity\", \"floatRequired\" : \"Infinity\"}", - Double.POSITIVE_INFINITY, - Float.POSITIVE_INFINITY - }, { - "{ \"doubleRequired\" : \"NaN\", \"floatRequired\" : \"NaN\"}", - Double.NaN, - Float.NaN - }, { - "{ \"doubleRequired\" : \"-Infinity\", \"floatRequired\" : \"-Infinity\"}", - Double.NEGATIVE_INFINITY, - Float.NEGATIVE_INFINITY - } - }; + Object[][] inputs = + {{"{ \"doubleRequired\" : \"Infinity\", \"floatRequired\" : \"Infinity\"}", Double.POSITIVE_INFINITY, + Float.POSITIVE_INFINITY}, + {"{ \"doubleRequired\" : \"NaN\", \"floatRequired\" : \"NaN\"}", Double.NaN, Float.NaN}, + {"{ \"doubleRequired\" : \"-Infinity\", \"floatRequired\" : \"-Infinity\"}", Double.NEGATIVE_INFINITY, + Float.NEGATIVE_INFINITY}}; RecordDataSchema recordDataSchema = (RecordDataSchema) TestUtil.dataSchemaFromString(schemaText); for (Object[] input : inputs) { DataMap dataMap = TestUtil.dataMapFromString((String) input[0]); @@ -1110,12 +384,9 @@ public void testInfinityAndNan() throws IOException { assertEquals(avroRecord.get("doubleRequired"), input[1]); assertEquals(avroRecord.get("floatRequired"), input[2]); } - - } - private void testDataTranslation(String schemaText, String[][] row) throws IOException - { + private void testDataTranslation(String schemaText, String[][] row) throws IOException { boolean debug = false; if (debug) out.print(schemaText); @@ -1125,8 +396,7 @@ private void testDataTranslation(String schemaText, String[][] row) throws IOExc if (debug) out.println(avroSchema); // translate data - for (int col = 1; col < row.length; col++) - { + for (int col = 1; col < row.length; col++) { String result; GenericRecord avroRecord = null; Exception exc = null; @@ -1135,15 +405,12 @@ private void testDataTranslation(String schemaText, String[][] row) throws IOExc DataMap dataMap = TestUtil.dataMapFromString(row[col][0]); // translate from Pegasus to Avro - try - { + try { avroRecord = DataTranslator.dataMapToGenericRecord(dataMap, recordDataSchema, avroSchema); String avroJson = AvroUtil.jsonFromGenericRecord(avroRecord, false); if (debug) out.println(col + " GenericRecord: " + avroJson); result = avroJson; - } - catch (Exception e) - { + } catch (Exception e) { exc = e; result = TestUtil.stringFromException(e); if (debug) out.println(col + " Exception: " + result); @@ -1151,36 +418,30 @@ private void testDataTranslation(String schemaText, String[][] row) throws IOExc int start = 1; boolean oneWay = false; - if (start < row[col].length && row[col][start] == ONE_WAY) - { + if (start < row[col].length && row[col][start] == ONE_WAY) { oneWay = true; start++; } // verify - for (int i = start; i < row[col].length; i++) - { + for (int i = start; i < row[col].length; i++) { if (debug) out.println(col + " Test:" + row[col][i]); if (debug && exc != null && result.contains(row[col][i]) == false) exc.printStackTrace(out); - String expectedBeforeNamespaceProcessor = row[col][i]; + String expectedBeforeNamespaceProcessor = row[col][i]; String expected = TestAvroUtil.namespaceProcessor(expectedBeforeNamespaceProcessor); if (debug && expected != expectedBeforeNamespaceProcessor) out.println(" Expected:" + expected); assertTrue(result.contains(expected)); } - if (avroRecord != null) - { + if (avroRecord != null) { // translate from Avro back to Pegasus DataMap dataMapResult = DataTranslator.genericRecordToDataMap(avroRecord, recordDataSchema, avroSchema); - ValidationResult vr = ValidateDataAgainstSchema.validate(dataMapResult, - recordDataSchema, - new ValidationOptions(RequiredMode.MUST_BE_PRESENT, - CoercionMode.NORMAL)); + ValidationResult vr = ValidateDataAgainstSchema.validate(dataMapResult, recordDataSchema, + new ValidationOptions(RequiredMode.MUST_BE_PRESENT, CoercionMode.NORMAL)); DataMap fixedInputDataMap = (DataMap) vr.getFixed(); assertTrue(vr.isValid()); - if (oneWay == false) - { + if (oneWay == false) { assertEquals(dataMapResult, fixedInputDataMap); } @@ -1191,15 +452,13 @@ private void testDataTranslation(String schemaText, String[][] row) throws IOExc assertEquals(avroBytes, avroBytesAgain); // check result of roundtrip binary serialization - DataMap dataMapFromBinaryResult = DataTranslator.genericRecordToDataMap(avroRecordFromBytes, recordDataSchema, avroSchema); - vr = ValidateDataAgainstSchema.validate(dataMapFromBinaryResult, - recordDataSchema, - new ValidationOptions(RequiredMode.MUST_BE_PRESENT, - CoercionMode.NORMAL)); + DataMap dataMapFromBinaryResult = + DataTranslator.genericRecordToDataMap(avroRecordFromBytes, recordDataSchema, avroSchema); + vr = ValidateDataAgainstSchema.validate(dataMapFromBinaryResult, recordDataSchema, + new ValidationOptions(RequiredMode.MUST_BE_PRESENT, CoercionMode.NORMAL)); fixedInputDataMap = (DataMap) vr.getFixed(); assertTrue(vr.isValid()); - if (oneWay == false) - { + if (oneWay == false) { assertEquals(dataMapResult, fixedInputDataMap); } } @@ -1219,299 +478,140 @@ public Object[][] defaultToAvroOptionalTranslationProvider() { // 6. sixth element is the expected AvroJsonString after translation // 7. seventh element is whether this is a valid case // 8. eighth element tells the error message if this is invalid - return new Object[][] { + return new Object[][]{ // 1. If the dataMap has customer set values, the mode should have no impact on the DataTranslator. Tests for Int { // required int with default value "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START \"int\" ##T_END, \"default\" : 42 } ] }", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":\"int\",\"default\":42}]}", - "{\"bar\":1}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"bar\":1}", - false, - "", - }, - { - // required int with default value - "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START \"int\" ##T_END, \"default\" : 42 } ] }", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":\"int\",\"default\":42}]}", - "{\"bar\":1}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"bar\":1}", - false, - "", - }, + "{\"bar\":1}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"bar\":1}", false, "",}, { + // required int with default value + "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START \"int\" ##T_END, \"default\" : 42 } ] }", + PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, + "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":\"int\",\"default\":42}]}", + "{\"bar\":1}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"bar\":1}", false, "",}, { // required int with default value "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START \"int\" ##T_END, \"default\" : 42 } ] }", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":[\"null\",\"int\"],\"default\":null}]}", - "{\"bar\":1}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"bar\":{\"int\":1}}", // Json representation for Avro record - false, - "", - }, + "{\"bar\":1}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"bar\":{\"int\":1}}", + // Json representation for Avro record + false, "",}, { // required int with default value "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START \"int\" ##T_END, \"default\" : 42 } ] }", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":[\"null\",\"int\"],\"default\":null}]}", - "{\"bar\":1}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"bar\":{\"int\":1}}", // Json representation for Avro record - false, - "", - }, + "{\"bar\":1}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"bar\":{\"int\":1}}", + // Json representation for Avro record + false, "",}, // 2. If the dataMap has customer set values, the mode should have no impact on the DataTranslator. Tests for Array - { - "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"arrayRequired\", \"type\" : ##T_START { \"type\" : \"array\", \"items\" : \"string\" } ##T_END, \"default\": [ ] } ] }", + {"{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"arrayRequired\", \"type\" : ##T_START { \"type\" : \"array\", \"items\" : \"string\" } ##T_END, \"default\": [ ] } ] }", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"arrayRequired\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":[]}]}", - "{\"arrayRequired\":[\"a\",\"b\"]}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"arrayRequired\":[\"a\",\"b\"]}", - false, - "", - }, - { - "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"arrayRequired\", \"type\" : ##T_START { \"type\" : \"array\", \"items\" : \"string\" } ##T_END, \"default\": [ ] } ] }", + "{\"arrayRequired\":[\"a\",\"b\"]}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, + "{\"arrayRequired\":[\"a\",\"b\"]}", false, "",}, + {"{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"arrayRequired\", \"type\" : ##T_START { \"type\" : \"array\", \"items\" : \"string\" } ##T_END, \"default\": [ ] } ] }", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"arrayRequired\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":[]}]}", - "{\"arrayRequired\":[\"a\",\"b\"]}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"arrayRequired\":[\"a\",\"b\"]}", - false, - "", - }, - { - "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"arrayRequired\", \"type\" : ##T_START { \"type\" : \"array\", \"items\" : \"string\" } ##T_END, \"default\": [ ] } ] }", + "{\"arrayRequired\":[\"a\",\"b\"]}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, + "{\"arrayRequired\":[\"a\",\"b\"]}", false, "",}, + {"{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"arrayRequired\", \"type\" : ##T_START { \"type\" : \"array\", \"items\" : \"string\" } ##T_END, \"default\": [ ] } ] }", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"arrayRequired\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"string\"}],\"default\":null}]}", - "{\"arrayRequired\":[\"a\",\"b\"]}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, + "{\"arrayRequired\":[\"a\",\"b\"]}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"arrayRequired\":{\"array\":[\"a\",\"b\"]}}",//read as optional from Avro true, - false, - "", - }, - { - "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"arrayRequired\", \"type\" : ##T_START { \"type\" : \"array\", \"items\" : \"string\" } ##T_END, \"default\": [ ] } ] }", + false, "",}, + {"{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"arrayRequired\", \"type\" : ##T_START { \"type\" : \"array\", \"items\" : \"string\" } ##T_END, \"default\": [ ] } ] }", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"arrayRequired\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"string\"}],\"default\":null}]}", - "{\"arrayRequired\":[\"a\",\"b\"]}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, + "{\"arrayRequired\":[\"a\",\"b\"]}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"arrayRequired\":{\"array\":[\"a\",\"b\"]}}",//read as optional from Avro true, - false, - "", - }, - + false, "",}, // 3. If the dataMap has customer set values, the mode should have no impact on the DataTranslator. Tests for Union - { - "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START [\"int\", \"string\"] ##T_END, \"default\" : { \"int\" : 42 } } ] }", + {"{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START [\"int\", \"string\"] ##T_END, \"default\" : { \"int\" : 42 } } ] }", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":[\"int\",\"string\"],\"default\":42}]}", - "{\"bar\":{\"int\":42}}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"bar\":{\"int\":42}}", - false, - "", - }, - { - "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START [\"int\", \"string\"] ##T_END, \"default\" : { \"int\" : 42 } } ] }", + "{\"bar\":{\"int\":42}}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"bar\":{\"int\":42}}", + false, "",}, + {"{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START [\"int\", \"string\"] ##T_END, \"default\" : { \"int\" : 42 } } ] }", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":[\"int\",\"string\"],\"default\":42}]}", - "{\"bar\":{\"int\":42}}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"bar\":{\"int\":42}}", - false, - "", - }, - { - "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START [\"int\", \"string\"] ##T_END, \"default\" : { \"int\" : 42 } } ] }", + "{\"bar\":{\"int\":42}}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, + "{\"bar\":{\"int\":42}}", false, "",}, + {"{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START [\"int\", \"string\"] ##T_END, \"default\" : { \"int\" : 42 } } ] }", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":[\"null\",\"int\",\"string\"],\"default\":null}]}", - "{\"bar\":{\"int\":42}}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"bar\":{\"int\":42}}", - false, - "", - }, - { - "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START [\"int\", \"string\"] ##T_END, \"default\" : { \"int\" : 42 } } ] }", + "{\"bar\":{\"int\":42}}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"bar\":{\"int\":42}}", + false, "",}, + {"{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START [\"int\", \"string\"] ##T_END, \"default\" : { \"int\" : 42 } } ] }", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":[\"null\",\"int\",\"string\"],\"default\":null}]}", - "{\"bar\":{\"int\":42}}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"bar\":{\"int\":42}}", - false, - "", - }, + "{\"bar\":{\"int\":42}}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, + "{\"bar\":{\"int\":42}}", false, "",}, // 4. If the dataMap has customer set values, the mode should have no impact on the DataTranslator. Tests for Map - { - "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"mapRequired\", \"type\" : ##T_START { \"type\" : \"map\", \"values\" : \"string\" } ##T_END, \"default\": {} } ] }", + {"{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"mapRequired\", \"type\" : ##T_START { \"type\" : \"map\", \"values\" : \"string\" } ##T_END, \"default\": {} } ] }", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"mapRequired\",\"type\":{\"type\":\"map\",\"values\":\"string\"},\"default\":{}}]}", - "{\"mapRequired\":{\"somekey\":\"somevalue\"}}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"mapRequired\":{\"somekey\":\"somevalue\"}}", - false, - "", - }, - { - "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"mapRequired\", \"type\" : ##T_START { \"type\" : \"map\", \"values\" : \"string\" } ##T_END, \"default\": {} } ] }", + "{\"mapRequired\":{\"somekey\":\"somevalue\"}}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, + "{\"mapRequired\":{\"somekey\":\"somevalue\"}}", false, "",}, + {"{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"mapRequired\", \"type\" : ##T_START { \"type\" : \"map\", \"values\" : \"string\" } ##T_END, \"default\": {} } ] }", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"mapRequired\",\"type\":{\"type\":\"map\",\"values\":\"string\"},\"default\":{}}]}", - "{\"mapRequired\":{\"somekey\":\"somevalue\"}}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"mapRequired\":{\"somekey\":\"somevalue\"}}", - false, - "", - }, - { - "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"mapRequired\", \"type\" : ##T_START { \"type\" : \"map\", \"values\" : \"string\" } ##T_END, \"default\": {} } ] }", + "{\"mapRequired\":{\"somekey\":\"somevalue\"}}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, + "{\"mapRequired\":{\"somekey\":\"somevalue\"}}", false, "",}, + {"{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"mapRequired\", \"type\" : ##T_START { \"type\" : \"map\", \"values\" : \"string\" } ##T_END, \"default\": {} } ] }", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"mapRequired\",\"type\":[\"null\",{\"type\":\"map\",\"values\":\"string\"}],\"default\":null}]}", - "{\"mapRequired\":{\"somekey\":\"somevalue\"}}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"mapRequired\":{\"map\":{\"somekey\":\"somevalue\"}}}", - false, - "", - }, - { - "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"mapRequired\", \"type\" : ##T_START { \"type\" : \"map\", \"values\" : \"string\" } ##T_END, \"default\": {} } ] }", + "{\"mapRequired\":{\"somekey\":\"somevalue\"}}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, + "{\"mapRequired\":{\"map\":{\"somekey\":\"somevalue\"}}}", false, "",}, + {"{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"mapRequired\", \"type\" : ##T_START { \"type\" : \"map\", \"values\" : \"string\" } ##T_END, \"default\": {} } ] }", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"mapRequired\",\"type\":[\"null\",{\"type\":\"map\",\"values\":\"string\"}],\"default\":null}]}", - "{\"mapRequired\":{\"somekey\":\"somevalue\"}}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"mapRequired\":{\"map\":{\"somekey\":\"somevalue\"}}}", - false, - "", - }, + "{\"mapRequired\":{\"somekey\":\"somevalue\"}}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, + "{\"mapRequired\":{\"map\":{\"somekey\":\"somevalue\"}}}", false, "",}, // 5. If the dataMap has customer set values, the mode should have no impact on the DataTranslator. Tests for nested record - { - " { " + - " \"type\" : \"record\", " + - " \"name\" : \"Foo\", " + - " \"fields\" : [ { " + - " \"name\" : \"nestedField\", " + - " \"type\" : { " + - " \"type\" : \"record\", " + - " \"name\" : \"nestedRecord\", " + - " \"fields\" : [ { " + - " \"name\" : \"field1\", " + - " \"type\" : \"int\" " + - " } ] " + - " }, " + - " \"default\": { \"field1\" : 1} " + - " } ] " + - " } ", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, + {" { " + " \"type\" : \"record\", " + " \"name\" : \"Foo\", " + " \"fields\" : [ { " + + " \"name\" : \"nestedField\", " + " \"type\" : { " + " \"type\" : \"record\", " + + " \"name\" : \"nestedRecord\", " + " \"fields\" : [ { " + " \"name\" : \"field1\", " + + " \"type\" : \"int\" " + " } ] " + " }, " + " \"default\": { \"field1\" : 1} " + + " } ] " + " } ", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"nestedField\",\"type\":{\"type\":\"record\",\"name\":\"nestedRecord\",\"fields\":[{\"name\":\"field1\",\"type\":\"int\"}]},\"default\":{\"field1\":1}}]}", - " { " + - " \"nestedField\": { " + - " \"field1\":42 " + - " } " + - " } ", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"nestedField\":{\"field1\":42}}", - false, - "", - }, - { - " { " + - " \"type\" : \"record\", " + - " \"name\" : \"Foo\", " + - " \"fields\" : [ { " + - " \"name\" : \"nestedField\", " + - " \"type\" : { " + - " \"type\" : \"record\", " + - " \"name\" : \"nestedRecord\", " + - " \"fields\" : [ { " + - " \"name\" : \"field1\", " + - " \"type\" : \"int\" " + - " } ] " + - " }, " + - " \"default\": { \"field1\" : 1} " + - " } ] " + - " } ", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, + " { " + " \"nestedField\": { " + " \"field1\":42 " + " } " + " } ", + PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"nestedField\":{\"field1\":42}}", false, "",}, + {" { " + " \"type\" : \"record\", " + " \"name\" : \"Foo\", " + " \"fields\" : [ { " + + " \"name\" : \"nestedField\", " + " \"type\" : { " + " \"type\" : \"record\", " + + " \"name\" : \"nestedRecord\", " + " \"fields\" : [ { " + " \"name\" : \"field1\", " + + " \"type\" : \"int\" " + " } ] " + " }, " + " \"default\": { \"field1\" : 1} " + + " } ] " + " } ", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"nestedField\",\"type\":{\"type\":\"record\",\"name\":\"nestedRecord\",\"fields\":[{\"name\":\"field1\",\"type\":\"int\"}]},\"default\":{\"field1\":1}}]}", - " { " + - " \"nestedField\": { " + - " \"field1\":42 " + - " } " + - " } ", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"nestedField\":{\"field1\":42}}", - false, - "", - }, - { - " { " + - " \"type\" : \"record\", " + - " \"name\" : \"Foo\", " + - " \"fields\" : [ { " + - " \"name\" : \"nestedField\", " + - " \"type\" : { " + - " \"type\" : \"record\", " + - " \"name\" : \"nestedRecord\", " + - " \"fields\" : [ { " + - " \"name\" : \"field1\", " + - " \"type\" : \"int\" " + - " } ] " + - " }, " + - " \"default\": { \"field1\" : 1} " + - " } ] " + - " } ", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"nestedField\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"nestedRecord\",\"fields\":[{\"name\":\"field1\",\"type\":\"int\"}]}],\"default\":null}]}", - " { " + - " \"nestedField\": { " + - " \"field1\":42 " + - " } " + - " } ", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"nestedField\":{\"nestedRecord\":{\"field1\":42}}}", - false, - "", - }, - { - " { " + - " \"type\" : \"record\", " + - " \"name\" : \"Foo\", " + - " \"fields\" : [ { " + - " \"name\" : \"nestedField\", " + - " \"type\" : { " + - " \"type\" : \"record\", " + - " \"name\" : \"nestedRecord\", " + - " \"fields\" : [ { " + - " \"name\" : \"field1\", " + - " \"type\" : \"int\" " + - " } ] " + - " }, " + - " \"default\": { \"field1\" : 1} " + - " } ] " + - " } ", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, + " { " + " \"nestedField\": { " + " \"field1\":42 " + " } " + " } ", + PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"nestedField\":{\"field1\":42}}", false, "",}, + {" { " + " \"type\" : \"record\", " + " \"name\" : \"Foo\", " + " \"fields\" : [ { " + + " \"name\" : \"nestedField\", " + " \"type\" : { " + " \"type\" : \"record\", " + + " \"name\" : \"nestedRecord\", " + " \"fields\" : [ { " + " \"name\" : \"field1\", " + + " \"type\" : \"int\" " + " } ] " + " }, " + " \"default\": { \"field1\" : 1} " + + " } ] " + " } ", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"nestedField\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"nestedRecord\",\"fields\":[{\"name\":\"field1\",\"type\":\"int\"}]}],\"default\":null}]}", - " { " + - " \"nestedField\": { " + - " \"field1\":42 " + - " } " + - " } ", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"nestedField\":{\"nestedRecord\":{\"field1\":42}}}", - false, - "", - }, + " { " + " \"nestedField\": { " + " \"field1\":42 " + " } " + " } ", + PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"nestedField\":{\"nestedRecord\":{\"field1\":42}}}", + false, "",}, {" { " + " \"type\" : \"record\", " + " \"name\" : \"Foo\", " + " \"fields\" : [ { " + + " \"name\" : \"nestedField\", " + " \"type\" : { " + " \"type\" : \"record\", " + + " \"name\" : \"nestedRecord\", " + " \"fields\" : [ { " + " \"name\" : \"field1\", " + + " \"type\" : \"int\" " + " } ] " + " }, " + " \"default\": { \"field1\" : 1} " + + " } ] " + " } ", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, + "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"nestedField\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"nestedRecord\",\"fields\":[{\"name\":\"field1\",\"type\":\"int\"}]}],\"default\":null}]}", + " { " + " \"nestedField\": { " + " \"field1\":42 " + " } " + " } ", + PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, + "{\"nestedField\":{\"nestedRecord\":{\"field1\":42}}}", false, "",}, // 6. Test when input is missing, using int { @@ -1519,35 +619,19 @@ public Object[][] defaultToAvroOptionalTranslationProvider() { "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START \"int\" ##T_END, \"default\" : 42 } ] }", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":\"int\",\"default\":42}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"bar\":42}", - false, - "", - }, - { - // required int with default value - "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START \"int\" ##T_END, \"default\" : 42 } ] }", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, // this option translate to optional in Avro - "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":[\"null\",\"int\"],\"default\":null}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"bar\":{\"int\":42}}", - false, - "", - }, + "{}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"bar\":42}", false, "",}, { + // required int with default value + "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START \"int\" ##T_END, \"default\" : 42 } ] }", + PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, // this option translate to optional in Avro + "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":[\"null\",\"int\"],\"default\":null}]}", + "{}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"bar\":{\"int\":42}}", false, "",}, { // required int with default value "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START \"int\" ##T_END, \"default\" : 42 } ] }", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, // this option translate to optional in Avro "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":[\"null\",\"int\"],\"default\":null}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"bar\":null}", - false, - "", - }, + "{}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"bar\":null}", false, "",}, // 7. Test when input is missing, using union { @@ -1555,175 +639,76 @@ public Object[][] defaultToAvroOptionalTranslationProvider() { "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START [\"int\", \"string\"] ##T_END, \"default\" : { \"int\" : 42 } } ] }", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":[\"int\",\"string\"],\"default\":42}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"bar\":{\"int\":42}}", // DataTranslator should translate to the default value in the schema - false, - "", - }, - { - // required Union of [int string] with default value - "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START [\"int\", \"string\"] ##T_END, \"default\" : { \"int\" : 42 } } ] }", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":[\"null\",\"int\",\"string\"],\"default\":null}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"bar\":{\"int\":42}}", // DataTranslator should translate to the default value in the schema - false, - "", - }, + "{}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"bar\":{\"int\":42}}", + // DataTranslator should translate to the default value in the schema + false, "",}, { + // required Union of [int string] with default value + "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START [\"int\", \"string\"] ##T_END, \"default\" : { \"int\" : 42 } } ] }", + PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, + "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":[\"null\",\"int\",\"string\"],\"default\":null}]}", + "{}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"bar\":{\"int\":42}}", + // DataTranslator should translate to the default value in the schema + false, "",}, { // required Union of [int string] with default value "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START [\"int\", \"string\"] ##T_END, \"default\" : { \"int\" : 42 } } ] }", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":[\"null\",\"int\",\"string\"],\"default\":null}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"bar\":null}", //read as optional from Avro - false, - "", - }, + "{}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"bar\":null}", + //read as optional from Avro + false, "",}, // 8. Test when input is missing, using array - { - "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"arrayRequired\", \"type\" : ##T_START { \"type\" : \"array\", \"items\" : \"string\" } ##T_END, \"default\": [ ] } ] }", + {"{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"arrayRequired\", \"type\" : ##T_START { \"type\" : \"array\", \"items\" : \"string\" } ##T_END, \"default\": [ ] } ] }", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"arrayRequired\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":[]}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"arrayRequired\":[]}", - false, - "", - }, - { - "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"arrayRequired\", \"type\" : ##T_START { \"type\" : \"array\", \"items\" : \"string\" } ##T_END, \"default\": [ ] } ] }", + "{}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"arrayRequired\":[]}", false, "",}, + {"{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"arrayRequired\", \"type\" : ##T_START { \"type\" : \"array\", \"items\" : \"string\" } ##T_END, \"default\": [ ] } ] }", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"arrayRequired\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"string\"}],\"default\":null}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"arrayRequired\":{\"array\":[]}}", - false, - "", - }, - { - "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"arrayRequired\", \"type\" : ##T_START { \"type\" : \"array\", \"items\" : \"string\" } ##T_END, \"default\": [ ] } ] }", + "{}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"arrayRequired\":{\"array\":[]}}", false, "",}, + {"{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"arrayRequired\", \"type\" : ##T_START { \"type\" : \"array\", \"items\" : \"string\" } ##T_END, \"default\": [ ] } ] }", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"arrayRequired\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"string\"}],\"default\":null}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"arrayRequired\":null}", - false, - "", - }, + "{}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"arrayRequired\":null}", false, "",}, // 9. Test when input is missing, using map - { - "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"mapRequired\", \"type\" : ##T_START { \"type\" : \"map\", \"values\" : \"string\" } ##T_END, \"default\": {} } ] }", + {"{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"mapRequired\", \"type\" : ##T_START { \"type\" : \"map\", \"values\" : \"string\" } ##T_END, \"default\": {} } ] }", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"mapRequired\",\"type\":{\"type\":\"map\",\"values\":\"string\"},\"default\":{}}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"mapRequired\":{}}", - false, - "", - }, - { - "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"mapRequired\", \"type\" : ##T_START { \"type\" : \"map\", \"values\" : \"string\" } ##T_END, \"default\": {} } ] }", + "{}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"mapRequired\":{}}", false, "",}, + {"{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"mapRequired\", \"type\" : ##T_START { \"type\" : \"map\", \"values\" : \"string\" } ##T_END, \"default\": {} } ] }", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"mapRequired\",\"type\":[\"null\",{\"type\":\"map\",\"values\":\"string\"}],\"default\":null}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"mapRequired\":{\"map\":{}}}", - false, - "", - }, - { - "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"mapRequired\", \"type\" : ##T_START { \"type\" : \"map\", \"values\" : \"string\" } ##T_END, \"default\": {} } ] }", + "{}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"mapRequired\":{\"map\":{}}}", false, "",}, + {"{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"mapRequired\", \"type\" : ##T_START { \"type\" : \"map\", \"values\" : \"string\" } ##T_END, \"default\": {} } ] }", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"mapRequired\",\"type\":[\"null\",{\"type\":\"map\",\"values\":\"string\"}],\"default\":null}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"mapRequired\":null}", - false, - "", - }, + "{}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"mapRequired\":null}", false, "",}, - { - " { " + - " \"type\" : \"record\", " + - " \"name\" : \"Foo\", " + - " \"fields\" : [ { " + - " \"name\" : \"nestedField\", " + - " \"type\" : { " + - " \"type\" : \"record\", " + - " \"name\" : \"nestedRecord\", " + - " \"fields\" : [ { " + - " \"name\" : \"field1\", " + - " \"type\" : \"int\" " + - " } ] " + - " }, " + - " \"default\": { \"field1\" : 1} " + - " } ] " + - " } ", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, + {" { " + " \"type\" : \"record\", " + " \"name\" : \"Foo\", " + " \"fields\" : [ { " + + " \"name\" : \"nestedField\", " + " \"type\" : { " + " \"type\" : \"record\", " + + " \"name\" : \"nestedRecord\", " + " \"fields\" : [ { " + " \"name\" : \"field1\", " + + " \"type\" : \"int\" " + " } ] " + " }, " + " \"default\": { \"field1\" : 1} " + + " } ] " + " } ", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"nestedField\",\"type\":{\"type\":\"record\",\"name\":\"nestedRecord\",\"fields\":[{\"name\":\"field1\",\"type\":\"int\"}]},\"default\":{\"field1\":1}}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"nestedField\":{\"field1\":1}}", - false, - "", - }, - { - " { " + - " \"type\" : \"record\", " + - " \"name\" : \"Foo\", " + - " \"fields\" : [ { " + - " \"name\" : \"nestedField\", " + - " \"type\" : { " + - " \"type\" : \"record\", " + - " \"name\" : \"nestedRecord\", " + - " \"fields\" : [ { " + - " \"name\" : \"field1\", " + - " \"type\" : \"int\" " + - " } ] " + - " }, " + - " \"default\": { \"field1\" : 1} " + - " } ] " + - " } ", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, + "{}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"nestedField\":{\"field1\":1}}", false, "",}, + {" { " + " \"type\" : \"record\", " + " \"name\" : \"Foo\", " + " \"fields\" : [ { " + + " \"name\" : \"nestedField\", " + " \"type\" : { " + " \"type\" : \"record\", " + + " \"name\" : \"nestedRecord\", " + " \"fields\" : [ { " + " \"name\" : \"field1\", " + + " \"type\" : \"int\" " + " } ] " + " }, " + " \"default\": { \"field1\" : 1} " + + " } ] " + " } ", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"nestedField\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"nestedRecord\",\"fields\":[{\"name\":\"field1\",\"type\":\"int\"}]}],\"default\":null}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"nestedField\":{\"nestedRecord\":{\"field1\":1}}}", - false, - "", - }, - { - " { " + - " \"type\" : \"record\", " + - " \"name\" : \"Foo\", " + - " \"fields\" : [ { " + - " \"name\" : \"nestedField\", " + - " \"type\" : { " + - " \"type\" : \"record\", " + - " \"name\" : \"nestedRecord\", " + - " \"fields\" : [ { " + - " \"name\" : \"field1\", " + - " \"type\" : \"int\" " + - " } ] " + - " }, " + - " \"default\": { \"field1\" : 1} " + - " } ] " + - " } ", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, + "{}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, + "{\"nestedField\":{\"nestedRecord\":{\"field1\":1}}}", false, "",}, + {" { " + " \"type\" : \"record\", " + " \"name\" : \"Foo\", " + " \"fields\" : [ { " + + " \"name\" : \"nestedField\", " + " \"type\" : { " + " \"type\" : \"record\", " + + " \"name\" : \"nestedRecord\", " + " \"fields\" : [ { " + " \"name\" : \"field1\", " + + " \"type\" : \"int\" " + " } ] " + " }, " + " \"default\": { \"field1\" : 1} " + + " } ] " + " } ", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"nestedField\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"nestedRecord\",\"fields\":[{\"name\":\"field1\",\"type\":\"int\"}]}],\"default\":null}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"nestedField\":null}", - false, - "", - }, + "{}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"nestedField\":null}", false, "",}, // 11. Test when input is missing, using enum { @@ -1731,12 +716,9 @@ public Object[][] defaultToAvroOptionalTranslationProvider() { "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"enumRequired\", \"type\" : ##T_START { \"name\" : \"Fruits\", \"type\" : \"enum\", \"symbols\" : [ \"APPLE\", \"ORANGE\" ] } ##T_END, \"default\": \"APPLE\" } ] }", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"enumRequired\",\"type\":{\"type\":\"enum\",\"name\":\"Fruits\",\"symbols\":[\"APPLE\",\"ORANGE\"]},\"default\":\"APPLE\"}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"enumRequired\":\"APPLE\"}", //read as optional from Avro - false, - "", - }, + "{}", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"enumRequired\":\"APPLE\"}", + //read as optional from Avro + false, "",}, // The following case works under avro 1.4 but not avro 1.6, and is an invalid case // { @@ -1756,38 +738,22 @@ public Object[][] defaultToAvroOptionalTranslationProvider() { "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"enumRequired\", \"type\" : ##T_START { \"name\" : \"Fruits\", \"type\" : \"enum\", \"symbols\" : [ \"APPLE\", \"ORANGE\" ] } ##T_END, \"default\": \"APPLE\" } ] }", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"enumRequired\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Fruits\",\"symbols\":[\"APPLE\",\"ORANGE\"]}],\"default\":null}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"enumRequired\":null}", //read as optional from Avro - false, - "", - }, + "{}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"enumRequired\":null}", + //read as optional from Avro + false, "",}, // If complex field in pegasus schema has default value and it doesn't specify value for a nested required field, // then the default value specified at field level should be used. { // required int with default value - "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ " - + "{ \"name\" : \"bar\", \"type\" : " + "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ " + "{ \"name\" : \"bar\", \"type\" : " + "{ \"type\": \"record\", \"name\": \"Bar\", \"fields\": [" - + " { \"name\": \"barInt\", \"type\": ##T_START \"int\" ##T_END, \"default\" : 42 }" - + " ] }," - + " \"default\": {}" - + " } ] }", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ " - + "{ \"name\" : \"bar\", \"type\" : " + + " { \"name\": \"barInt\", \"type\": ##T_START \"int\" ##T_END, \"default\" : 42 }" + " ] }," + + " \"default\": {}" + " } ] }", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, + "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ " + "{ \"name\" : \"bar\", \"type\" : " + "{ \"type\": \"record\", \"name\": \"Bar\", \"fields\": [" - + " { \"name\": \"barInt\", \"type\": \"int\", \"default\" : 42 }" - + " ] }," - + " \"default\": { \"barInt\":42 }" - + " } ] }", - "{\"bar\":{\"barInt\":42}}", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"bar\":{\"barInt\":42}}", - false, - "", - }, - + + " { \"name\": \"barInt\", \"type\": \"int\", \"default\" : 42 }" + " ] }," + + " \"default\": { \"barInt\":42 }" + " } ] }", "{\"bar\":{\"barInt\":42}}", + PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"bar\":{\"barInt\":42}}", false, "",}, // Below are test case example for invalid option combination: // If the field in pegasus schema has been translated as required, its data cannot be translated as optional @@ -1796,43 +762,29 @@ public Object[][] defaultToAvroOptionalTranslationProvider() { "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START \"int\" ##T_END, \"default\" : 42 } ] }", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":\"int\",\"default\":42}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"bar\":null}", - true, - "null of int in field bar of foo", - }, - { - // required Union of [int string] with default value - "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START [\"int\", \"string\"] ##T_END, \"default\" : { \"int\" : 42 } } ] }", - PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, - "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":[\"int\",\"string\"],\"default\":42}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"bar\":null}", //read as optional from Avro - true, // This case is wrong because expecting "null" for avro record but the avro schema doesn't contain null - "Error processing /bar\n" + "ERROR :: /bar :: cannot find null in union [\"int\",\"string\"]", - }, + "{}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"bar\":null}", true, + "null of int in field bar of foo",}, { + // required Union of [int string] with default value + "{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : ##T_START [\"int\", \"string\"] ##T_END, \"default\" : { \"int\" : 42 } } ] }", + PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, + "{\"type\":\"record\",\"name\":\"foo\",\"fields\":[{\"name\":\"bar\",\"type\":[\"int\",\"string\"],\"default\":42}]}", + "{}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"bar\":null}", //read as optional from Avro + true, // This case is wrong because expecting "null" for avro record but the avro schema doesn't contain null + "Error processing /bar\n" + "ERROR :: /bar :: cannot find null in union [\"int\",\"string\"]",}, { // required enum with default value "{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"enumRequired\", \"type\" : ##T_START { \"name\" : \"Fruits\", \"type\" : \"enum\", \"symbols\" : [ \"APPLE\", \"ORANGE\" ] } ##T_END, \"default\": \"APPLE\" } ] }", PegasusToAvroDefaultFieldTranslationMode.TRANSLATE, "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[{\"name\":\"enumRequired\",\"type\":{\"type\":\"enum\",\"name\":\"Fruits\",\"symbols\":[\"APPLE\",\"ORANGE\"]},\"default\":\"APPLE\"}]}", - "{}", - PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, - "{\"enumRequired\":null}", //read as optional from Avro + "{}", PegasusToAvroDefaultFieldTranslationMode.DO_NOT_TRANSLATE, "{\"enumRequired\":null}", + //read as optional from Avro true, - "Not an enum: null for schema: {\"type\":\"enum\",\"name\":\"Fruits\",\"symbols\":[\"APPLE\",\"ORANGE\"]}" - }, - }; + "Not an enum: null for schema: {\"type\":\"enum\",\"name\":\"Fruits\",\"symbols\":[\"APPLE\",\"ORANGE\"]}"},}; } - - @Test(dataProvider = "defaultToAvroOptionalTranslationProvider", - description = "generic record to data map should not care about the specific list implementation") - public void testPegasusDefaultToAvroOptionalTranslation(Object... testSchemaTextAndDataMap) throws IOException - { + @Test(dataProvider = "defaultToAvroOptionalTranslationProvider", description = "generic record to data map should not care about the specific list implementation") + public void testPegasusDefaultToAvroOptionalTranslation(Object... testSchemaTextAndDataMap) throws IOException { // Test if the pegasus default field has been correctly translated // i.e. if value present, translate it // if no value present, don't translate it @@ -1858,24 +810,22 @@ public void testPegasusDefaultToAvroOptionalTranslation(Object... testSchemaText try { // Test this also works for TypeRef - if (rawPegasusTestSchemaText.contains("##T_START")) - { + if (rawPegasusTestSchemaText.contains("##T_START")) { String noTyperefSchemaText = rawPegasusTestSchemaText.replace("##T_START", "").replace("##T_END", ""); String typerefSchemaText = rawPegasusTestSchemaText.replace("##T_START", "{ \"type\" : \"typeref\", \"name\" : \"Ref\", \"ref\" : ") - .replace("##T_END", "}"); + .replace("##T_END", "}"); schemaTextForTesting = Arrays.asList(noTyperefSchemaText, typerefSchemaText); - } else - { + } else { schemaTextForTesting = Arrays.asList(rawPegasusTestSchemaText); } - for (String pegasusSchemaText: schemaTextForTesting) { + for (String pegasusSchemaText : schemaTextForTesting) { //Create pegasus schema RecordDataSchema recordDataSchema = (RecordDataSchema) TestUtil.dataSchemaFromString(pegasusSchemaText); //Translate to Avro Schema so can create the GenericRecord data holder Schema avroSchema = SchemaTranslator.dataToAvroSchema(recordDataSchema, - new DataToAvroSchemaTranslationOptions(schemaTranslationMode)); + new DataToAvroSchemaTranslationOptions(schemaTranslationMode)); // AvroSchema translated needs to be as expected Schema expectedAvroSchema = Schema.parse(expectedAvroSchemaString); @@ -1887,9 +837,10 @@ public void testPegasusDefaultToAvroOptionalTranslation(Object... testSchemaText //Create option, pass into data translator DataMapToAvroRecordTranslationOptions options = new DataMapToAvroRecordTranslationOptionsBuilder().defaultFieldDataTranslationMode(dataTranslationMode) - .build(); + .build(); //Translate to generic record - GenericRecord avroRecord = DataTranslator.dataMapToGenericRecord(dataMap, recordDataSchema, avroSchema, options); + GenericRecord avroRecord = + DataTranslator.dataMapToGenericRecord(dataMap, recordDataSchema, avroSchema, options); String avroJson = AvroUtil.jsonFromGenericRecord(avroRecord, false); @@ -1898,10 +849,9 @@ public void testPegasusDefaultToAvroOptionalTranslation(Object... testSchemaText // validation result test DataMap dataMapResult = DataTranslator.genericRecordToDataMap(avroRecord, recordDataSchema, avroSchema); - ValidationResult vr = ValidateDataAgainstSchema.validate(dataMapResult, - recordDataSchema, - new ValidationOptions(RequiredMode.CAN_BE_ABSENT_IF_HAS_DEFAULT, // Not filling back Default value - CoercionMode.NORMAL)); + ValidationResult vr = ValidateDataAgainstSchema.validate(dataMapResult, recordDataSchema, + new ValidationOptions(RequiredMode.CAN_BE_ABSENT_IF_HAS_DEFAULT, // Not filling back Default value + CoercionMode.NORMAL)); DataMap fixedInputDataMap = (DataMap) vr.getFixed(); assertTrue(vr.isValid()); assertEquals(dataMapResult, fixedInputDataMap); @@ -1913,71 +863,48 @@ public void testPegasusDefaultToAvroOptionalTranslation(Object... testSchemaText assertEquals(avroBytes, avroBytesAgain); // check result of roundtrip binary serialization - DataMap dataMapFromBinaryResult = DataTranslator.genericRecordToDataMap(avroRecordFromBytes, recordDataSchema, avroSchema); - vr = ValidateDataAgainstSchema.validate(dataMapFromBinaryResult, - recordDataSchema, - new ValidationOptions(RequiredMode.CAN_BE_ABSENT_IF_HAS_DEFAULT, // Not filling back Default value - CoercionMode.NORMAL)); + DataMap dataMapFromBinaryResult = + DataTranslator.genericRecordToDataMap(avroRecordFromBytes, recordDataSchema, avroSchema); + vr = ValidateDataAgainstSchema.validate(dataMapFromBinaryResult, recordDataSchema, + new ValidationOptions(RequiredMode.CAN_BE_ABSENT_IF_HAS_DEFAULT, // Not filling back Default value + CoercionMode.NORMAL)); fixedInputDataMap = (DataMap) vr.getFixed(); assertTrue(vr.isValid()); assertEquals(dataMapResult, fixedInputDataMap); } - } - catch (Exception e) - { + } catch (Exception e) { assertTrue(isError); assertEquals(e.getMessage(), errorMsg); } } @Test - public void testAvroSchemaMissingFields() throws IOException - { - final String P_SCHEMA = - "{" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"fields\" : [\n" + - "{ \"name\": \"field1\", \"type\": \"int\" }," + - "{ \"name\": \"field2\", \"type\": \"int\", \"optional\": true }," + - "{ \"name\": \"field3\", \"type\": \"int\", \"optional\": true, \"default\": 42 }," + - "{ \"name\": \"field4\", \"type\": \"int\", \"default\": 42 }," + - "{ \"name\": \"field5\", \"type\": \"null\" }" + - "] }"; + public void testAvroSchemaMissingFields() throws IOException { + final String P_SCHEMA = "{" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [\n" + + "{ \"name\": \"field1\", \"type\": \"int\" }," + + "{ \"name\": \"field2\", \"type\": \"int\", \"optional\": true }," + + "{ \"name\": \"field3\", \"type\": \"int\", \"optional\": true, \"default\": 42 }," + + "{ \"name\": \"field4\", \"type\": \"int\", \"default\": 42 }," + + "{ \"name\": \"field5\", \"type\": \"null\" }" + "] }"; Schema avroSchema = Schema.parse("{ \"name\": \"foo\", \"type\": \"record\", \"fields\":[]}"); - DataMap map = DataTranslator.genericRecordToDataMap(new GenericData.Record(avroSchema), (RecordDataSchema)TestUtil.dataSchemaFromString(P_SCHEMA), avroSchema); + DataMap map = DataTranslator.genericRecordToDataMap(new GenericData.Record(avroSchema), + (RecordDataSchema) TestUtil.dataSchemaFromString(P_SCHEMA), avroSchema); assertEquals(map.size(), 0); } @Test - public void testMissingDefaultFieldsOnDataMap() throws IOException - { - final String SCHEMA = - "{" + - " \"type\":\"record\"," + - " \"name\":\"Foo\"," + - " \"fields\":[" + - " {" + - " \"name\":\"field1\"," + - " \"type\":\"string\"" + - " }," + - " {" + - " \"name\":\"field2\"," + - " \"type\":{" + - " \"type\":\"array\"," + - " \"items\":\"string\"" + - " }," + - " \"default\":[ ]" + - " }" + - " ]" + - "}"; - RecordDataSchema pegasusSchema = (RecordDataSchema)TestUtil.dataSchemaFromString(SCHEMA); + public void testMissingDefaultFieldsOnDataMap() throws IOException { + final String SCHEMA = "{" + " \"type\":\"record\"," + " \"name\":\"Foo\"," + " \"fields\":[" + " {" + + " \"name\":\"field1\"," + " \"type\":\"string\"" + " }," + " {" + + " \"name\":\"field2\"," + " \"type\":{" + " \"type\":\"array\"," + + " \"items\":\"string\"" + " }," + " \"default\":[ ]" + " }" + " ]" + "}"; + RecordDataSchema pegasusSchema = (RecordDataSchema) TestUtil.dataSchemaFromString(SCHEMA); Schema avroShema = Schema.parse(SCHEMA); DataMap dataMap = new DataMap(); dataMap.put("field1", "test"); GenericRecord record = DataTranslator.dataMapToGenericRecord(dataMap, pegasusSchema, avroShema); - assertEquals(record.get("field2"), new GenericData.Array<>(0, Schema.createArray( - Schema.create(Schema.Type.STRING)))); + assertEquals(record.get("field2"), + new GenericData.Array<>(0, Schema.createArray(Schema.create(Schema.Type.STRING)))); } private List getList(Supplier> supplier) { @@ -1993,58 +920,31 @@ private static T safeCast(Object toCast, Class clazz) { return Optional.of(toCast) .filter(clazz::isInstance) .map(clazz::cast) - .orElseThrow(() -> new ClassCastException(String.format("Cast failed to class: %s for object: %s", clazz.getCanonicalName(), toCast))); + .orElseThrow(() -> new ClassCastException( + String.format("Cast failed to class: %s for object: %s", clazz.getCanonicalName(), toCast))); } @DataProvider() public Object[][] arrayFieldProvider() { - return new Object[][] { - { - null - }, - { - Collections.emptyList() - }, - { - Collections.unmodifiableList(Arrays.asList("foo", "bar", "baz")) - }, - { - getList(() -> new ArrayList<>(Arrays.asList("foo", "bar"))) - }, - { - getList(() -> new LinkedList<>(Arrays.asList("foo", "bar"))) - }, - { - getList(() -> { - GenericArray array = new GenericData.Array<>(1, Schema.createArray(Schema.create(Schema.Type.STRING))); - array.add("foo"); - return array; - }) - } - }; + return new Object[][]{{null}, {Collections.emptyList()}, + {Collections.unmodifiableList(Arrays.asList("foo", "bar", "baz"))}, + {getList(() -> new ArrayList<>(Arrays.asList("foo", "bar")))}, + {getList(() -> new LinkedList<>(Arrays.asList("foo", "bar")))}, {getList(() -> { + GenericArray array = new GenericData.Array<>(1, Schema.createArray(Schema.create(Schema.Type.STRING))); + array.add("foo"); + return array; + })}}; } @Test(dataProvider = "arrayFieldProvider", description = "generic record to data map should not care about the specific list implementation") public void testArrayDataTranslation(List arrayFieldValue) throws IOException { final String arrayField = "arrayField"; - final String SCHEMA = - "{" + - " \"type\":\"record\"," + - " \"name\":\"Foo\"," + - " \"fields\":[" + - " {" + - " \"name\":\"arrayField\"," + - " \"type\":{" + - " \"type\":\"array\"," + - " \"items\":\"string\"" + - " }," + - " \"default\":[ ]" + - " }" + - " ]" + - "}"; + final String SCHEMA = "{" + " \"type\":\"record\"," + " \"name\":\"Foo\"," + " \"fields\":[" + " {" + + " \"name\":\"arrayField\"," + " \"type\":{" + " \"type\":\"array\"," + + " \"items\":\"string\"" + " }," + " \"default\":[ ]" + " }" + " ]" + "}"; // generate generic record from data map and pegasus schema - RecordDataSchema pegasusSchema = (RecordDataSchema)TestUtil.dataSchemaFromString(SCHEMA); + RecordDataSchema pegasusSchema = (RecordDataSchema) TestUtil.dataSchemaFromString(SCHEMA); Schema avroShema = Schema.parse(SCHEMA); DataMap dataMap = new DataMap(); GenericRecord record = DataTranslator.dataMapToGenericRecord(dataMap, pegasusSchema, avroShema); @@ -2066,42 +966,43 @@ public void testArrayDataTranslation(List arrayFieldValue) throws IOExce * So that DataTranslator is able to find the corresponding schema between Avro and Pegasus. */ @Test - public void testNamespaceOverrideWithOptionalField() throws IOException - { - String schemaText = "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"namespace\" : \"a.b.c\",\n" + - " \"fields\" : [\n" + - " { \"name\" : \"a\", \"type\" : { \"type\" : \"record\", \"name\" : \"FooFoo\", \"fields\" : [ { \"name\" : \"b\", \"type\" : \"int\" } ] }, \"optional\": true }\n" + - " ]\n" + - "}\n"; + public void testNamespaceOverrideWithOptionalField() throws IOException { + String schemaText = + "{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"namespace\" : \"a.b.c\",\n" + + " \"fields\" : [\n" + + " { \"name\" : \"a\", \"type\" : { \"type\" : \"record\", \"name\" : \"FooFoo\", \"fields\" : [ { \"name\" : \"b\", \"type\" : \"int\" } ] }, \"optional\": true }\n" + + " ]\n" + "}\n"; RecordDataSchema recordDataSchema = (RecordDataSchema) TestUtil.dataSchemaFromString(schemaText); - String avroSchemaText = "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"namespace\" : \"avro.a.b.c\",\n" + - " \"fields\" : [\n" + - " { \"name\" : \"a\", \"type\" : [ \"null\", { \"type\" : \"record\", \"name\" : \"FooFoo\", \"fields\" : [ { \"name\" : \"b\", \"type\" : \"int\" } ] } ] }\n" + - " ]\n" + - "}\n"; + String avroSchemaText = + "{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"namespace\" : \"avro.a.b.c\",\n" + + " \"fields\" : [\n" + + " { \"name\" : \"a\", \"type\" : [ \"null\", { \"type\" : \"record\", \"name\" : \"FooFoo\", \"fields\" : [ { \"name\" : \"b\", \"type\" : \"int\" } ] } ] }\n" + + " ]\n" + "}\n"; Schema avroSchema = Schema.parse(avroSchemaText); - GenericRecord avroRecord = AvroUtil.genericRecordFromJson(TestAvroUtil.namespaceProcessor("{ \"a\" : { \"##NS(avro.a.b.c.)FooFoo\": { \"b\" : 1 } } }"), avroSchema); + GenericRecord avroRecord = AvroUtil.genericRecordFromJson( + TestAvroUtil.namespaceProcessor("{ \"a\" : { \"##NS(avro.a.b.c.)FooFoo\": { \"b\" : 1 } } }"), avroSchema); // Test Avro-to-Pegasus - AvroRecordToDataMapTranslationOptions avroRecordToDataMapTranslationOptions = new AvroRecordToDataMapTranslationOptions(); - avroRecordToDataMapTranslationOptions.setAvroToDataSchemaNamespaceMapping(Collections.singletonMap("avro.a.b.c", "a.b.c")); - DataMap pegasusDataMap = DataTranslator.genericRecordToDataMap(avroRecord, recordDataSchema, avroSchema, avroRecordToDataMapTranslationOptions); + AvroRecordToDataMapTranslationOptions avroRecordToDataMapTranslationOptions = + new AvroRecordToDataMapTranslationOptions(); + avroRecordToDataMapTranslationOptions.setAvroToDataSchemaNamespaceMapping( + Collections.singletonMap("avro.a.b.c", "a.b.c")); + DataMap pegasusDataMap = DataTranslator.genericRecordToDataMap(avroRecord, recordDataSchema, avroSchema, + avroRecordToDataMapTranslationOptions); Assert.assertEquals(pegasusDataMap.getDataMap("a").get("b"), 1); // Test Pegasus-to-Avro - DataMapToAvroRecordTranslationOptions dataMapToAvroRecordTranslationOptions = new DataMapToAvroRecordTranslationOptions(); - dataMapToAvroRecordTranslationOptions.setAvroToDataSchemaNamespaceMapping(Collections.singletonMap("avro.a.b.c", "a.b.c")); - GenericRecord reconvertedAvroRecord = DataTranslator.dataMapToGenericRecord(pegasusDataMap, recordDataSchema, avroSchema, dataMapToAvroRecordTranslationOptions); + DataMapToAvroRecordTranslationOptions dataMapToAvroRecordTranslationOptions = + new DataMapToAvroRecordTranslationOptions(); + dataMapToAvroRecordTranslationOptions.setAvroToDataSchemaNamespaceMapping( + Collections.singletonMap("avro.a.b.c", "a.b.c")); + GenericRecord reconvertedAvroRecord = + DataTranslator.dataMapToGenericRecord(pegasusDataMap, recordDataSchema, avroSchema, + dataMapToAvroRecordTranslationOptions); Assert.assertEquals(((GenericRecord) reconvertedAvroRecord.get("a")).get("b"), 1); } @@ -2115,42 +1016,43 @@ public void testNamespaceOverrideWithOptionalField() throws IOException * So that DataTranslator is able to find the corresponding schema between Avro and Pegasus. */ @Test - public void testNamespaceOverrideWithUnion() throws IOException - { - String schemaText = "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"namespace\" : \"a.b.c\",\n" + - " \"fields\" : [\n" + - " { \"name\" : \"a\", \"type\" : [ \"int\", { \"type\" : \"record\", \"name\" : \"FooFoo\", \"fields\" : [ { \"name\" : \"b\", \"type\" : \"int\" } ] } ] }\n" + - " ]\n" + - "}\n"; + public void testNamespaceOverrideWithUnion() throws IOException { + String schemaText = + "{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"namespace\" : \"a.b.c\",\n" + + " \"fields\" : [\n" + + " { \"name\" : \"a\", \"type\" : [ \"int\", { \"type\" : \"record\", \"name\" : \"FooFoo\", \"fields\" : [ { \"name\" : \"b\", \"type\" : \"int\" } ] } ] }\n" + + " ]\n" + "}\n"; RecordDataSchema recordDataSchema = (RecordDataSchema) TestUtil.dataSchemaFromString(schemaText); - String avroSchemaText = "{\n" + - " \"type\" : \"record\",\n" + - " \"name\" : \"Foo\",\n" + - " \"namespace\" : \"avro.a.b.c\",\n" + - " \"fields\" : [\n" + - " { \"name\" : \"a\", \"type\" : [ \"int\", { \"type\" : \"record\", \"name\" : \"FooFoo\", \"fields\" : [ { \"name\" : \"b\", \"type\" : \"int\" } ] } ] }\n" + - " ]\n" + - "}\n"; + String avroSchemaText = + "{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"namespace\" : \"avro.a.b.c\",\n" + + " \"fields\" : [\n" + + " { \"name\" : \"a\", \"type\" : [ \"int\", { \"type\" : \"record\", \"name\" : \"FooFoo\", \"fields\" : [ { \"name\" : \"b\", \"type\" : \"int\" } ] } ] }\n" + + " ]\n" + "}\n"; Schema avroSchema = Schema.parse(avroSchemaText); - GenericRecord avroRecord = AvroUtil.genericRecordFromJson(TestAvroUtil.namespaceProcessor("{ \"a\" : { \"##NS(avro.a.b.c.)FooFoo\": { \"b\" : 1 } } }"), avroSchema); + GenericRecord avroRecord = AvroUtil.genericRecordFromJson( + TestAvroUtil.namespaceProcessor("{ \"a\" : { \"##NS(avro.a.b.c.)FooFoo\": { \"b\" : 1 } } }"), avroSchema); // Test Avro-to-Pegasus - AvroRecordToDataMapTranslationOptions avroRecordToDataMapTranslationOptions = new AvroRecordToDataMapTranslationOptions(); - avroRecordToDataMapTranslationOptions.setAvroToDataSchemaNamespaceMapping(Collections.singletonMap("avro.a.b.c", "a.b.c")); - DataMap pegasusDataMap = DataTranslator.genericRecordToDataMap(avroRecord, recordDataSchema, avroSchema, avroRecordToDataMapTranslationOptions); + AvroRecordToDataMapTranslationOptions avroRecordToDataMapTranslationOptions = + new AvroRecordToDataMapTranslationOptions(); + avroRecordToDataMapTranslationOptions.setAvroToDataSchemaNamespaceMapping( + Collections.singletonMap("avro.a.b.c", "a.b.c")); + DataMap pegasusDataMap = DataTranslator.genericRecordToDataMap(avroRecord, recordDataSchema, avroSchema, + avroRecordToDataMapTranslationOptions); Assert.assertEquals(pegasusDataMap.getDataMap("a").getDataMap("a.b.c.FooFoo").get("b"), 1); // Test Pegasus-to-Avro - DataMapToAvroRecordTranslationOptions dataMapToAvroRecordTranslationOptions = new DataMapToAvroRecordTranslationOptions(); - dataMapToAvroRecordTranslationOptions.setAvroToDataSchemaNamespaceMapping(Collections.singletonMap("avro.a.b.c", "a.b.c")); - GenericRecord reconvertedAvroRecord = DataTranslator.dataMapToGenericRecord(pegasusDataMap, recordDataSchema, avroSchema, dataMapToAvroRecordTranslationOptions); + DataMapToAvroRecordTranslationOptions dataMapToAvroRecordTranslationOptions = + new DataMapToAvroRecordTranslationOptions(); + dataMapToAvroRecordTranslationOptions.setAvroToDataSchemaNamespaceMapping( + Collections.singletonMap("avro.a.b.c", "a.b.c")); + GenericRecord reconvertedAvroRecord = + DataTranslator.dataMapToGenericRecord(pegasusDataMap, recordDataSchema, avroSchema, + dataMapToAvroRecordTranslationOptions); Assert.assertEquals(((GenericRecord) reconvertedAvroRecord.get("a")).get("b"), 1); } @@ -2182,7 +1084,6 @@ public void testDataMapToSpecificRecordTranslatorUnion(String field1, String fie Assert.assertEquals(event.get(event.getSchema().getField(field3).pos()), enumData); } - //Test nested records and Array @Test public void testDataMapToSpecificRecordTranslatorInnerRecord() throws IOException { @@ -2198,9 +1099,9 @@ public void testDataMapToSpecificRecordTranslatorInnerRecord() throws IOExceptio innerMap2.put("eventData", new DataMap(ImmutableMap.of("long", 1L))); innerMap2.put("enumData", EnumData.APPROVED.toString()); - TestEventRecordOfRecord testEventRecordOfRecord = - DataTranslator.dataMapToSpecificRecord(new DataMap(ImmutableMap.of("innerField", innerMap2, "stringArray", new DataList(Arrays.asList("val1")))), recordDataSchema, - avroSchema); + TestEventRecordOfRecord testEventRecordOfRecord = DataTranslator.dataMapToSpecificRecord( + new DataMap(ImmutableMap.of("innerField", innerMap2, "stringArray", new DataList(Arrays.asList("val1")))), + recordDataSchema, avroSchema); TestEventWithUnionAndEnum innerEvent = DataTranslator.dataMapToSpecificRecord(innerMap2, innerRecordDataSchema, innerAvroSchema); Assert.assertEquals(testEventRecordOfRecord.get(0), innerEvent); @@ -2221,7 +1122,8 @@ public void testArrayOfRecords() throws IOException { recordDataSchema, RecordArray.TEST_SCHEMA); StringRecord stringRecordEvent = DataTranslator.dataMapToSpecificRecord(stringRecord, - (RecordDataSchema) TestUtil.dataSchemaFromString(StringRecord.TEST_SCHEMA.toString()), StringRecord.TEST_SCHEMA); + (RecordDataSchema) TestUtil.dataSchemaFromString(StringRecord.TEST_SCHEMA.toString()), + StringRecord.TEST_SCHEMA); Assert.assertEquals(recordArrayEvent.get(0), Arrays.asList(stringRecordEvent)); } @@ -2236,13 +1138,14 @@ public void testMapOfRecords() throws IOException { DataMap recordMap = new DataMap(ImmutableMap.of("key", stringRecord)); RecordMap mapRecordEvent = - DataTranslator.dataMapToSpecificRecord(new DataMap(ImmutableMap.of("recordMap", recordMap)), - recordDataSchema, RecordMap.TEST_SCHEMA); + DataTranslator.dataMapToSpecificRecord(new DataMap(ImmutableMap.of("recordMap", recordMap)), recordDataSchema, + RecordMap.TEST_SCHEMA); StringRecord stringRecordEvent = DataTranslator.dataMapToSpecificRecord(stringRecord, - (RecordDataSchema) TestUtil.dataSchemaFromString(StringRecord.TEST_SCHEMA.toString()), StringRecord.TEST_SCHEMA); + (RecordDataSchema) TestUtil.dataSchemaFromString(StringRecord.TEST_SCHEMA.toString()), + StringRecord.TEST_SCHEMA); - Assert.assertEquals(((HashMap)mapRecordEvent.get(0)).get("key"), stringRecordEvent); + Assert.assertEquals(((HashMap) mapRecordEvent.get(0)).get("key"), stringRecordEvent); } @Test @@ -2259,22 +1162,32 @@ public void testMapArrayUnion() throws IOException { RecordDataSchema recordDataSchemaOfMapOfMap = (RecordDataSchema) TestUtil.dataSchemaFromString(MapOfMapOfArrayOfMapArrayUnion.TEST_SCHEMA.toString()); - DataMap arrayData = new DataMap(ImmutableMap.of("mapOrArray", new DataMap(ImmutableMap.of("array", new DataList(Arrays.asList("a")))))); - MapArrayUnion arrayUnion = DataTranslator.dataMapToSpecificRecord(arrayData, recordDataSchemaMapArrayUnion, MapArrayUnion.TEST_SCHEMA); + DataMap arrayData = new DataMap( + ImmutableMap.of("mapOrArray", new DataMap(ImmutableMap.of("array", new DataList(Arrays.asList("a")))))); + MapArrayUnion arrayUnion = + DataTranslator.dataMapToSpecificRecord(arrayData, recordDataSchemaMapArrayUnion, MapArrayUnion.TEST_SCHEMA); - DataMap mapData = new DataMap(ImmutableMap.of("mapOrArray", new DataMap(ImmutableMap.of("map", new DataMap( - ImmutableMap.of("key", "value") - ))))); - MapArrayUnion mapUnion = DataTranslator.dataMapToSpecificRecord(mapData, recordDataSchemaMapArrayUnion, MapArrayUnion.TEST_SCHEMA); + DataMap mapData = new DataMap(ImmutableMap.of("mapOrArray", + new DataMap(ImmutableMap.of("map", new DataMap(ImmutableMap.of("key", "value")))))); + MapArrayUnion mapUnion = + DataTranslator.dataMapToSpecificRecord(mapData, recordDataSchemaMapArrayUnion, MapArrayUnion.TEST_SCHEMA); - DataMap arrayOfMapArrayUnionData = new DataMap(ImmutableMap.of("recordArray", new DataList(Arrays.asList(arrayData, mapData)))); - ArrayOfMapArrayUnion arrayOfMapArrayUnion = DataTranslator.dataMapToSpecificRecord(arrayOfMapArrayUnionData, recordDataSchema, ArrayOfMapArrayUnion.TEST_SCHEMA); + DataMap arrayOfMapArrayUnionData = + new DataMap(ImmutableMap.of("recordArray", new DataList(Arrays.asList(arrayData, mapData)))); + ArrayOfMapArrayUnion arrayOfMapArrayUnion = + DataTranslator.dataMapToSpecificRecord(arrayOfMapArrayUnionData, recordDataSchema, + ArrayOfMapArrayUnion.TEST_SCHEMA); DataMap mapOfArrayOfMapArrayUnionData = new DataMap(ImmutableMap.of("recordMap", arrayOfMapArrayUnionData)); - MapOfArrayOfMapArrayUnion mapOfArrayOfMapArrayUnion = DataTranslator.dataMapToSpecificRecord(mapOfArrayOfMapArrayUnionData, recordDataSchemaOfMap, MapOfArrayOfMapArrayUnion.TEST_SCHEMA); + MapOfArrayOfMapArrayUnion mapOfArrayOfMapArrayUnion = + DataTranslator.dataMapToSpecificRecord(mapOfArrayOfMapArrayUnionData, recordDataSchemaOfMap, + MapOfArrayOfMapArrayUnion.TEST_SCHEMA); - DataMap mapOfMapOfArrayOfMapArrayUnionData = new DataMap(ImmutableMap.of("recordMap", new DataMap(ImmutableMap.of("recordMap", mapOfArrayOfMapArrayUnionData)))); - MapOfMapOfArrayOfMapArrayUnion mapOfMapOfArrayOfMapArrayUnion = DataTranslator.dataMapToSpecificRecord(mapOfMapOfArrayOfMapArrayUnionData, recordDataSchemaOfMapOfMap, MapOfMapOfArrayOfMapArrayUnion.TEST_SCHEMA); + DataMap mapOfMapOfArrayOfMapArrayUnionData = new DataMap( + ImmutableMap.of("recordMap", new DataMap(ImmutableMap.of("recordMap", mapOfArrayOfMapArrayUnionData)))); + MapOfMapOfArrayOfMapArrayUnion mapOfMapOfArrayOfMapArrayUnion = + DataTranslator.dataMapToSpecificRecord(mapOfMapOfArrayOfMapArrayUnionData, recordDataSchemaOfMapOfMap, + MapOfMapOfArrayOfMapArrayUnion.TEST_SCHEMA); Assert.assertTrue(arrayUnion.get(0) instanceof List); Assert.assertEquals(((List) arrayUnion.get(0)).get(0), "a"); @@ -2282,14 +1195,55 @@ public void testMapArrayUnion() throws IOException { Assert.assertEquals(((Map) mapUnion.get(0)).get("key"), "value"); Assert.assertTrue(arrayOfMapArrayUnion.get(0) instanceof List); - Assert.assertTrue(((MapArrayUnion)((List) arrayOfMapArrayUnion.get(0)).get(0)).get(0) instanceof List); - Assert.assertTrue(((MapArrayUnion)((List) arrayOfMapArrayUnion.get(0)).get(1)).get(0) instanceof Map); + Assert.assertTrue(((MapArrayUnion) ((List) arrayOfMapArrayUnion.get(0)).get(0)).get(0) instanceof List); + Assert.assertTrue(((MapArrayUnion) ((List) arrayOfMapArrayUnion.get(0)).get(1)).get(0) instanceof Map); Assert.assertTrue(mapOfArrayOfMapArrayUnion.get(0) instanceof Map); - Assert.assertEquals(((Map) mapOfArrayOfMapArrayUnion.get(0)).get("recordArray"), Arrays.asList(arrayUnion, mapUnion)); + Assert.assertEquals(((Map) mapOfArrayOfMapArrayUnion.get(0)).get("recordArray"), + Arrays.asList(arrayUnion, mapUnion)); Assert.assertTrue(mapOfMapOfArrayOfMapArrayUnion.get(0) instanceof Map); - Assert.assertEquals(((Map) mapOfMapOfArrayOfMapArrayUnion.get(0)).get("recordMap"), mapOfArrayOfMapArrayUnion); + Assert.assertEquals(((Map) mapOfMapOfArrayOfMapArrayUnion.get(0)).get("recordMap"), + mapOfArrayOfMapArrayUnion); + } + + @Test + public void testUnionWithoutFieldDescriptor() throws IOException { + String protoToAvroSchema = "{\n" + " \"type\" : \"record\",\n" + " \"name\" : \"Foo\",\n" + " \"fields\" : [ {\n" + + " \"name\" : \"oneofField\",\n" + " \"type\" : [ \"null\", {\n" + " \"type\" : \"record\",\n" + + " \"name\" : \"OneofFieldWrapper\",\n" + " \"fields\" : [ {\n" + + " \"name\" : \"oneofIntField\",\n" + " \"type\" : [ \"null\", {\n" + + " \"type\" : \"int\",\n" + " \"li.data.proto.numberFieldType\" : \"int32\"\n" + + " } ],\n" + " \"default\" : null,\n" + " \"li.data.proto.fieldNumber\" : 1\n" + + " }, {\n" + " \"name\" : \"oneofStringField\",\n" + " \"type\" : [ \"null\", \"string\" ],\n" + + " \"default\" : null,\n" + " \"li.data.proto.fieldNumber\" : 2\n" + " } ],\n" + + " \"li.data.proto.oneOfFieldWrapperType\" : true\n" + " } ],\n" + " \"default\" : null\n" + " } ],\n" + + " \"li.data.proto.fullyQualifiedName\" : \"proto.com.linkedin.alcantara.test.DTTest\"\n" + "}\n"; + + String protoToPdlToAvroSchema = + "{\"type\":\"record\"," + "\"name\":\"Foo\"," + "\"fields\":[{\"name\":\"oneofField\"," + "\"type\":[\"null\",{" + + "\"type\":\"record\"," + "\"name\":\"FooOneofField\"," + "\"fields\":[{" + "\"name\":\"oneofIntField\"," + + "\"type\":[\"null\",\"int\"]," + "\"default\":null},{" + "\"name\":\"oneofStringField\"," + + "\"type\":[\"null\",\"string\"]," + "\"default\":null},{" + "\"name\":\"fieldDiscriminator\"," + + "\"type\":{" + "\"type\":\"enum\"," + "\"name\":\"FooOneofFieldDiscriminator\"," + + "\"symbols\":[\"oneofIntField\",\"oneofStringField\"]}" + " }]}]," + "\"default\":null}]}"; + Schema protoToAvro = Schema.parse(protoToAvroSchema); + Schema protoToPdlToAvro = Schema.parse(protoToPdlToAvroSchema); + GenericRecord avroRecord = AvroUtil.genericRecordFromJson( + "{\n" + " \"oneofField\" : {\n" + " \"OneofFieldWrapper\" : {\n" + " \"oneofIntField\" : {\n" + + " \"int\" : 1\n" + " },\n" + " \"oneofStringField\" : null\n" + " }\n" + " }\n" + + "}", protoToAvro); + GenericRecord avroRecordFromPdl = AvroUtil.genericRecordFromJson( + "{\n" + " \"oneofField\" : {\n" + " \"FooOneofField\" : {\n" + " \"oneofIntField\" : {\n" + + " \"int\" : 1\n" + " },\n" + " \"oneofStringField\" : null,\n" + + " \"fieldDiscriminator\" : \"oneofIntField\"\n" + " }\n" + " }\n" + "}", protoToPdlToAvro); + RecordDataSchema recordDataSchema = (RecordDataSchema) TestUtil.dataSchemaFromString(protoToPdlToAvroSchema); + DataMap expectedMap = DataTranslator.genericRecordToDataMap(avroRecordFromPdl, recordDataSchema, protoToPdlToAvro); + DataMap translatedMap = DataTranslator.genericRecordToDataMap(avroRecord, recordDataSchema, protoToPdlToAvro); + + Assert.assertEquals(translatedMap, expectedMap); } } + +