diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml new file mode 100644 index 0000000..db98e0a --- /dev/null +++ b/.github/workflows/maven.yml @@ -0,0 +1,35 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Java CI with Maven + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: maven + - name: Build with Maven + run: mvn -B package --file pom.xml + + # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive +# - name: Update dependency graph +# uses: advanced-security/maven-dependency-submission-action@571e99aab1055c2e71a1e2309b9691de18d6b7d6 diff --git a/.gitignore b/.gitignore index 560d08c..d8b2bee 100755 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ build.xml nb-configuration.xml *.versionsBackup .gradle +.factorypath +.nondex diff --git a/CliArg/pom.xml b/CliArg/pom.xml index d96d0a4..6a72d51 100644 --- a/CliArg/pom.xml +++ b/CliArg/pom.xml @@ -6,7 +6,7 @@ org.jsonex jcParent - 0.1.21 + 0.1.27 ../pom.xml CliArg diff --git a/CliArg/src/main/java/org/jsonex/cliarg/CLIParser.java b/CliArg/src/main/java/org/jsonex/cliarg/CLIParser.java index 783c386..27ff251 100644 --- a/CliArg/src/main/java/org/jsonex/cliarg/CLIParser.java +++ b/CliArg/src/main/java/org/jsonex/cliarg/CLIParser.java @@ -1,20 +1,26 @@ package org.jsonex.cliarg; +import lombok.Data; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import org.jsonex.core.util.BeanConvertContext; import org.jsonex.core.util.ClassUtil; +import static org.jsonex.core.util.LangUtil.doIf; +import static org.jsonex.core.util.LangUtil.doIfNotNull; +import static org.jsonex.core.util.ListUtil.isIn; import org.jsonex.jsoncoder.JSONCoder; import org.jsonex.jsoncoder.JSONCoderOption; import org.jsonex.treedoc.TDNode.Type; import org.jsonex.treedoc.json.TDJSONOption; -import lombok.Data; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import java.util.*; - -import static org.jsonex.core.util.LangUtil.doIf; -import static org.jsonex.core.util.ListUtil.isIn; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; /** * Parse the input command line arguments against the {@link CLISpec}. The parsed result will be stored in the target @@ -106,7 +112,7 @@ private void parseArg(String arg) { } Param param = spec.indexedParams.get(paramIndex++); missingParams.remove(param.name); - param.property.set(target, parseValue(param, arg)); + doIfNotNull(parseValue(param, arg), v -> param.property.set(target, v)); } private Object parseValue(Param param, String value) { @@ -124,7 +130,8 @@ private Object parseValue(Param param, String value) { ? JSONCoder.decode(value, cls, opt) : JSONCoder.decodeTo(value, param.getProperty().get(target), opt.setMergeArray(true)); } catch (Exception e) { - log.error("Error parsing parameter:" + param.name, e); + errorMessages.put(param.name, value + ";" + e.toString()); + // log.error("Error parsing parameter:" + param.name, e); } return null; } @@ -133,9 +140,9 @@ private Object parseValue(Param param, String value) { public String getErrorsAsString() { StringBuilder sb = new StringBuilder(); - doIf(!missingParams.isEmpty(), () -> sb.append("\nMissing required arguments:" + missingParams)); - doIf(!extraArgs.isEmpty(), () -> sb.append("\nUnexpected arguments:" + extraArgs)); - doIf(!errorMessages.isEmpty(), () -> sb.append("\nError parsing following arguments:" + errorMessages)); + doIf(!missingParams.isEmpty(), () -> sb.append("\nMissing required arguments:").append(missingParams)); + doIf(!extraArgs.isEmpty(), () -> sb.append("\nUnexpected arguments:").append(extraArgs)); + doIf(!errorMessages.isEmpty(), () -> sb.append("\nError parsing following arguments:").append(errorMessages)); return sb.toString(); } } diff --git a/CliArg/src/main/java/org/jsonex/cliarg/CLISpec.java b/CliArg/src/main/java/org/jsonex/cliarg/CLISpec.java index ed069f8..b4155b4 100644 --- a/CliArg/src/main/java/org/jsonex/cliarg/CLISpec.java +++ b/CliArg/src/main/java/org/jsonex/cliarg/CLISpec.java @@ -20,6 +20,7 @@ import static org.jsonex.core.util.ListUtil.setAt; /** + *
  * CLI specification based on annotated java bean of `cls`. Following annotations will be processed:
  *
  * Class level:
@@ -27,9 +28,10 @@
  *   {@link Summary}: Summary of the command (Optional)
  *   {@link Description}: Description of the command (Optional)
  *   {@link Examples}: Array of string representation of samples usages (Optional)
- *
  * For field level annotations, please refer to class {@link Param}
  *
+ * 
+ * * @param */ @Data diff --git a/CliArg/src/main/java/org/jsonex/cliarg/Param.java b/CliArg/src/main/java/org/jsonex/cliarg/Param.java index 54abbda..6cadbbc 100644 --- a/CliArg/src/main/java/org/jsonex/cliarg/Param.java +++ b/CliArg/src/main/java/org/jsonex/cliarg/Param.java @@ -11,27 +11,28 @@ import static org.jsonex.core.util.StringUtil.noNull; /** - * Represent an command line parameter, it can be either argument or option - * If index is not null indicate it's argument - * Argument default to required unless explicitly specified. - * required argument can't follow non-required argument which index less than it - * If index is null indicates it's an option, option default to not required, unless specified - * - * For option of Boolean type, it will be mapped as flag, that means the value of the option can be omitted. * + *
+ * Represent a command line parameter, it can be either argument or option
+ * If index is not null indicate it is an argument.
+ *    Argument default to required unless explicitly specified.
+ *    Required argument can't follow non-required argument which index less than it
+ * If index is null indicates it's an option, option default to not required, unless specified
+ *    For option of Boolean type, it will be mapped as flag, that means the value of the option can be omitted.
+
  * For Param of complex type or array/list, the value can be specified as JSON(ex) string, the top level "{" or "[",
  * can be, omitted. The quote for key and value can be omitted.
- *
- * For array parameters, it also possible to specify the values as separate options. The values will be merged
- *
+
+ * For array parameters, it is also possible to specify the values as separate options. The values will be merged
+
  * Following Annotation will be processed for each parameter:
- *
  * {@link Name}  Name of the parameter, optional, default to field name
  * {@link ShortName}  The optional short name
  * {@link Description}  The optional description
  * {@link Index}  Indicate this an indexed parameter
  * {@link Required}  Indicate if this field is required. all the index fields are required unless explicitly indicated.
  *     All the non-index fields are not required unless explicitly indicated.
+ * 
*/ @Data public class Param { diff --git a/CliArg/src/test/java/org/jsonex/cliarg/CliParserTest.java b/CliArg/src/test/java/org/jsonex/cliarg/CliParserTest.java index a3ed5a7..2bee19b 100644 --- a/CliArg/src/test/java/org/jsonex/cliarg/CliParserTest.java +++ b/CliArg/src/test/java/org/jsonex/cliarg/CliParserTest.java @@ -49,7 +49,7 @@ public static class Arg1 { @Test public void testParse() { - CLISpec spec = new CLISpec(Arg1.class); + CLISpec spec = new CLISpec<>(Arg1.class); assertMatchesSnapshot("spec", spec); log.info("spec:\n" + spec.printUsage()); @@ -57,7 +57,7 @@ public void testParse() { String[] args = { "abc", "10", "name:n1,x:1,y:2", "-o", "VAL2", "--optInt", "100", "--arrayArg", "str1,str2,'It\\'s escapted'", "--arrayArg", "array as separate option"}; - CLIParser parser = spec.parse(args, 0); + CLIParser parser = spec.parse(args, 0); log.info("parsedValue:\n" + parser.target); assertMatchesSnapshot("parserTarget", parser.target); diff --git a/CliArg/src/test/resources/org/jsonex/cliarg/__snapshot__/CliParserTest_testParse_parserTarget.json b/CliArg/src/test/resources/org/jsonex/cliarg/__snapshot__/CliParserTest_testParse_parserTarget.json index 775cbae..b714c60 100644 --- a/CliArg/src/test/resources/org/jsonex/cliarg/__snapshot__/CliParserTest_testParse_parserTarget.json +++ b/CliArg/src/test/resources/org/jsonex/cliarg/__snapshot__/CliParserTest_testParse_parserTarget.json @@ -1,18 +1,18 @@ { - "strParam":"abc", + "arrayArg":[ + "str1", + "str2", + "It's escapted", + "array as separate option" + ], + "noShortName":0, "numParam":10, + "opt":"VAL2", + "optInt":100, "point":{ "name":"n1", "x":1, "y":2 }, - "opt":"VAL2", - "optInt":100, - "noShortName":0, - "arrayArg":[ - "str1", - "str2", - "It's escapted", - "array as separate option" - ] + "strParam":"abc" } \ No newline at end of file diff --git a/CliArg/src/test/resources/org/jsonex/cliarg/__snapshot__/CliParserTest_testParse_spec.json b/CliArg/src/test/resources/org/jsonex/cliarg/__snapshot__/CliParserTest_testParse_spec.json index efee3d4..f90e309 100644 --- a/CliArg/src/test/resources/org/jsonex/cliarg/__snapshot__/CliParserTest_testParse_spec.json +++ b/CliArg/src/test/resources/org/jsonex/cliarg/__snapshot__/CliParserTest_testParse_spec.json @@ -1,58 +1,57 @@ { "cls":"org.jsonex.cliarg.CliParserTest.Arg1", "defVal":{ + "noShortName":0, "numParam":0, "opt":"VAL1", - "optInt":10, - "noShortName":0 + "optInt":10 }, - "name":"TestArg1", - "summary":"This is a test arg1", "description":"Description of test Args", "examples":[ "arg1 2", "arg1 4" ], "firstOptionalIndex":2, - "optionParams":[ - { - "name":"opt", - "shortName":"o", - "description":"Opt", - "defVal":"VAL1", - "required":true - }, + "indexedParams":[ { - "name":"optInt", - "shortName":"i", - "defVal":10 + "description":"Str parameter", + "index":0, + "name":"strParam" }, { - "name":"noShortName", - "defVal":0 + "defVal":0, + "description":"number parameter", + "index":1, + "name":"numParam" }, { - "name":"arrayArg", - "description":"array of string, you can specify either use ',' separated string (jsonex array, with escape supported), or pass as multiple options" + "description":"Object Point", + "index":2, + "name":"point", + "required":false } ], - "indexedParams":[ + "name":"TestArg1", + "optionParams":[ { - "name":"strParam", - "index":0, - "description":"Str parameter" + "defVal":"VAL1", + "description":"Opt", + "name":"opt", + "required":true, + "shortName":"o" }, { - "name":"numParam", - "index":1, - "description":"number parameter", - "defVal":0 + "defVal":10, + "name":"optInt", + "shortName":"i" }, { - "name":"point", - "index":2, - "description":"Object Point", - "required":false + "defVal":0, + "name":"noShortName" + }, + { + "description":"array of string, you can specify either use ',' separated string (jsonex array, with escape supported), or pass as multiple options", + "name":"arrayArg" } ], "requiredParams":[ @@ -60,5 +59,6 @@ "numParam", "opt" ], + "summary":"This is a test arg1", "usage":"TestArg1 -o {opt} [-i {optInt}] [--noShortName {noShortName}] [--arrayArg {arrayArg}] [point]" } \ No newline at end of file diff --git a/HiveUDF/pom.xml b/HiveUDF/pom.xml new file mode 100644 index 0000000..cc080dc --- /dev/null +++ b/HiveUDF/pom.xml @@ -0,0 +1,53 @@ + + + 4.0.0 + + org.jsonex + jcParent + 0.1.27 + ../pom.xml + + HiveUDF + HiveUDF + + + org.projectlombok + lombok + + + junit + junit + test + + + ${project.groupId} + JSONCoder + + + ${project.groupId} + csv + + + ${project.groupId} + SnapshotTest + test + + + ${project.groupId} + SnapshotTest + test + + + org.apache.hive + hive-exec + 4.0.0-alpha-2 + + + org.slf4j + slf4j-simple + test + + + diff --git a/HiveUDF/src/main/java/org/jsonex/hiveudf/ToCSVUDF.java b/HiveUDF/src/main/java/org/jsonex/hiveudf/ToCSVUDF.java new file mode 100644 index 0000000..142c146 --- /dev/null +++ b/HiveUDF/src/main/java/org/jsonex/hiveudf/ToCSVUDF.java @@ -0,0 +1,88 @@ +package org.jsonex.hiveudf; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.jsonex.core.util.ListUtil; +import org.jsonex.core.util.StringUtil; +import org.jsonex.csv.CSVOption; +import org.jsonex.csv.CSVWriter; +import org.jsonex.jsoncoder.JSONCoder; +import org.jsonex.jsoncoder.JSONCoderOption; + +import java.util.*; + +import static org.jsonex.core.util.ListUtil.map; + + +@Description(name = "to_csv", + value = "to_csv([opts], col1, col2, ...) - Serialize to CSV, if column value is not simple object, the value will be " + + "serialized as json. If can provide an optional opts parameter as the first parameters in a JSON format to " + + "specify field separator or quote. \n" + + "Avaliable options: fieldSep:char, quoteChar:char, noHeader:boolean, headers:array", + extended = "Example:\n" + + " > SELECT to_csv(*) FROM someTable \n" + + " > SELECT to_csv('{fieldSep:|,quoteChar:\"\\'\"}', *) FROM someTable \n" + + " > SELECT to_csv('{noHead:true}', *) FROM someTable \n" + + " > SELECT to_csv('{headers:[,,,col3,]}', *) FROM someTable \n" +) +@Slf4j +public class ToCSVUDF extends GenericUDF { + @Data + public static class CSVOpt extends CSVOption { + String[] headers; + boolean noHeader; + } + + ObjectInspector[] inspectors; + int recordNum; + + @Override + public ObjectInspector initialize(ObjectInspector[] inspectors) throws UDFArgumentException { + this.inspectors = inspectors; + recordNum = 0; + return PrimitiveObjectInspectorFactory.javaStringObjectInspector; + } + + // @Override public String[] getRequiredJars() { return new String[]{"ivy://org.jsonex:JSONCoder:0.1.21?transitive=true"}; } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + StringBuilder sb = new StringBuilder(); + CSVOpt opt = new CSVOpt(); + Map map = UDFUtil.toJavaMap(arguments, inspectors); + Optional firstKeyOpt = ListUtil.first(map.keySet()); + if (!firstKeyOpt.isPresent()) + return ""; + String firstKey = firstKeyOpt.get(); + Object firstVal = map.get(firstKey); + if (firstKey.equals("0") && firstVal instanceof String && ((String) firstVal).startsWith("{")) { // It's csv opt parameter. + JSONCoder.get().decodeTo((String) firstVal, opt); + map.remove(firstKey); + } + if (recordNum++ == 0 && !opt.noHeader) { + List headers = new ArrayList<>(map.keySet()); + if (opt.headers != null) { + for (int i = 0; i < headers.size(); i++) + if (i < opt.headers.length && !StringUtil.isEmpty(opt.headers[i])) + headers.set(i, opt.headers[i]); + } + + sb.append(CSVWriter.get().encodeRecord(headers, opt)).append("\n"); + } + List values = map(map.values(), v -> v instanceof List || v instanceof Map + ? JSONCoder.get().encode(v, new JSONCoderOption().setStrictOrdering(true)) : v); + sb.append(CSVWriter.get().encodeRecord(values, opt)); + return sb.toString(); + } + + @Override + public String getDisplayString(String[] children) { + return getStandardDisplayString("ToCSVUDF", children); + } +} \ No newline at end of file diff --git a/HiveUDF/src/main/java/org/jsonex/hiveudf/ToJsonUDF.java b/HiveUDF/src/main/java/org/jsonex/hiveudf/ToJsonUDF.java new file mode 100644 index 0000000..a2187d0 --- /dev/null +++ b/HiveUDF/src/main/java/org/jsonex/hiveudf/ToJsonUDF.java @@ -0,0 +1,65 @@ +package org.jsonex.hiveudf; + +import lombok.extern.slf4j.Slf4j; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.jsonex.jsoncoder.JSONCoder; +import org.jsonex.jsoncoder.JSONCoderOption; + + +/** + add jar ivy://org.jsonex:HiveUDF:0.1.23?transitive=true; + add jar file:///Users/jianwche/opensource/jsonex/HiveUDF/target/HiveUDF-0.1.23.jar; + add jar file:///Users/jianwche/opensource/jsonex/JSONCoder/target/JSONCoder-0.1.23.jar; + add jar file:///Users/jianwche/opensource/jsonex/core/target/core-0.1.23.jar; + add jar file:///Users/jianwche/opensource/jsonex/treedoc/target/treedoc-0.1.23.jar; + add jar file:///Users/jianwche/opensource/jsonex/csv/target/csv-0.1.23.jar; + CREATE TEMPORARY FUNCTION to_json AS 'org.jsonex.hiveudf.ToJsonUDF'; + CREATE TEMPORARY FUNCTION to_csv AS 'org.jsonex.hiveudf.ToCSVUDF'; + + create table a(i int, m MAP>, s STRUCT); + insert into a values(1, map('ab',named_struct('gender', 'm', 'age', 10), 'cd', named_struct('gender', 'f', 'age', 11)), named_struct('address', 'ca', 'zip', '123')); + create table a(i int, m ARRAY>); + select to_json(*) from a; + select to_json(s) from a; + */ +@Description(name = "to_json", + value = "to_json(obj1, obj2,...) - Serialize to json. \n", + extended = "Example:\n" + + " > select to_json(*) from tbl") +@Slf4j +public class ToJsonUDF extends GenericUDF { + ObjectInspector[] inspectors; + + @Override + public ObjectInspector initialize(ObjectInspector[] inspectors) throws UDFArgumentException { + this.inspectors = inspectors; + return PrimitiveObjectInspectorFactory.javaStringObjectInspector; + } + + // @Override public String[] getRequiredJars() { return new String[]{"ivy://org.jsonex:JSONCoder:0.1.21?transitive=true"}; } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { +// log.info("args=" + toJson(arguments) + "\ninspects:" + toJson(inspectors) + "\nchildren:" + toJson(children)); +// log.info("args=" + toJson(arguments) + "\ninspects:" + toJson(inspectors)); + return arguments.length == 1 + ? toJson(UDFUtil.toJavaObj(arguments[0].get(), inspectors[0])) + : toJson(UDFUtil.toJavaMap(arguments, inspectors)); + } + + private static String toJson(Object obj) { +// JSONCoderOption opt = JSONCoderOption.of().setShowType(true).setShowPrivateField(true).setDedupWithRef(true) +// .setShowTransientField(true).addSkippedClasses(HiveConf.class); + return JSONCoder.get().encode(obj, JSONCoderOption.of().setStrictOrdering(true)); + } + + @Override + public String getDisplayString(String[] children) { + return getStandardDisplayString("ToJsonUDF", children); + } +} \ No newline at end of file diff --git a/HiveUDF/src/main/java/org/jsonex/hiveudf/UDFUtil.java b/HiveUDF/src/main/java/org/jsonex/hiveudf/UDFUtil.java new file mode 100644 index 0000000..5983a10 --- /dev/null +++ b/HiveUDF/src/main/java/org/jsonex/hiveudf/UDFUtil.java @@ -0,0 +1,59 @@ +package org.jsonex.hiveudf; + +import lombok.SneakyThrows; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.objectinspector.*; +import org.jsonex.core.util.ClassUtil; + +import java.util.*; + +public class UDFUtil { + @SneakyThrows + public static Map toJavaMap(GenericUDF.DeferredObject[] arguments, ObjectInspector[] inspectors) { + Map map = new LinkedHashMap<>(); + for (int i = 0; i < arguments.length; i++) + map.put(getColumnName(arguments, i), + toJavaObj( + arguments[i].get(), + inspectors[i])); + return map; + } + + public static String getColumnName(GenericUDF.DeferredObject[] arguments, int i) { + try { + // Use undocumented attributes, may not work for different version + // Defined in object: ExprNodeGenericFuncEvaluator$DeferredExprObject + return (String) ClassUtil.getObjectByPath(null, arguments[i], "eval.expr.column"); + } catch (Exception e) { + return String.valueOf(i); + } + } + + public static Object toJavaObj(Object val, ObjectInspector inspector) { + if (inspector instanceof PrimitiveObjectInspector) + return ((PrimitiveObjectInspector)inspector).getPrimitiveJavaObject(val); + if (inspector instanceof MapObjectInspector) { + Map result = new HashMap<>(); + MapObjectInspector mapInspector = (MapObjectInspector) inspector; + for (Map.Entry entry : mapInspector.getMap(val).entrySet()) + result.put(toJavaObj(entry.getKey(), mapInspector.getMapKeyObjectInspector()), + toJavaObj(entry.getValue(), mapInspector.getMapValueObjectInspector())); + return result; + } else if (inspector instanceof ListObjectInspector) { + List result = new ArrayList<>(); + ListObjectInspector listInspector = (ListObjectInspector) inspector; + for (Object item : listInspector.getList(val)) + result.add(toJavaObj(item, listInspector.getListElementObjectInspector())); + return result; + } else if (inspector instanceof StructObjectInspector) { + Map result = new HashMap<>(); + StructObjectInspector structInspector = (StructObjectInspector) inspector; + List list = structInspector.getStructFieldsDataAsList(val); + int i = 0; + for (StructField field : structInspector.getAllStructFieldRefs()) + result.put(field.getFieldName(), toJavaObj(list.get(i++), field.getFieldObjectInspector())); + return result; + } + return val; + } +} diff --git a/HiveUDF/src/test/java/org/jsonex/hiveudf/TestUtil.java b/HiveUDF/src/test/java/org/jsonex/hiveudf/TestUtil.java new file mode 100644 index 0000000..9eef7bb --- /dev/null +++ b/HiveUDF/src/test/java/org/jsonex/hiveudf/TestUtil.java @@ -0,0 +1,34 @@ +package org.jsonex.hiveudf; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.jsonex.core.util.MapBuilder; +import org.jsonex.jsoncoder.JSONCoder; +import org.jsonex.jsoncoder.JSONCoderOption; + +import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardMapObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.writableIntObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.writableStringObjectInspector; +import static org.jsonex.core.util.ListUtil.listOf; + +public class TestUtil { + public static Object buildMapArgs() { + return MapBuilder.mapOf(new Text("key"), listOf(new Text("m"), new IntWritable(10))).build(); + } + + public static ObjectInspector buildMapOI() { + return getStandardMapObjectInspector( + writableStringObjectInspector, + getStandardStructObjectInspector( + listOf("gender", "age"), + listOf(writableStringObjectInspector, writableIntObjectInspector)) + ); + } + + public static String toJson(Object obj) { + JSONCoderOption opt = JSONCoderOption.of().setJsonOption(false, '\'', 0).setStrictOrdering(true); + return JSONCoder.encode(obj, opt); + } +} diff --git a/HiveUDF/src/test/java/org/jsonex/hiveudf/ToCSVUDFTest.java b/HiveUDF/src/test/java/org/jsonex/hiveudf/ToCSVUDFTest.java new file mode 100644 index 0000000..cd21193 --- /dev/null +++ b/HiveUDF/src/test/java/org/jsonex/hiveudf/ToCSVUDFTest.java @@ -0,0 +1,42 @@ +package org.jsonex.hiveudf; + +import lombok.SneakyThrows; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.junit.Ignore; +import org.junit.Test; + +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.writableIntObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.writableStringObjectInspector; +import static org.jsonex.hiveudf.TestUtil.buildMapArgs; +import static org.jsonex.hiveudf.TestUtil.buildMapOI; +import static org.jsonex.snapshottest.Snapshot.assertMatchesSnapshot; + +// Reference: https://github.com/apache/hive/blob/master/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFSortArray.java +@Ignore("Failed to run on Java17 with error: java.lang.NoClassDefFoundError: Could not initialize class org.apache.hadoop.hive.common.StringInternUtils\n") +public class ToCSVUDFTest { + ToCSVUDF udf = new ToCSVUDF(); + + @SneakyThrows + @Test public void testEvaluateWithoutOption() { + udf.initialize(new ObjectInspector[]{ buildMapOI(), writableIntObjectInspector }); + assertMatchesSnapshot(udf.evaluate(new DeferredObject[]{ + new GenericUDF.DeferredJavaObject(buildMapArgs()), + new GenericUDF.DeferredJavaObject(new IntWritable(100)) + })); + } + + @SneakyThrows + @Test public void testEvaluateWithOptions() { + udf.initialize(new ObjectInspector[]{writableStringObjectInspector, buildMapOI(), writableIntObjectInspector}); + udf.getDisplayString(new String[]{ "string", "struct" }); + assertMatchesSnapshot(udf.evaluate(new DeferredObject[]{ + new GenericUDF.DeferredJavaObject(new Text("{fieldSep:|,quoteChar:\"\\'\"}")), + new GenericUDF.DeferredJavaObject(buildMapArgs()), + new GenericUDF.DeferredJavaObject(new IntWritable(100)) + })); + } +} diff --git a/HiveUDF/src/test/java/org/jsonex/hiveudf/ToJsonUDFTest.java b/HiveUDF/src/test/java/org/jsonex/hiveudf/ToJsonUDFTest.java new file mode 100644 index 0000000..d1dfeb4 --- /dev/null +++ b/HiveUDF/src/test/java/org/jsonex/hiveudf/ToJsonUDFTest.java @@ -0,0 +1,37 @@ +package org.jsonex.hiveudf; + +import lombok.SneakyThrows; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.io.IntWritable; +import org.junit.Ignore; +import org.junit.Test; + +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.writableIntObjectInspector; +import static org.jsonex.hiveudf.TestUtil.buildMapArgs; +import static org.jsonex.hiveudf.TestUtil.buildMapOI; +import static org.jsonex.snapshottest.Snapshot.assertMatchesSnapshot; + +// Reference: https://github.com/apache/hive/blob/master/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFSortArray.java +@Ignore("Failed to run on Java17 with error: java.lang.NoClassDefFoundError: Could not initialize class org.apache.hadoop.hive.common.StringInternUtils\n") +public class ToJsonUDFTest { + ToJsonUDF udf = new ToJsonUDF(); + + @SneakyThrows + @Test public void testEvaluateSingleArg() { + udf.initialize(new ObjectInspector[]{ buildMapOI() }); + assertMatchesSnapshot(udf.evaluate(new DeferredObject[]{ new GenericUDF.DeferredJavaObject(buildMapArgs()) })); + } + + @SneakyThrows + @Test public void testEvaluateMultiArg() { + udf.initialize(new ObjectInspector[]{ buildMapOI(), writableIntObjectInspector}); + udf.getDisplayString(new String[]{ "struct", "int" }); + assertMatchesSnapshot( + udf.evaluate(new DeferredObject[]{ + new GenericUDF.DeferredJavaObject(buildMapArgs()), + new GenericUDF.DeferredJavaObject(new IntWritable(100)) + })); + } +} diff --git a/HiveUDF/src/test/java/org/jsonex/hiveudf/UDFUtilTest.java b/HiveUDF/src/test/java/org/jsonex/hiveudf/UDFUtilTest.java new file mode 100644 index 0000000..2f0df05 --- /dev/null +++ b/HiveUDF/src/test/java/org/jsonex/hiveudf/UDFUtilTest.java @@ -0,0 +1,15 @@ +package org.jsonex.hiveudf; + +import org.junit.Ignore; +import org.junit.Test; + +import static org.jsonex.hiveudf.TestUtil.*; +import static org.junit.Assert.assertEquals; + +// Reference: https://github.com/apache/hive/blob/master/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFSortArray.java +@Ignore("Failed to run on Java17 with error: java.lang.NoClassDefFoundError: Could not initialize class org.apache.hadoop.hive.common.StringInternUtils\n") +public class UDFUtilTest { + @Test public void testToJavaObj() { + assertEquals("{key:{age:10,gender:'m'}}", toJson(UDFUtil.toJavaObj(buildMapArgs(), buildMapOI()))); + } +} diff --git a/HiveUDF/src/test/resources/org/jsonex/hiveudf/__snapshot__/ToCSVUDFTest_testEvaluateWithOptions.txt b/HiveUDF/src/test/resources/org/jsonex/hiveudf/__snapshot__/ToCSVUDFTest_testEvaluateWithOptions.txt new file mode 100644 index 0000000..e399bf1 --- /dev/null +++ b/HiveUDF/src/test/resources/org/jsonex/hiveudf/__snapshot__/ToCSVUDFTest_testEvaluateWithOptions.txt @@ -0,0 +1,2 @@ +'1'|'2' +{"key":{"age":10,"gender":"m"}}|100 \ No newline at end of file diff --git a/HiveUDF/src/test/resources/org/jsonex/hiveudf/__snapshot__/ToCSVUDFTest_testEvaluateWithoutOption.txt b/HiveUDF/src/test/resources/org/jsonex/hiveudf/__snapshot__/ToCSVUDFTest_testEvaluateWithoutOption.txt new file mode 100644 index 0000000..f85eb63 --- /dev/null +++ b/HiveUDF/src/test/resources/org/jsonex/hiveudf/__snapshot__/ToCSVUDFTest_testEvaluateWithoutOption.txt @@ -0,0 +1,2 @@ +"0","1" +"{""key"":{""age"":10,""gender"":""m""}}",100 \ No newline at end of file diff --git a/HiveUDF/src/test/resources/org/jsonex/hiveudf/__snapshot__/ToJsonUDFTest_testEvaluateMultiArg.txt b/HiveUDF/src/test/resources/org/jsonex/hiveudf/__snapshot__/ToJsonUDFTest_testEvaluateMultiArg.txt new file mode 100644 index 0000000..5f5afe1 --- /dev/null +++ b/HiveUDF/src/test/resources/org/jsonex/hiveudf/__snapshot__/ToJsonUDFTest_testEvaluateMultiArg.txt @@ -0,0 +1 @@ +{"0":{"key":{"age":10,"gender":"m"}},"1":100} \ No newline at end of file diff --git a/HiveUDF/src/test/resources/org/jsonex/hiveudf/__snapshot__/ToJsonUDFTest_testEvaluateSingleArg.txt b/HiveUDF/src/test/resources/org/jsonex/hiveudf/__snapshot__/ToJsonUDFTest_testEvaluateSingleArg.txt new file mode 100644 index 0000000..a7e1484 --- /dev/null +++ b/HiveUDF/src/test/resources/org/jsonex/hiveudf/__snapshot__/ToJsonUDFTest_testEvaluateSingleArg.txt @@ -0,0 +1 @@ +{"key":{"age":10,"gender":"m"}} \ No newline at end of file diff --git a/JSONCoder/pom.xml b/JSONCoder/pom.xml index 0b41e8b..3e64c9b 100644 --- a/JSONCoder/pom.xml +++ b/JSONCoder/pom.xml @@ -6,7 +6,7 @@ org.jsonex jcParent - 0.1.21 + 0.1.27 ../pom.xml JSONCoder diff --git a/JSONCoder/src/main/java/org/jsonex/jsoncoder/FieldSelectOption.java b/JSONCoder/src/main/java/org/jsonex/jsoncoder/FieldSelectOption.java new file mode 100644 index 0000000..04740ff --- /dev/null +++ b/JSONCoder/src/main/java/org/jsonex/jsoncoder/FieldSelectOption.java @@ -0,0 +1,18 @@ +package org.jsonex.jsoncoder; + +import lombok.Data; + +@Data +public class FieldSelectOption { + /** If true, when convert from a java bean, the readonly field will be ignored */ + boolean ignoreReadOnly; + + /** If true, subclass field won't be encoded */ + boolean ignoreSubClassFields; + + /** If true, for java bean type, only field include private will be returned, no setter getter method will be returned */ + boolean showPrivateField; + + /** by default, transientField won't be serialized. Set this to true will serialize it */ + boolean showTransientField; +} diff --git a/JSONCoder/src/main/java/org/jsonex/jsoncoder/JSONCoder.java b/JSONCoder/src/main/java/org/jsonex/jsoncoder/JSONCoder.java index 826bc3b..ee47c57 100644 --- a/JSONCoder/src/main/java/org/jsonex/jsoncoder/JSONCoder.java +++ b/JSONCoder/src/main/java/org/jsonex/jsoncoder/JSONCoder.java @@ -29,6 +29,9 @@ public class JSONCoder { public static JSONCoder get() { return getGlobal(); } + public static String stringify(Object obj) { return get().encode(obj); } + public static String stringify(Object obj, int indentFact) { return encode(obj, JSONCoderOption.ofIndentFactor(indentFact)); } + public static T parse(String str, Class cls) { return get().decode(str, cls); } @SuppressWarnings("unchecked") public static T decode(DecodeReq req, JSONCoderOption opt) { diff --git a/JSONCoder/src/main/java/org/jsonex/jsoncoder/JSONCoderOption.java b/JSONCoder/src/main/java/org/jsonex/jsoncoder/JSONCoderOption.java index 6a4b9a0..5008805 100644 --- a/JSONCoder/src/main/java/org/jsonex/jsoncoder/JSONCoderOption.java +++ b/JSONCoder/src/main/java/org/jsonex/jsoncoder/JSONCoderOption.java @@ -13,36 +13,54 @@ import lombok.Setter; import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; -import org.jsonex.core.factory.CacheThreadLocal; import org.jsonex.core.factory.InjectableFactory; +import org.jsonex.core.factory.ScopeThreadLocal; import org.jsonex.core.type.Tuple; import org.jsonex.core.type.Tuple.Pair; +import org.jsonex.core.type.Union.Union2; import org.jsonex.core.util.ClassUtil; -import org.jsonex.jsoncoder.coder.*; +import static org.jsonex.core.util.LangUtil.doIfElse; +import static org.jsonex.core.util.LangUtil.orElse; +import static org.jsonex.core.util.LangUtil.safe; +import static org.jsonex.core.util.ListUtil.listOf; +import org.jsonex.jsoncoder.coder.CoderAtomicBoolean; +import org.jsonex.jsoncoder.coder.CoderAtomicInteger; +import org.jsonex.jsoncoder.coder.CoderBigInteger; +import org.jsonex.jsoncoder.coder.CoderClass; +import org.jsonex.jsoncoder.coder.CoderDate; +import org.jsonex.jsoncoder.coder.CoderEnum; +import org.jsonex.jsoncoder.coder.CoderURI; +import org.jsonex.jsoncoder.coder.CoderURL; +import org.jsonex.jsoncoder.coder.CoderXMLGregorianCalendar; import org.jsonex.jsoncoder.fieldTransformer.FieldTransformer; import org.jsonex.jsoncoder.fieldTransformer.FieldTransformer.FieldInfo; +import static org.jsonex.jsoncoder.fieldTransformer.FieldTransformer.exclude; +import org.jsonex.treedoc.TDNode; import org.jsonex.treedoc.json.TDJSONOption; import org.slf4j.Logger; import java.text.Format; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.TimeZone; import java.util.concurrent.atomic.AtomicReference; -import static org.jsonex.core.util.LangUtil.*; -import static org.jsonex.core.util.ListUtil.listOf; -import static org.jsonex.jsoncoder.fieldTransformer.FieldTransformer.exclude; - @SuppressWarnings("UnusedReturnValue") @Accessors(chain=true) @Slf4j public class JSONCoderOption { @Getter final static JSONCoderOption global = new JSONCoderOption(null); static { global.addCoder(CoderDate.get(), CoderEnum.get(), CoderXMLGregorianCalendar.get(), CoderAtomicInteger.get(), - CoderBigInteger.get(), CoderClass.get(), CoderURI.get(), CoderURL.get()); + CoderAtomicBoolean.get(), CoderBigInteger.get(), CoderClass.get(), CoderURI.get(), CoderURL.get()); - global.addSkippedClasses(Format.class); + global.addSkippedClasses(Format.class, ClassLoader.class); global.fallbackDateFormats.add("yyyy-MM-dd HH:mm:ss.SSS.Z"); //Just for backward compatibility. global.fallbackDateFormats.add("yyyy/MM/dd HH:mm:ss.SSS.Z"); global.fallbackDateFormats.add("yyyy-MM-dd HH:mm:ss.SSS"); @@ -61,16 +79,6 @@ public class JSONCoderOption { @Getter @Setter int maxDepth = 30; @Getter @Setter int maxElementsPerNode = 2000; - /** - * If true, when convert from an java bean, the readonly field will be ignored - */ - @Getter @Setter boolean ignoreReadOnly; - - /** - * If true, subclass field won't be encoded - */ - @Getter @Setter boolean ignoreSubClassFields; - /** * If true, enum name will be encoded */ @@ -85,12 +93,7 @@ public class JSONCoderOption { * If true, duplicated object will be serialized as a reference to existing object's hash */ @Getter @Setter boolean dedupWithRef; - - /** - * If true, for java bean type, only field include private will be returned, no setter getter method will be returned. - */ - @Getter @Setter boolean showPrivateField; - + @Getter @Setter String parsingDateFormat = "yyyy-MM-dd'T'HH:mm:ss.SSSX"; /** * Used by {@link CoderDate}.encode()}, If Date format is null, date will be encoded as long with value of Date.getTime() @@ -102,7 +105,7 @@ public class JSONCoderOption { // For performance reason, we need to cache SimpleDateFormat in the same thread as SimpleDateFormat is not threadsafe private static final InjectableFactory._2 dateFormatCache = - InjectableFactory._2.of(JSONCoderOption::buildDateFormat, CacheThreadLocal.get()); + InjectableFactory._2.of(JSONCoderOption::buildDateFormat, ScopeThreadLocal.get()); public SimpleDateFormat getCachedParsingDateFormat() { return getCachedDateFormat(parsingDateFormat); } public SimpleDateFormat getCachedDateFormat() { return getCachedDateFormat(dateFormat); } @@ -135,6 +138,8 @@ private static SimpleDateFormat buildDateFormat(String format, TimeZone timeZone @Getter private final List, FieldTransformer>> filters = new ArrayList<>(); + @Getter private final List, String>, FieldSelectOption>> fieldSelectOptions = new ArrayList<>(); + @Getter FieldSelectOption defaultFieldSelectOption = new FieldSelectOption(); @Getter private final List> coderList = new ArrayList<>(); /** @@ -142,14 +147,14 @@ private static SimpleDateFormat buildDateFormat(String format, TimeZone timeZone * e.g. BO object as there are no proper implementation of equals and hashCode * which could cause duplicated copy of Object to be output. * - * The priority is based on the index of the wrapper. So if want to add highest priority - * need to use equalsWrapper.add(0, wrapper). + * The priority is based on the index of the wrapper. So to add for the highest priority + * it needs to use equalsWrapper.add(0, wrapper). * */ @Getter private final List> equalsWrapper = new ArrayList<>(); // JSON coder config - @Getter @Setter private TDJSONOption jsonOption = new TDJSONOption(); + @Getter @Setter private TDJSONOption jsonOption = TDJSONOption.ofDefaultRootType(TDNode.Type.SIMPLE); public enum LogLevel { OFF { public void log(Logger log, String msg, Throwable e) { /* Noop */ }}, @@ -163,21 +168,27 @@ public enum LogLevel { @Getter @Setter private LogLevel warnLogLevel = LogLevel.INFO; /** - * Accept specified sub-class using `$type` attribute. This feature is disabled by default for security reason + * Accept specified subclass using `$type` attribute. This feature is disabled by default for security reason */ @Getter @Setter private boolean allowPolymorphicClasses = false; /** - * Merge array. By default, when decode to exiting object, array or collection will be override instead of merge. + * Merge array. By default, when decode to exiting object, array or collection will be overridden instead of merge. * If this set true, it will merge the array (concatenation) */ @Getter @Setter private boolean mergeArray = false; /** * As Java Map and Set implementation, the order may not be strictly consistent cross JVM implementation - * set this to true, it will sort the keys in a predicated order + * set this to true, it will sort the keys in a deterministic order */ - @Getter @Setter private boolean strictOrder = false; + @Getter @Setter private boolean sortMapAndSet = false; + + /** + * In JVM implementation, Object getter method iteration order is not deterministic. Set this to true, it will + * order the object keys. It won't sort Map's key order, for map or set ordering, please use sortMapAndSet + */ + @Getter @Setter private boolean sortObjectKeys = false; public JSONCoderOption() { this(global); } private JSONCoderOption(JSONCoderOption parent) { this.parent = parent; } @@ -185,7 +196,11 @@ public enum LogLevel { public static JSONCoderOption ofIndentFactor(int factor) { return new JSONCoderOption().setJsonOption(TDJSONOption.ofIndentFactor(factor)); } - + + public JSONCoderOption setStrictOrdering(boolean value) { + return setSortMapAndSet(value).setSortObjectKeys(value); + } + ICoder findCoder(Class cls){ for (ICoder bc : coderList){ if(bc.getType().isAssignableFrom(cls)) @@ -210,10 +225,10 @@ public boolean isClassSkipped(Class cls) { public FieldInfo transformField(Class cls, FieldInfo fieldInfo, BeanCoderContext ctx) { for (Pair, FieldTransformer> filter : filters) { - if (!filter._1.isAssignableFrom(cls)) + if (!filter._0.isAssignableFrom(cls)) continue; // TODO: Fix when to stop the filter chain strategy - fieldInfo = filter._2.apply(fieldInfo, ctx); + fieldInfo = filter._1.apply(fieldInfo, ctx); } return parent == null ? fieldInfo : parent.transformField(cls, fieldInfo, ctx); @@ -221,9 +236,9 @@ public FieldInfo transformField(Class cls, FieldInfo fieldInfo, BeanCoderCont public boolean isExcluded(Class cls, String name, BeanCoderContext ctx) { for (Pair, FieldTransformer> filter : filters) { - if (!filter._1.isAssignableFrom(cls)) + if (!filter._0.isAssignableFrom(cls)) continue; - if (!filter._2.shouldInclude(name, ctx)) + if (!filter._1.shouldInclude(name, ctx)) return true; } return parent == null ? false : parent.isExcluded(cls, name, ctx); @@ -250,16 +265,6 @@ public Date parseDateFullback(String dateStr) throws ParseException { throw exp == null ? new ParseException(dateStr, 0) : exp; return parent.parseDateFullback(dateStr); } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - public boolean isIgnoreSubClassFields(Class cls){ - if(ignoreSubClassFields) - return true; - for(Class iCls : ignoreSubClassFieldsClasses) - if(iCls.isAssignableFrom(cls)) - return true; - return parent != null && parent.isIgnoreSubClassFields(cls); - } public JSONCoderOption addDefaultFilter(FieldTransformer filter) { return addFilterFor(Object.class, filter, false); @@ -306,10 +311,58 @@ public JSONCoderOption addCoder(ICoder... codes) { return this; } + // Keep this for backward compatible public JSONCoderOption setJsonOption(boolean alwaysQuoteName, char quoteChar, int indentFactor) { - jsonOption.setAlwaysQuoteName(alwaysQuoteName) - .setQuoteChar(quoteChar) + return setJsonOption(alwaysQuoteName, String.valueOf(quoteChar), indentFactor); + } + + public JSONCoderOption setJsonOption(boolean alwaysQuoteName, String quoteChars, int indentFactor) { + jsonOption.setAlwaysQuoteKey(alwaysQuoteName) + .setQuoteChars(quoteChars) .setIndentFactor(indentFactor); return this; } + + public FieldSelectOption getFieldSelectOption(Class cls) { + for (Pair, String>, FieldSelectOption> opt : fieldSelectOptions) { + if (matches(opt._0, cls)) + return opt._1; + } + return defaultFieldSelectOption; + } + + private static boolean matches(Union2, String> key, Class cls) { + if (key._0 != null) + return key._0.isAssignableFrom(cls); + // Package + String pkg = key._1; + String clsPkg = cls.getPackage().getName(); + // TODO: optimize to avoid create lots of string object with substring + return pkg.endsWith("*") ? clsPkg.startsWith(pkg.substring(0, pkg.length() - 1)) : clsPkg == pkg; + } + + public JSONCoderOption addFieldSelectOptionFor(Class cls, FieldSelectOption filter) { + return addFieldSelectOptionFor(cls, filter, false); + } + + public JSONCoderOption addFieldSelectOptionFor(Class cls, FieldSelectOption opt, boolean last) { + Pair, String>, FieldSelectOption> clsOpt = Pair.of(Union2.of_0(cls), opt); + doIfElse(last, () -> fieldSelectOptions.add(clsOpt), () -> fieldSelectOptions.add(0, clsOpt)); + return this; + } + + public JSONCoderOption addFieldSelectOptionForPackage(String pkg, FieldSelectOption filter) { + return addFieldSelectOptionForPackage(pkg, filter, false); + } + + public JSONCoderOption addFieldSelectOptionForPackage(String pkg, FieldSelectOption opt, boolean last) { + Pair, String>, FieldSelectOption> clsOpt = Pair.of(Union2.of_1(pkg), opt); + doIfElse(last, () -> fieldSelectOptions.add(clsOpt), () -> fieldSelectOptions.add(0, clsOpt)); + return this; + } + + public JSONCoderOption setIgnoreReadOnly(boolean v) { defaultFieldSelectOption.setIgnoreReadOnly(v); return this;} + public JSONCoderOption setShowPrivateField(boolean v) { defaultFieldSelectOption.setShowPrivateField(v); return this;} + public JSONCoderOption setShowTransientField(boolean v) { defaultFieldSelectOption.setShowTransientField(v); return this;} + public JSONCoderOption setIgnoreSubClassFields(boolean v) { defaultFieldSelectOption.setIgnoreSubClassFields(v); return this;} } diff --git a/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderArray.java b/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderArray.java index 933882d..87cab2c 100644 --- a/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderArray.java +++ b/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderArray.java @@ -31,6 +31,8 @@ public class CoderArray implements ICoder { public TDNode encode(Object obj, Type type, BeanCoderContext ctx, TDNode target) { target.setType(TDNode.Type.ARRAY); Class cls = ClassUtil.getGenericClass(type); + if (cls == null) + cls = obj.getClass(); for (int i = 0; i < Array.getLength(obj) && i < ctx.getOption().getMaxElementsPerNode(); i++) ctx.encode(Array.get(obj, i), cls.getComponentType(), target.createChild()); return target; diff --git a/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderAtomicBoolean.java b/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderAtomicBoolean.java new file mode 100644 index 0000000..9f335e4 --- /dev/null +++ b/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderAtomicBoolean.java @@ -0,0 +1,31 @@ +/************************************************************* + Copyright 2018-2019 eBay Inc. + Author/Developer: Jianwu Chen + + Use of this source code is governed by an MIT-style + license that can be found in the LICENSE file or at + https://opensource.org/licenses/MIT. + ************************************************************/ + +package org.jsonex.jsoncoder.coder; + +import org.jsonex.core.factory.InjectableInstance; +import org.jsonex.jsoncoder.BeanCoderContext; +import org.jsonex.jsoncoder.ICoder; +import org.jsonex.treedoc.TDNode; + +import java.lang.reflect.Type; +import java.util.concurrent.atomic.AtomicBoolean; + +public class CoderAtomicBoolean implements ICoder { + public static final InjectableInstance it = InjectableInstance.of(CoderAtomicBoolean.class); + public static CoderAtomicBoolean get() { return it.get(); } + + @Override public Class getType() {return AtomicBoolean.class;} + + @Override public TDNode encode(AtomicBoolean obj, Type type, BeanCoderContext context, TDNode target) { return target.setValue(obj.get()); } + + @Override public AtomicBoolean decode(TDNode tdNode, Type type, Object targetObj, BeanCoderContext context) { + return new AtomicBoolean((boolean)tdNode.getValue()); + } +} \ No newline at end of file diff --git a/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderCollection.java b/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderCollection.java index 454440b..5d518cb 100644 --- a/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderCollection.java +++ b/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderCollection.java @@ -31,7 +31,7 @@ public class CoderCollection implements ICoder { @Override public TDNode encode(Collection obj, Type type, BeanCoderContext ctx, TDNode target) { target.setType(TDNode.Type.ARRAY); - if (ctx.getOption().isStrictOrder() + if (ctx.getOption().isSortMapAndSet() && obj instanceof Set && !(obj instanceof SortedSet || obj instanceof LinkedHashSet || obj instanceof EnumSet)) { Set set = new TreeSet(FullbackComparator.it); // Due to instability of Set iteration order, we copy it to TreeSet to make iteration stable set.addAll(obj); diff --git a/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderMap.java b/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderMap.java index 0fa3c99..3f3d7e3 100644 --- a/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderMap.java +++ b/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderMap.java @@ -40,7 +40,7 @@ public class CoderMap implements ICoder { JSONCoderOption opt = ctx.getOption(); Map map = (Map)obj; - if (opt.isStrictOrder() + if (opt.isSortMapAndSet() && !(map instanceof SortedMap) && ! (map instanceof LinkedHashMap) && ! (map instanceof EnumMap)) { TreeMap treeMap = new TreeMap<>(FullbackComparator.it); // Due to instability of map iterator, we copy it to TreeMap to make it in stable order. treeMap.putAll(map); diff --git a/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderObject.java b/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderObject.java index ba34e41..3e8e644 100644 --- a/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderObject.java +++ b/JSONCoder/src/main/java/org/jsonex/jsoncoder/coder/CoderObject.java @@ -15,10 +15,7 @@ import org.jsonex.core.util.BeanProperty; import org.jsonex.core.util.ClassUtil; import org.jsonex.core.util.StringUtil; -import org.jsonex.jsoncoder.BeanCoderContext; -import org.jsonex.jsoncoder.BeanCoderException; -import org.jsonex.jsoncoder.ICoder; -import org.jsonex.jsoncoder.JSONCoderOption; +import org.jsonex.jsoncoder.*; import org.jsonex.jsoncoder.fieldTransformer.FieldTransformer.FieldInfo; import org.jsonex.treedoc.TDNode; import org.jsonex.treedoc.json.TDJSONWriter; @@ -30,6 +27,7 @@ import java.util.Date; import java.util.Map; import java.util.Objects; +import java.util.TreeMap; import java.util.function.Function; import static org.jsonex.core.util.LangUtil.getIfInstanceOf; @@ -49,21 +47,24 @@ public class CoderObject implements ICoder { JSONCoderOption opt = ctx.getOption(); Class cls = obj.getClass(); // Use the real object; - if (opt.isIgnoreSubClassFields(cls) && type != null) + FieldSelectOption selectOpt = opt.getFieldSelectOption(cls); + if (selectOpt.isIgnoreSubClassFields() && type != null) cls = ClassUtil.getGenericClass(type); if (opt.isShowType() || cls != obj.getClass()) target.createChild(TYPE_KEY).setValue(obj.getClass().getName()); Map pds = ClassUtil.getProperties(cls); + if (opt.isSortObjectKeys()) + pds = new TreeMap<>(pds); for (BeanProperty pd : pds.values()) { - if (!pd.isReadable(opt.isShowPrivateField())) + if (!pd.isReadable(selectOpt.isShowPrivateField())) continue; - if (pd.isImmutable(opt.isShowPrivateField()) && opt.isIgnoreReadOnly()) + if (pd.isImmutable(selectOpt.isShowPrivateField()) && selectOpt.isIgnoreReadOnly()) continue; // Only mutable attribute will be encoded - if (pd.isTransient()) + if (pd.isTransient() && !selectOpt.isShowTransientField()) continue; if (opt.isExcluded(cls, pd.getName(), ctx)) diff --git a/JSONCoder/src/test/java/org/jsonex/jsoncoder/JSONCoderTest.java b/JSONCoder/src/test/java/org/jsonex/jsoncoder/JSONCoderTest.java index 8af349f..6041825 100644 --- a/JSONCoder/src/test/java/org/jsonex/jsoncoder/JSONCoderTest.java +++ b/JSONCoder/src/test/java/org/jsonex/jsoncoder/JSONCoderTest.java @@ -43,7 +43,8 @@ public class JSONCoderTest { private static String toJSONString(Object obj, JSONCoderOption option) { return JSONCoder.encode(obj, option); } private static String toJSONString(Object obj) { - return JSONCoder.global.encode(obj, JSONCoderOption.ofIndentFactor(2).setWarnLogLevel(JSONCoderOption.LogLevel.DEBUG)); + return JSONCoder.global.encode(obj, JSONCoderOption.ofIndentFactor(2). + setStrictOrdering(true).setWarnLogLevel(JSONCoderOption.LogLevel.DEBUG)); } @Before public void before() { @@ -123,10 +124,14 @@ private TestBean buildTestBean() { assertEquals(str, toJSONString(tb1)); } + @Test public void testEncodeArray() { + assertEquals("[1,2,3]", JSONCoder.get().encode(new int[]{1,2,3})); + } + @Test public void testCyclicReference() { TestBean tb = new TestBean().setBean2(new TestBean2()); tb.getBean2().testBean = tb; - String str = toJSONString(tb, new JSONCoderOption().setJsonOption(false, '`', 2)); + String str = toJSONString(tb, new JSONCoderOption().setJsonOption(false, '`', 2).setStrictOrdering(true)); assertMatchesSnapshot(str); assertTrue("Cyclic reference should be encoded as $ref:str=" + str, str.indexOf("testBean:{\n $ref:`../../`\n") > 0); @@ -140,7 +145,8 @@ private TestBean buildTestBean() { tb.bean2List = Collections.singletonList(tb2); tb2.setInts(tb.getInts()); // Duplicated arrays - String str = toJSONString(tb, JSONCoderOption.of().setDedupWithRef(true).setJsonOption(false, '"', 2)); + String str = toJSONString(tb, JSONCoderOption.of().setDedupWithRef(true) + .setJsonOption(false, '"', 2).setStrictOrdering(true)); assertMatchesSnapshot(str); assertTrue("Generate ref if dedupWithRef is set", str.contains("$ref")); @@ -150,7 +156,7 @@ private TestBean buildTestBean() { } @Test public void testEnumNameOption() { - JSONCoderOption codeOption = JSONCoderOption.ofIndentFactor(2).setShowEnumName(true); + JSONCoderOption codeOption = JSONCoderOption.ofIndentFactor(2).setShowEnumName(true).setStrictOrdering(true); String str = toJSONString(buildTestBean(), codeOption); assertMatchesSnapshot(str); assertTrue("Should contain both enum id and name when showEnumName is set", str.indexOf("12345-value1") > 0); @@ -158,7 +164,7 @@ private TestBean buildTestBean() { } @Test public void testCustomQuote() { - JSONCoderOption codeOption = JSONCoderOption.ofIndentFactor(2); + JSONCoderOption codeOption = JSONCoderOption.ofIndentFactor(2).setStrictOrdering(true); codeOption.getJsonOption().setQuoteChar('\''); String str = toJSONString(buildTestBean(), codeOption); assertMatchesSnapshot("strWithSingleQuote", str); @@ -166,7 +172,7 @@ private TestBean buildTestBean() { str.indexOf("'String1: \\'\"'") > 0); assertEquals(toJSONString(JSONCoder.global.decode(str, TestBean.class), codeOption), str); - codeOption.getJsonOption().setAlwaysQuoteName(false); // Make quote optional for attribute names + codeOption.getJsonOption().setAlwaysQuoteKey(false); // Make quote optional for attribute names str = toJSONString(buildTestBean(), codeOption); assertMatchesSnapshot("strWithNoKeyQuote", str); @@ -348,7 +354,7 @@ private SimpleDateFormat buildDateFormat(String format) { bean.testBean.setStrField("str2"); bean.testBean.publicStrField = "publicStr"; - JSONCoderOption opt = JSONCoderOption.ofIndentFactor(2); + JSONCoderOption opt = JSONCoderOption.ofIndentFactor(2).setStrictOrdering(true); opt.addFilterFor(TestBean2.class, include("ints", "enumField2", "testBean", "strField")); @@ -487,21 +493,25 @@ static class ClsWithTypeVar { ClsWithTypeVar bean = new ClsWithTypeVar(); bean.listOfMap.add(new MapBuilder("str1", 1).getMap()); bean.refOfMap.set(new MapBuilder("str1", 1).getMap()); - String json = JSONCoder.global.encode(bean); + String json = toJSONString(bean); assertMatchesSnapshot(json); ClsWithTypeVar bean1 = JSONCoder.global.decode(json, ClsWithTypeVar.class); - String json1 = JSONCoder.global.encode(bean1); + String json1 = toJSONString(bean1); assertEquals(json, json1); } - @Test public void testStrictOrder() { + @Test public void testSortMapAndSet() { Set set = new HashSet<>(listOf(new NoneComparable("a"), new NoneComparable("b"), new NoneComparable("c"), new NoneComparable("d"))); - assertMatchesSnapshot("set", JSONCoder.encode(set, JSONCoderOption.of().setStrictOrder(true))); + assertMatchesSnapshot("set", JSONCoder.encode(set, JSONCoderOption.of().setStrictOrdering(true))); Map map = new HashMap<>(); map.put(new NoneComparable("a"), "value 1"); map.put(new NoneComparable("b"), "value 2"); - assertMatchesSnapshot("map", JSONCoder.encode(map, JSONCoderOption.of().setStrictOrder(true))); + assertMatchesSnapshot("map", JSONCoder.encode(map, JSONCoderOption.of().setStrictOrdering(true))); + } + + @Test public void testSortObjectKey() { + assertMatchesSnapshot(JSONCoder.encode(buildTestBean(), JSONCoderOption.of().setStrictOrdering(true))); } private void expectDecodeWithException(String str, Class cls, String expectedError) { diff --git a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testBasicEncoding.txt b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testBasicEncoding.txt index 9685d98..a43483e 100644 --- a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testBasicEncoding.txt +++ b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testBasicEncoding.txt @@ -1,75 +1,75 @@ { - "fieldInBaseClass":"Overridden Value", - "treeMap":{ - "key1":"value1" - }, - "linkedList1":[ - "value1" - ], - "intField":100, - "floatField":1.4, - "charField":"A", - "booleanField":false, - "strField":"String1: '\"", - "dateField":"1970-01-01T03:23:32.121Z", + "atomicInteger":1001, "bean2":{ "enumField":"value2", "enumField2":12345, - "strEnum":"strEnumV1", + "enumMap":{ + "value1":"MapValue1" + }, "enumSet":[ "value1", "value2" ], - "enumMap":{ - "value1":"MapValue1" - }, + "strEnum":"strEnumV1", "testBean":{ "$ref":"../../" } }, - "ints":[ - 4, - 3, - 2, - 1 + "bean2List":[ + { + "enumMap":{}, + "enumSet":[ + "value1", + "value2" + ], + "strField":"AAA" + }, + { + "enumMap":{}, + "enumSet":[ + "value1", + "value2" + ], + "strField":"BBB" + } ], "bean2s":[ { - "strField":"1", + "enumMap":{}, "enumSet":[ "value1", "value2" ], - "enumMap":{} + "strField":"1" }, { - "strField":"2", + "enumMap":{}, "enumSet":[ "value1", "value2" ], - "enumMap":{} + "strField":"2" } ], - "readonlyField":"This's a readonly field", - "atomicInteger":1001, "bigInteger":"123456789012345678901234567890", - "someClass":"java.util.Date", - "xmlCalendar":"1970-01-01T00:00:00.000Z", + "booleanField":false, + "charField":"A", + "dateField":"1970-01-01T03:23:32.121Z", "dateNumberMap":[ null ], - "publicLinkedList":[ + "fieldInBaseClass":"Overridden Value", + "floatField":1.4, + "intField":100, + "ints":[ + 4, + 3, + 2, + 1 + ], + "linkedList1":[ "value1" ], - "publicTreeMap":{ - "key1":"value1" - }, - "publicStrField":"PublicString", - "publicMap":{ - "key1":"1970-01-01T00:00:00.000Z", - "key2":"1970-01-01T00:20:12.121Z" - }, "publicBigDecimal1":"123456789012345678901234567", "publicBigDecimal2":"12", "publicInts":[ @@ -78,26 +78,26 @@ 2, 1 ], - "bean2List":[ - { - "strField":"AAA", - "enumSet":[ - "value1", - "value2" - ], - "enumMap":{} - }, - { - "strField":"BBB", - "enumSet":[ - "value1", - "value2" - ], - "enumMap":{} - } + "publicLinkedList":[ + "value1" ], + "publicMap":{ + "key1":"1970-01-01T00:00:00.000Z", + "key2":"1970-01-01T00:20:12.121Z" + }, + "publicStrField":"PublicString", "publicStringSet":[ "str1", "str2" - ] + ], + "publicTreeMap":{ + "key1":"value1" + }, + "readonlyField":"This's a readonly field", + "someClass":"java.util.Date", + "strField":"String1: '\"", + "treeMap":{ + "key1":"value1" + }, + "xmlCalendar":"1970-01-01T00:00:00.000Z" } \ No newline at end of file diff --git a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testBasicEncoding_testBean.toString.txt b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testBasicEncoding_testBean.toString.txt new file mode 100644 index 0000000..39a75fa --- /dev/null +++ b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testBasicEncoding_testBean.toString.txt @@ -0,0 +1 @@ +org.jsonex.jsoncoder.TestBean@6a396c1e \ No newline at end of file diff --git a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCustomQuote_strWithBackQuote.txt b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCustomQuote_strWithBackQuote.txt index 4f464f4..b030d7c 100644 --- a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCustomQuote_strWithBackQuote.txt +++ b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCustomQuote_strWithBackQuote.txt @@ -1,75 +1,75 @@ { - fieldInBaseClass:`Overridden Value`, - treeMap:{ - key1:`value1` - }, - linkedList1:[ - `value1` - ], - intField:100, - floatField:1.4, - charField:`A`, - booleanField:false, - strField:`String1: '"`, - dateField:`1970-01-01T03:23:32.121Z`, + atomicInteger:1001, bean2:{ enumField:`value2`, enumField2:12345, - strEnum:`strEnumV1`, + enumMap:{ + value1:`MapValue1` + }, enumSet:[ `value1`, `value2` ], - enumMap:{ - value1:`MapValue1` - }, + strEnum:`strEnumV1`, testBean:{ $ref:`../../` } }, - ints:[ - 4, - 3, - 2, - 1 + bean2List:[ + { + enumMap:{}, + enumSet:[ + `value1`, + `value2` + ], + strField:`AAA` + }, + { + enumMap:{}, + enumSet:[ + `value1`, + `value2` + ], + strField:`BBB` + } ], bean2s:[ { - strField:`1`, + enumMap:{}, enumSet:[ `value1`, `value2` ], - enumMap:{} + strField:`1` }, { - strField:`2`, + enumMap:{}, enumSet:[ `value1`, `value2` ], - enumMap:{} + strField:`2` } ], - readonlyField:`This's a readonly field`, - atomicInteger:1001, bigInteger:`123456789012345678901234567890`, - someClass:`java.util.Date`, - xmlCalendar:`1970-01-01T00:00:00.000Z`, + booleanField:false, + charField:`A`, + dateField:`1970-01-01T03:23:32.121Z`, dateNumberMap:[ null ], - publicLinkedList:[ + fieldInBaseClass:`Overridden Value`, + floatField:1.4, + intField:100, + ints:[ + 4, + 3, + 2, + 1 + ], + linkedList1:[ `value1` ], - publicTreeMap:{ - key1:`value1` - }, - publicStrField:`PublicString`, - publicMap:{ - key1:`1970-01-01T00:00:00.000Z`, - key2:`1970-01-01T00:20:12.121Z` - }, publicBigDecimal1:`123456789012345678901234567`, publicBigDecimal2:`12`, publicInts:[ @@ -78,26 +78,26 @@ 2, 1 ], - bean2List:[ - { - strField:`AAA`, - enumSet:[ - `value1`, - `value2` - ], - enumMap:{} - }, - { - strField:`BBB`, - enumSet:[ - `value1`, - `value2` - ], - enumMap:{} - } + publicLinkedList:[ + `value1` ], + publicMap:{ + key1:`1970-01-01T00:00:00.000Z`, + key2:`1970-01-01T00:20:12.121Z` + }, + publicStrField:`PublicString`, publicStringSet:[ `str1`, `str2` - ] + ], + publicTreeMap:{ + key1:`value1` + }, + readonlyField:`This's a readonly field`, + someClass:`java.util.Date`, + strField:`String1: '"`, + treeMap:{ + key1:`value1` + }, + xmlCalendar:`1970-01-01T00:00:00.000Z` } \ No newline at end of file diff --git a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCustomQuote_strWithNoKeyQuote.txt b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCustomQuote_strWithNoKeyQuote.txt index 16eb0a8..7734957 100644 --- a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCustomQuote_strWithNoKeyQuote.txt +++ b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCustomQuote_strWithNoKeyQuote.txt @@ -1,75 +1,75 @@ { - fieldInBaseClass:'Overridden Value', - treeMap:{ - key1:'value1' - }, - linkedList1:[ - 'value1' - ], - intField:100, - floatField:1.4, - charField:'A', - booleanField:false, - strField:'String1: \'"', - dateField:'1970-01-01T03:23:32.121Z', + atomicInteger:1001, bean2:{ enumField:'value2', enumField2:12345, - strEnum:'strEnumV1', + enumMap:{ + value1:'MapValue1' + }, enumSet:[ 'value1', 'value2' ], - enumMap:{ - value1:'MapValue1' - }, + strEnum:'strEnumV1', testBean:{ $ref:'../../' } }, - ints:[ - 4, - 3, - 2, - 1 + bean2List:[ + { + enumMap:{}, + enumSet:[ + 'value1', + 'value2' + ], + strField:'AAA' + }, + { + enumMap:{}, + enumSet:[ + 'value1', + 'value2' + ], + strField:'BBB' + } ], bean2s:[ { - strField:'1', + enumMap:{}, enumSet:[ 'value1', 'value2' ], - enumMap:{} + strField:'1' }, { - strField:'2', + enumMap:{}, enumSet:[ 'value1', 'value2' ], - enumMap:{} + strField:'2' } ], - readonlyField:'This\'s a readonly field', - atomicInteger:1001, bigInteger:'123456789012345678901234567890', - someClass:'java.util.Date', - xmlCalendar:'1970-01-01T00:00:00.000Z', + booleanField:false, + charField:'A', + dateField:'1970-01-01T03:23:32.121Z', dateNumberMap:[ null ], - publicLinkedList:[ + fieldInBaseClass:'Overridden Value', + floatField:1.4, + intField:100, + ints:[ + 4, + 3, + 2, + 1 + ], + linkedList1:[ 'value1' ], - publicTreeMap:{ - key1:'value1' - }, - publicStrField:'PublicString', - publicMap:{ - key1:'1970-01-01T00:00:00.000Z', - key2:'1970-01-01T00:20:12.121Z' - }, publicBigDecimal1:'123456789012345678901234567', publicBigDecimal2:'12', publicInts:[ @@ -78,26 +78,26 @@ 2, 1 ], - bean2List:[ - { - strField:'AAA', - enumSet:[ - 'value1', - 'value2' - ], - enumMap:{} - }, - { - strField:'BBB', - enumSet:[ - 'value1', - 'value2' - ], - enumMap:{} - } + publicLinkedList:[ + 'value1' ], + publicMap:{ + key1:'1970-01-01T00:00:00.000Z', + key2:'1970-01-01T00:20:12.121Z' + }, + publicStrField:'PublicString', publicStringSet:[ 'str1', 'str2' - ] + ], + publicTreeMap:{ + key1:'value1' + }, + readonlyField:'This\'s a readonly field', + someClass:'java.util.Date', + strField:'String1: \'"', + treeMap:{ + key1:'value1' + }, + xmlCalendar:'1970-01-01T00:00:00.000Z' } \ No newline at end of file diff --git a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCustomQuote_strWithSingleQuote.txt b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCustomQuote_strWithSingleQuote.txt index a9b1e5a..5c2809e 100644 --- a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCustomQuote_strWithSingleQuote.txt +++ b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCustomQuote_strWithSingleQuote.txt @@ -1,75 +1,75 @@ { - 'fieldInBaseClass':'Overridden Value', - 'treeMap':{ - 'key1':'value1' - }, - 'linkedList1':[ - 'value1' - ], - 'intField':100, - 'floatField':1.4, - 'charField':'A', - 'booleanField':false, - 'strField':'String1: \'"', - 'dateField':'1970-01-01T03:23:32.121Z', + 'atomicInteger':1001, 'bean2':{ 'enumField':'value2', 'enumField2':12345, - 'strEnum':'strEnumV1', + 'enumMap':{ + 'value1':'MapValue1' + }, 'enumSet':[ 'value1', 'value2' ], - 'enumMap':{ - 'value1':'MapValue1' - }, + 'strEnum':'strEnumV1', 'testBean':{ '$ref':'../../' } }, - 'ints':[ - 4, - 3, - 2, - 1 + 'bean2List':[ + { + 'enumMap':{}, + 'enumSet':[ + 'value1', + 'value2' + ], + 'strField':'AAA' + }, + { + 'enumMap':{}, + 'enumSet':[ + 'value1', + 'value2' + ], + 'strField':'BBB' + } ], 'bean2s':[ { - 'strField':'1', + 'enumMap':{}, 'enumSet':[ 'value1', 'value2' ], - 'enumMap':{} + 'strField':'1' }, { - 'strField':'2', + 'enumMap':{}, 'enumSet':[ 'value1', 'value2' ], - 'enumMap':{} + 'strField':'2' } ], - 'readonlyField':'This\'s a readonly field', - 'atomicInteger':1001, 'bigInteger':'123456789012345678901234567890', - 'someClass':'java.util.Date', - 'xmlCalendar':'1970-01-01T00:00:00.000Z', + 'booleanField':false, + 'charField':'A', + 'dateField':'1970-01-01T03:23:32.121Z', 'dateNumberMap':[ null ], - 'publicLinkedList':[ + 'fieldInBaseClass':'Overridden Value', + 'floatField':1.4, + 'intField':100, + 'ints':[ + 4, + 3, + 2, + 1 + ], + 'linkedList1':[ 'value1' ], - 'publicTreeMap':{ - 'key1':'value1' - }, - 'publicStrField':'PublicString', - 'publicMap':{ - 'key1':'1970-01-01T00:00:00.000Z', - 'key2':'1970-01-01T00:20:12.121Z' - }, 'publicBigDecimal1':'123456789012345678901234567', 'publicBigDecimal2':'12', 'publicInts':[ @@ -78,26 +78,26 @@ 2, 1 ], - 'bean2List':[ - { - 'strField':'AAA', - 'enumSet':[ - 'value1', - 'value2' - ], - 'enumMap':{} - }, - { - 'strField':'BBB', - 'enumSet':[ - 'value1', - 'value2' - ], - 'enumMap':{} - } + 'publicLinkedList':[ + 'value1' ], + 'publicMap':{ + 'key1':'1970-01-01T00:00:00.000Z', + 'key2':'1970-01-01T00:20:12.121Z' + }, + 'publicStrField':'PublicString', 'publicStringSet':[ 'str1', 'str2' - ] + ], + 'publicTreeMap':{ + 'key1':'value1' + }, + 'readonlyField':'This\'s a readonly field', + 'someClass':'java.util.Date', + 'strField':'String1: \'"', + 'treeMap':{ + 'key1':'value1' + }, + 'xmlCalendar':'1970-01-01T00:00:00.000Z' } \ No newline at end of file diff --git a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCyclicReference.txt b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCyclicReference.txt index 5ad08a6..532d8c7 100644 --- a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCyclicReference.txt +++ b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testCyclicReference.txt @@ -1,23 +1,23 @@ { - fieldInBaseClass:`Overridden Value`, - treeMap:{}, - linkedList1:[], - intField:0, - floatField:0.0, - charField:`\u0000`, - booleanField:false, + atomicInteger:100, bean2:{ + enumMap:{}, enumSet:[ `value1`, `value2` ], - enumMap:{}, testBean:{ $ref:`../../` } }, - readonlyField:`This's a readonly field`, - atomicInteger:100, + booleanField:false, + charField:`\u0000`, + fieldInBaseClass:`Overridden Value`, + floatField:0.0, + intField:0, + linkedList1:[], publicLinkedList:[], - publicTreeMap:{} + publicTreeMap:{}, + readonlyField:`This's a readonly field`, + treeMap:{} } \ No newline at end of file diff --git a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testDedupWithRef.txt b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testDedupWithRef.txt index b49b50e..4ac9ddf 100644 --- a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testDedupWithRef.txt +++ b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testDedupWithRef.txt @@ -1,34 +1,34 @@ { - fieldInBaseClass:"Overridden Value", - treeMap:{}, - linkedList1:[], - intField:0, - floatField:0.0, - charField:"\u0000", - booleanField:false, + atomicInteger:100, bean2:{ + enumMap:{}, + enumSet:[ + "value1", + "value2" + ], ints:[ 1, 2, 3 ], - enumSet:[ - "value1", - "value2" - ], - enumMap:{}, $id:1 }, + bean2List:[ + { + $ref:"#1" + } + ], + booleanField:false, + charField:"\u0000", + fieldInBaseClass:"Overridden Value", + floatField:0.0, + intField:0, ints:{ $ref:"/bean2/ints" }, - readonlyField:"This's a readonly field", - atomicInteger:100, + linkedList1:[], publicLinkedList:[], publicTreeMap:{}, - bean2List:[ - { - $ref:"#1" - } - ] + readonlyField:"This's a readonly field", + treeMap:{} } \ No newline at end of file diff --git a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testEnumNameOption.txt b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testEnumNameOption.txt index c741755..04231d2 100644 --- a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testEnumNameOption.txt +++ b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testEnumNameOption.txt @@ -1,75 +1,75 @@ { - "fieldInBaseClass":"Overridden Value", - "treeMap":{ - "key1":"value1" - }, - "linkedList1":[ - "value1" - ], - "intField":100, - "floatField":1.4, - "charField":"A", - "booleanField":false, - "strField":"String1: '\"", - "dateField":"1970-01-01T03:23:32.121Z", + "atomicInteger":1001, "bean2":{ "enumField":"value2", "enumField2":"12345-value1", - "strEnum":"strEnumV1-value1", + "enumMap":{ + "value1":"MapValue1" + }, "enumSet":[ "value1", "value2" ], - "enumMap":{ - "value1":"MapValue1" - }, + "strEnum":"strEnumV1-value1", "testBean":{ "$ref":"../../" } }, - "ints":[ - 4, - 3, - 2, - 1 + "bean2List":[ + { + "enumMap":{}, + "enumSet":[ + "value1", + "value2" + ], + "strField":"AAA" + }, + { + "enumMap":{}, + "enumSet":[ + "value1", + "value2" + ], + "strField":"BBB" + } ], "bean2s":[ { - "strField":"1", + "enumMap":{}, "enumSet":[ "value1", "value2" ], - "enumMap":{} + "strField":"1" }, { - "strField":"2", + "enumMap":{}, "enumSet":[ "value1", "value2" ], - "enumMap":{} + "strField":"2" } ], - "readonlyField":"This's a readonly field", - "atomicInteger":1001, "bigInteger":"123456789012345678901234567890", - "someClass":"java.util.Date", - "xmlCalendar":"1970-01-01T00:00:00.000Z", + "booleanField":false, + "charField":"A", + "dateField":"1970-01-01T03:23:32.121Z", "dateNumberMap":[ null ], - "publicLinkedList":[ + "fieldInBaseClass":"Overridden Value", + "floatField":1.4, + "intField":100, + "ints":[ + 4, + 3, + 2, + 1 + ], + "linkedList1":[ "value1" ], - "publicTreeMap":{ - "key1":"value1" - }, - "publicStrField":"PublicString", - "publicMap":{ - "key1":"1970-01-01T00:00:00.000Z", - "key2":"1970-01-01T00:20:12.121Z" - }, "publicBigDecimal1":"123456789012345678901234567", "publicBigDecimal2":"12", "publicInts":[ @@ -78,26 +78,26 @@ 2, 1 ], - "bean2List":[ - { - "strField":"AAA", - "enumSet":[ - "value1", - "value2" - ], - "enumMap":{} - }, - { - "strField":"BBB", - "enumSet":[ - "value1", - "value2" - ], - "enumMap":{} - } + "publicLinkedList":[ + "value1" ], + "publicMap":{ + "key1":"1970-01-01T00:00:00.000Z", + "key2":"1970-01-01T00:20:12.121Z" + }, + "publicStrField":"PublicString", "publicStringSet":[ "str1", "str2" - ] + ], + "publicTreeMap":{ + "key1":"value1" + }, + "readonlyField":"This's a readonly field", + "someClass":"java.util.Date", + "strField":"String1: '\"", + "treeMap":{ + "key1":"value1" + }, + "xmlCalendar":"1970-01-01T00:00:00.000Z" } \ No newline at end of file diff --git a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testFieldWithTypeVariable.txt b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testFieldWithTypeVariable.txt index 443b67d..5d09342 100644 --- a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testFieldWithTypeVariable.txt +++ b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testFieldWithTypeVariable.txt @@ -1 +1,13 @@ -{"listOfMap":[{"str1":1}],"refOfMap":{"^":{"str1":1}},"value":"abc"} \ No newline at end of file +{ + "listOfMap":[ + { + "str1":1 + } + ], + "refOfMap":{ + "^":{ + "str1":1 + } + }, + "value":"abc" +} \ No newline at end of file diff --git a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testFilter.txt b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testFilter.txt index f5e6a1c..80ade9f 100644 --- a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testFilter.txt +++ b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testFilter.txt @@ -1,22 +1,22 @@ { - "strField":"str1", + "enumField2":12345, "ints":[ 1, 2 ], - "enumField2":12345, + "strField":"str1", "testBean":{ + "atomicInteger":100, + "booleanField":false, + "charField":"\u0000", "fieldInBaseClass":"Overridden Value", - "treeMap":{}, - "linkedList1":[], - "intField":0, "floatField":0.0, - "charField":"\u0000", - "booleanField":false, - "strField":"str2", - "readonlyField":"This's a readonly field", - "atomicInteger":100, + "intField":0, + "linkedList1":[], "publicLinkedList":[], - "publicTreeMap":{} + "publicTreeMap":{}, + "readonlyField":"This's a readonly field", + "strField":"str2", + "treeMap":{} } } \ No newline at end of file diff --git a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testIncrementDecode_withMergeArrayOption.txt b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testIncrementDecode_withMergeArrayOption.txt index 5c4ab55..50ba70f 100644 --- a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testIncrementDecode_withMergeArrayOption.txt +++ b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testIncrementDecode_withMergeArrayOption.txt @@ -1,35 +1,35 @@ { - "strField":"strVal1", "enumField":"value1", + "enumMap":{}, + "enumSet":[ + "value1", + "value2" + ], "ints":[ 1, 2, 3, 4 ], - "enumSet":[ - "value1", - "value2" - ], - "enumMap":{}, + "strField":"strVal1", "testBean":{ + "atomicInteger":100, + "booleanField":false, + "charField":"\u0000", + "dateField":"2017-10-01T00:00:00.000Z", "fieldInBaseClass":"Overridden Value", - "treeMap":{}, + "floatField":2.0, + "intField":0, "linkedList1":[ "a", "b", "c", "d" ], - "intField":0, - "floatField":2.0, - "charField":"\u0000", - "booleanField":false, - "dateField":"2017-10-01T00:00:00.000Z", - "readonlyField":"This's a readonly field", - "atomicInteger":100, "publicLinkedList":[], + "publicStrField":"publicStrVal", "publicTreeMap":{}, - "publicStrField":"publicStrVal" + "readonlyField":"This's a readonly field", + "treeMap":{} } } \ No newline at end of file diff --git a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testIncrementDecode_withoutMergeArrayOption.txt b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testIncrementDecode_withoutMergeArrayOption.txt index cc92c20..ce7afcf 100644 --- a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testIncrementDecode_withoutMergeArrayOption.txt +++ b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testIncrementDecode_withoutMergeArrayOption.txt @@ -1,31 +1,31 @@ { - "strField":"strVal1", "enumField":"value1", - "ints":[ - 3, - 4 - ], + "enumMap":{}, "enumSet":[ "value1", "value2" ], - "enumMap":{}, + "ints":[ + 3, + 4 + ], + "strField":"strVal1", "testBean":{ + "atomicInteger":100, + "booleanField":false, + "charField":"\u0000", + "dateField":"2017-10-01T00:00:00.000Z", "fieldInBaseClass":"Overridden Value", - "treeMap":{}, + "floatField":2.0, + "intField":0, "linkedList1":[ "c", "d" ], - "intField":0, - "floatField":2.0, - "charField":"\u0000", - "booleanField":false, - "dateField":"2017-10-01T00:00:00.000Z", - "readonlyField":"This's a readonly field", - "atomicInteger":100, "publicLinkedList":[], + "publicStrField":"publicStrVal", "publicTreeMap":{}, - "publicStrField":"publicStrVal" + "readonlyField":"This's a readonly field", + "treeMap":{} } } \ No newline at end of file diff --git a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testStrictOrder_map.txt b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testSortMapAndSet_map.txt similarity index 100% rename from JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testStrictOrder_map.txt rename to JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testSortMapAndSet_map.txt diff --git a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testStrictOrder_set.txt b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testSortMapAndSet_set.txt similarity index 100% rename from JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testStrictOrder_set.txt rename to JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testSortMapAndSet_set.txt diff --git a/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testSortObjectKey.txt b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testSortObjectKey.txt new file mode 100644 index 0000000..00c0a23 --- /dev/null +++ b/JSONCoder/src/test/resources/org/jsonex/jsoncoder/__snapshot__/JSONCoderTest_testSortObjectKey.txt @@ -0,0 +1 @@ +{"atomicInteger":1001,"bean2":{"enumField":"value2","enumField2":12345,"enumMap":{"value1":"MapValue1"},"enumSet":["value1","value2"],"strEnum":"strEnumV1","testBean":{"$ref":"../../"}},"bean2List":[{"enumMap":{},"enumSet":["value1","value2"],"strField":"AAA"},{"enumMap":{},"enumSet":["value1","value2"],"strField":"BBB"}],"bean2s":[{"enumMap":{},"enumSet":["value1","value2"],"strField":"1"},{"enumMap":{},"enumSet":["value1","value2"],"strField":"2"}],"bigInteger":"123456789012345678901234567890","booleanField":false,"charField":"A","dateField":"1970-01-01T03:23:32.121Z","dateNumberMap":[null],"fieldInBaseClass":"Overridden Value","floatField":1.4,"intField":100,"ints":[4,3,2,1],"linkedList1":["value1"],"publicBigDecimal1":"123456789012345678901234567","publicBigDecimal2":"12","publicInts":[4,3,2,1],"publicLinkedList":["value1"],"publicMap":{"key1":"1970-01-01T00:00:00.000Z","key2":"1970-01-01T00:20:12.121Z"},"publicStrField":"PublicString","publicStringSet":["str1","str2"],"publicTreeMap":{"key1":"value1"},"readonlyField":"This's a readonly field","someClass":"java.util.Date","strField":"String1: '\"","treeMap":{"key1":"value1"},"xmlCalendar":"1970-01-01T00:00:00.000Z"} \ No newline at end of file diff --git a/JSONEX.md b/JSONEX.md index 96df3df..6e94b49 100644 --- a/JSONEX.md +++ b/JSONEX.md @@ -10,16 +10,29 @@ JSON is a popular format for data serialization and configuration, but due to th * **Does not support multi-line String literal**: As configuration, we often need to embedded structured text * **No comma allowed at end of last element**: This causes many of merge issues and make comment out a single line difficult. In javascript, this is allowed +## Design Goal +* **Easy to use** for many kinds of use cases + * configure files + * command line arguments + * URL parameters + * Data storage. Serialize/Deserialize objects + * Data transportation through network. +* **Storage efficient**: remove as much redundancy as possible +* **Easy to parse** The parser should be straight forward to implement + ## Proposal To solve the above limitations, we propose **JSONEX** with following extensions -* Fully compatible with ES6 object literal syntax (So no need specific parser for Javascript) +* Fully compatible with ES6 object literal syntax for majority variables (So no need specific parser for Javascript) * Standard JSON is a validate JSONEX * Support line/block comments as Javascript -* Quote for Key is optional, only if key is not a valid javascript identifier, quote is mandatory +* Quote for Key is optional, required only if key is not a valid javascript identifier, quote is mandatory +* Quote for value is optional, required only if value has special characters confuses parser (Non ES6 compatible) * Quote can use either ("), (') or (\`) +* Optional top level brace (Non ES6 compatible) * Multi-line String literal support with back quote (`). * Commas are allowed for last element (Make it merge friendly) * Object attributes order matters, those order will be persisted +* Path compression such as a:b:c (Non ES6 compabile) ## Examples @@ -56,6 +69,120 @@ To solve the above limitations, we propose **JSONEX** with following extensions line2`, } ``` +## Comparison with JSON for some features + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FeatureJSONexJSON
optional json top level braces - objecta:1,b:2 + +```json +{ + "a": 1, + "b": 2 +} +``` +
optional json top level braces - arraya,b,1 + +```json +["a", "b", 1] +``` +
optional json top level braces - array of objects{a:1},{b:2},c + +```json +[ + {"a": 1}, + {"b": 2}, + "c" +] +``` +
Path compressiona:b:{c:1, d:2} + +```json +{ + "a":{ + "b":{ + "c":1, + "d":2 + } + } +} +``` +
Type wrapperbuyer:user{name:abc, age:10} + +```json +{ + "buyer":{ + "$type":"user", + "name":"abc", + "age":10 + } +} +``` +
Optional Quotes{a: value1, b: value2} + +```json +{"a": "value1", "b": "value2"} +``` +
Commas are allowed for last element{a: 1, b: 2, } + +```json +{"a": 1, "b": 2} +``` +
Multi-line value
{
+    a: 
+      `abc 
+       line2`, 
+    b: 2 
+  }
+ +```json +{"a": "abc\nline2", "b": 2} +``` +
+ ## Other similar effort - [Json5](https://json5.org/) diff --git a/README.md b/README.md index 06a30a9..8ab5c48 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # JSONCoder -[![Build Status](https://travis-ci.org/eBay/jsonex.svg?branch=master)](https://travis-ci.org/eBay/jsonex) +[![Build Status](https://github.com/jianwu/jsonex/actions/workflows/maven.yml/badge.svg)](https://github.com/jianwu/jsonex/actions/) [![codecov](https://codecov.io/gh/eBay/jsonex/branch/master/graph/badge.svg)](https://codecov.io/gh/eBay/jsonex) ## Description Jsonex JSONCoder is a light-weight generic object serialization / deserialization library similar to Jackson, GSON or FastJson. This library has been widely used in various eBay projects for years. It's not a replacement for other popular libraries. But it solves some specific problems which are not available or not well-supported in other alternatives. @@ -55,14 +55,14 @@ Please refer the unit test class for more detailed usage patterns: You can get current version by searching [maven central](https://search.maven.org/search?q=g:org.jsonex) - Simple Serialization / Deserialization - ```java + ```java // serialization JSONCoder.global.encode(o) // de-serialization SomeClass obj = JSONCoder.global.decode(str, SomeClass.class); ``` - Filter out fields and classes - ```java + ```java JSONCoderOption opt = new JSONCoderOption(); // For TestBean2 and it's sub-classes, only include field: "enumField2", "testBean" opt.addFilterFor(TestBean2.class, include("enumField2", "testBean"));("field1ForClass1", "field2ForClass1"); @@ -75,21 +75,21 @@ Please refer the unit test class for more detailed usage patterns: String result = JSONCoder.encode(bean, opt); ``` - Mask out certain fields: for privacy reason, quite often when we serialize an object, we need to maskout certain fields such as emailAddress, here's example to do that: - ```java + ```java String result = JSONCoder.encode(bean, JSONCoderOption.ofIndentFactor(2).addFilterFor(SomeBean.class, mask("field1", "field2"))); ``` - Deserialize with generic types - ```java + ```java String str = "['str1', 'str2', 'str3']"; List result = JSONCoder.global.decode(new DecodeReq>(){}.setSource(str)); ``` - Deserialize and merge to existing object (Incremental decode) - ```java + ```java TestBean bean = JSONCoder.global.decodeTo(jsonStr, bean); ``` - Set custom Quote and custom indentations - ```java + ```java JSONCoderOption opt = new JSONCoderOption(); opt.getJsonOption().setQuoteChar('`'); opt.getJsonOption().setIndentFactor(2); @@ -112,7 +112,19 @@ Please refer the unit test class for more detailed usage patterns: .addCoder(new CoderBigInteger()); String jsonStr = JSONCoder.global.encode(new BigInteger("1234"), opt); ``` - + +## Developer Guild +- Build with non-determinism check (credit to @chessvivek on PR: https://github.com/eBay/jsonex/pull/47) +```bash +mvn edu.illinois:nondex-maven-plugin:1.1.2:nondex -pl JSONCoder +mvn edu.illinois:nondex-maven-plugin:1.1.2:nondex -pl SnapshotTest +mvn edu.illinois:nondex-maven-plugin:1.1.2:nondex -pl core +mvn edu.illinois:nondex-maven-plugin:1.1.2:nondex -pl csv +mvn edu.illinois:nondex-maven-plugin:1.1.2:nondex -pl treedoc +# mvn edu.illinois:nondex-maven-plugin:1.1.2:nondex -pl CliArg # Fail expected, as Cli Annotation depends on field ordering +mvn edu.illinois:nondex-maven-plugin:1.1.2:nondex -pl HiveUDF +``` + ## Limitations and Future enhancements * Performance improvement * Support of variable placeholder in JSON doc diff --git a/SnapshotTest/pom.xml b/SnapshotTest/pom.xml index 1dfec15..e296aeb 100644 --- a/SnapshotTest/pom.xml +++ b/SnapshotTest/pom.xml @@ -6,7 +6,7 @@ org.jsonex jcParent - 0.1.21 + 0.1.27 ../pom.xml SnapshotTest @@ -24,7 +24,7 @@ ${project.groupId} JSONCoder - + org.slf4j slf4j-simple diff --git a/SnapshotTest/src/main/java/org/jsonex/snapshottest/SnapshotOption.java b/SnapshotTest/src/main/java/org/jsonex/snapshottest/SnapshotOption.java index 24ffd47..269f54d 100644 --- a/SnapshotTest/src/main/java/org/jsonex/snapshottest/SnapshotOption.java +++ b/SnapshotTest/src/main/java/org/jsonex/snapshottest/SnapshotOption.java @@ -23,7 +23,7 @@ public class SnapshotOption { /** This method is only available if the serializer is SnapshotSerializerJsonCoder */ @Transient public JSONCoderOption getJsonCoderOption() { - return getIfInstanceOfElseThrow(serializer, SnapshotSerializerJsonCoder.class, s -> s.getOption()); + return getIfInstanceOfOrElseThrow(serializer, SnapshotSerializerJsonCoder.class, s -> s.getOption()); } public SnapshotOption mutateJsonCoderOption(Function mutator) { diff --git a/SnapshotTest/src/main/java/org/jsonex/snapshottest/SnapshotSerializerJsonCoder.java b/SnapshotTest/src/main/java/org/jsonex/snapshottest/SnapshotSerializerJsonCoder.java index cb9d2e4..f770cee 100644 --- a/SnapshotTest/src/main/java/org/jsonex/snapshottest/SnapshotSerializerJsonCoder.java +++ b/SnapshotTest/src/main/java/org/jsonex/snapshottest/SnapshotSerializerJsonCoder.java @@ -6,7 +6,7 @@ import lombok.Setter; public class SnapshotSerializerJsonCoder implements SnapshotSerializer { - @Getter @Setter private transient JSONCoderOption option = JSONCoderOption.ofIndentFactor(2).setStrictOrder(true); + @Getter @Setter private transient JSONCoderOption option = JSONCoderOption.ofIndentFactor(2).setStrictOrdering(true); @Override public String serialize(Object obj) { diff --git a/SnapshotTest/src/test/resources/org/jsonex/snapshottest/__snapshot__/SnapshotTest_testSnapshot_snapshot.json b/SnapshotTest/src/test/resources/org/jsonex/snapshottest/__snapshot__/SnapshotTest_testSnapshot_snapshot.json index 132101a..d482e00 100644 --- a/SnapshotTest/src/test/resources/org/jsonex/snapshottest/__snapshot__/SnapshotTest_testSnapshot_snapshot.json +++ b/SnapshotTest/src/test/resources/org/jsonex/snapshottest/__snapshot__/SnapshotTest_testSnapshot_snapshot.json @@ -1,13 +1,13 @@ { - "option":{ - "testResourceRoot":"src/test/resources", - "serializer":{} - }, - "testClass":"org.jsonex.snapshottest.SnapshotTest", - "testMethod":"testSnapshot", - "name":"test", "actual":"This is actual value", "actualString":"This is actual value", "file":"src/test/resources/org/jsonex/snapshottest/__snapshot__/SnapshotTest_testSnapshot_test.txt", - "tempFile":"src/test/resources/org/jsonex/snapshottest/__snapshot__/SnapshotTest_testSnapshot_test.txt.tmp" + "name":"test", + "option":{ + "serializer":{}, + "testResourceRoot":"src/test/resources" + }, + "tempFile":"src/test/resources/org/jsonex/snapshottest/__snapshot__/SnapshotTest_testSnapshot_test.txt.tmp", + "testClass":"org.jsonex.snapshottest.SnapshotTest", + "testMethod":"testSnapshot" } \ No newline at end of file diff --git a/core/pom.xml b/core/pom.xml index 345eae4..8aa7f22 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -6,7 +6,7 @@ org.jsonex jcParent - 0.1.21 + 0.1.27 ../pom.xml core diff --git a/core/src/main/java/org/jsonex/core/charsource/ArrayCharSource.java b/core/src/main/java/org/jsonex/core/charsource/ArrayCharSource.java index 5bc961f..09cb0b6 100644 --- a/core/src/main/java/org/jsonex/core/charsource/ArrayCharSource.java +++ b/core/src/main/java/org/jsonex/core/charsource/ArrayCharSource.java @@ -36,7 +36,7 @@ public class ArrayCharSource extends CharSource { @Override public boolean isEof(int i) { return startIndex + bookmark.getPos() + i >= endIndex; } - @Override public boolean readUntil(Predicate predicate, StringBuilder target, int minLen, int maxLen) { + @Override public boolean readUntil(StringBuilder target, Predicate predicate, int minLen, int maxLen) { int startPos = bookmark.getPos(); int len = 0; boolean matched = false; diff --git a/core/src/main/java/org/jsonex/core/charsource/CharSource.java b/core/src/main/java/org/jsonex/core/charsource/CharSource.java index 8517c16..1c0dba8 100644 --- a/core/src/main/java/org/jsonex/core/charsource/CharSource.java +++ b/core/src/main/java/org/jsonex/core/charsource/CharSource.java @@ -41,21 +41,21 @@ public abstract class CharSource { * * @return true The terminate condition matches. otherwise, could be EOF or length matches */ - public abstract boolean readUntil(Predicate predicate, StringBuilder target, int minLen, int maxLen); + public abstract boolean readUntil(StringBuilder target, Predicate predicate, int minLen, int maxLen); public boolean readUntil(Predicate predicate, StringBuilder target) { - return readUntil(predicate, target, 0, MAX_STRING_LEN); + return readUntil(target, predicate, 0, MAX_STRING_LEN); } public boolean skipUntil(Predicate predicate) { - return readUntil(predicate, null, 0, Integer.MAX_VALUE); + return readUntil(null, predicate, 0, Integer.MAX_VALUE); } /** @return true Terminal conditions matches */ - public boolean readUntil(String chars, @Nullable Collection strs, StringBuilder target, boolean include, int minLen, int maxLen) { - return readUntil(s -> (chars.indexOf(s.peek(0)) >= 0 || startsWithAny(strs))== include, target, minLen, maxLen); + public boolean readUntil(StringBuilder target, String chars, @Nullable Collection strs, boolean include, int minLen, int maxLen) { + return readUntil(target, s -> (chars.indexOf(s.peek(0)) >= 0 || startsWithAny(strs)) == include, minLen, maxLen); } - public boolean readUntil(String terminator, StringBuilder target) { return readUntil(terminator, null, target); } + public boolean readUntil(StringBuilder target, String terminator) { return readUntil(target, terminator, null); } /** @return true Terminal conditions matches */ - public boolean readUntil(String terminator, Collection strs, StringBuilder target) { - return readUntil(terminator, strs, target, true, 0, MAX_STRING_LEN); + public boolean readUntil(StringBuilder target, String terminator, Collection strs) { + return readUntil(target, terminator, strs, true, 0, MAX_STRING_LEN); } public String readUntil(String terminator) { return readUntil(terminator, null,0, Integer.MAX_VALUE); } /** @return true Terminal conditions matches */ @@ -63,13 +63,13 @@ public boolean readUntil(String terminator, Collection strs, StringBuild /** @return true Terminal conditions matches */ public String readUntil(String terminator, Collection strs, int minLen, int maxLen) { StringBuilder sb = new StringBuilder(); - readUntil(terminator, strs, sb, true, minLen, maxLen); + readUntil(sb, terminator, strs, true, minLen, maxLen); return sb.toString(); } /** @return true Indicates more character in the stream */ public boolean skipUntil(String chars, boolean include) { - return readUntil(chars, null, null, include, 0, Integer.MAX_VALUE); + return readUntil(null, chars, null, include, 0, Integer.MAX_VALUE); } /** @return true Indicates more character in the stream */ public boolean skipUntil(String terminator) { return skipUntil(terminator, true); } @@ -79,7 +79,7 @@ public boolean skipUntil(String chars, boolean include) { /** @return true Indicates more character in the stream */ public boolean skipChars(String chars) { return skipUntil(chars, false); } - public boolean read(StringBuilder target, int len) { return readUntil(s -> true, target, len, len); } + public boolean read(StringBuilder target, int len) { return readUntil(target, s -> true, len, len); } public String read(int len) { StringBuilder sb = new StringBuilder(); @@ -90,19 +90,19 @@ public String read(int len) { public boolean skip() { return read(null, 1); } public boolean skip(int len) { return read(null, len); } - public boolean readUntilMatch(String str, boolean skipStr, StringBuilder target, int minLen, int maxLen) { - boolean matches = readUntil(s -> startsWith(str), target, minLen, maxLen); + public boolean readUntilMatch(StringBuilder target, String str, boolean skipStr, int minLen, int maxLen) { + boolean matches = readUntil(target, s -> startsWith(str), minLen, maxLen); if (matches && skipStr) skip(str.length()); return matches; } public boolean readUntilMatch(String str, boolean skipStr, StringBuilder target) { - return readUntilMatch(str, skipStr, target, 0, MAX_STRING_LEN); + return readUntilMatch(target, str, skipStr, 0, MAX_STRING_LEN); } public boolean skipUntilMatch(String str, boolean skipStr) { - return readUntilMatch(str, skipStr, null, 0, Integer.MAX_VALUE); + return readUntilMatch(null, str, skipStr, 0, Integer.MAX_VALUE); } public String peekString(int len) { @@ -145,17 +145,17 @@ private String getTermStrWithQuoteAndEscape(char quote) { } public String readQuotedString(char quote) { - return readQuotedString(quote, new StringBuilder()).toString(); + return readQuotedString( new StringBuilder(), quote).toString(); } - public StringBuilder readQuotedString(char quote, StringBuilder sb) { + public StringBuilder readQuotedString(StringBuilder sb, char quote) { String terminator = getTermStrWithQuoteAndEscape(quote); // Not calling getBookmark() to avoid clone an object int pos = getPos(); int line = bookmark.getLine(); int col = bookmark.getCol(); while (true) { - if (!readUntil(terminator, sb)) + if (!readUntil(sb, terminator)) throw new EOFRuntimeException("Can't find matching quote at position:" + pos + ";line:" + line + ";col:" + col); char c = read(); if (c == quote) { diff --git a/core/src/main/java/org/jsonex/core/charsource/ReaderCharSource.java b/core/src/main/java/org/jsonex/core/charsource/ReaderCharSource.java index 99df291..97daae6 100644 --- a/core/src/main/java/org/jsonex/core/charsource/ReaderCharSource.java +++ b/core/src/main/java/org/jsonex/core/charsource/ReaderCharSource.java @@ -86,7 +86,7 @@ private boolean fill() { return p >= loadPos; } - @Override public boolean readUntil(Predicate predicate, StringBuilder target, int minLen, int maxLen) { + @Override public boolean readUntil(StringBuilder target, Predicate predicate, int minLen, int maxLen) { if (target != null) { backupTarget = target; backupMark = getPos(); diff --git a/core/src/main/java/org/jsonex/core/factory/CacheGlobal.java b/core/src/main/java/org/jsonex/core/factory/CacheGlobal.java deleted file mode 100644 index ca911be..0000000 --- a/core/src/main/java/org/jsonex/core/factory/CacheGlobal.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.jsonex.core.factory; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -public class CacheGlobal implements CacheProvider { - public final static InjectableInstance it = InjectableInstance.of(CacheGlobal.class); - public static CacheGlobal get() { return it.get(); } - - private final Map cache = new ConcurrentHashMap<>(); - - @Override public ObjectCache getCache(Object key) { - return cache.computeIfAbsent(key, (k) -> new ObjectCacheMapImpl(new ConcurrentHashMap<>())); - } -} diff --git a/core/src/main/java/org/jsonex/core/factory/CacheThreadLocal.java b/core/src/main/java/org/jsonex/core/factory/CacheThreadLocal.java deleted file mode 100644 index aaa9b6d..0000000 --- a/core/src/main/java/org/jsonex/core/factory/CacheThreadLocal.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.jsonex.core.factory; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -public class CacheThreadLocal implements CacheProvider { - public final static InjectableInstance it = InjectableInstance.of(CacheThreadLocal.class); - public static CacheThreadLocal get() { return it.get(); } - - // Seems ThreadLocal.withInitial() is not synchronized when create initial, so potentially it could be called multiple times in race condition - private ThreadLocal> cache = ThreadLocal.withInitial(ConcurrentHashMap::new); - - @Override public ObjectCache getCache(Object scope) { - return cache.get().computeIfAbsent(scope, (k) -> new ObjectCacheMapImpl(new ConcurrentHashMap<>())); - } -} diff --git a/core/src/main/java/org/jsonex/core/factory/InjectableFactory.java b/core/src/main/java/org/jsonex/core/factory/InjectableFactory.java index e880961..0d0164c 100644 --- a/core/src/main/java/org/jsonex/core/factory/InjectableFactory.java +++ b/core/src/main/java/org/jsonex/core/factory/InjectableFactory.java @@ -9,8 +9,8 @@ package org.jsonex.core.factory; -import org.jsonex.core.factory.CacheProvider.NoCache; -import org.jsonex.core.factory.CacheProvider.ObjectCache; +import org.jsonex.core.factory.ScopeProvider.NoCache; +import org.jsonex.core.factory.ScopeProvider.Scope; import org.jsonex.core.type.Func; import org.jsonex.core.type.Tuple; import lombok.Getter; @@ -33,13 +33,13 @@ public class InjectableFactory { private Function objectCreator; @Getter private static List> globalCreateHandlers = new ArrayList<>(); @Getter private List> createHandlers = new ArrayList<>(); - private final CacheProvider cacheProvider; + private final ScopeProvider scopeProvider; private final Function initialCreator; - protected InjectableFactory(Function creator, CacheProvider cacheProvider) { + protected InjectableFactory(Function creator, ScopeProvider scopeProvider) { initialCreator = creator; - this.cacheProvider = cacheProvider; + this.scopeProvider = scopeProvider; setCreator(creator); } @@ -52,8 +52,8 @@ public static InjectableFactory of(Function objectCreat return of(objectCreator, NoCache.get()); } - public static InjectableFactory of(Function objectCreator, CacheProvider cacheProvider) { - return new InjectableFactory<>(objectCreator, cacheProvider); + public static InjectableFactory of(Function objectCreator, ScopeProvider scopeProvider) { + return new InjectableFactory<>(objectCreator, scopeProvider); } public TI get() { return get(null); } @@ -62,8 +62,8 @@ public TI get(TP param) { return getCache().get(getCacheKey(param), (key) -> create(param)); } - protected ObjectCache getCache() { - return cacheProvider.getCache(this); + protected Scope getCache() { + return scopeProvider.getCache(this); } // Have to use a placeholder for null for ConcurrentHashMap unfortunately @@ -96,51 +96,51 @@ private TI create(TP param) { } public static class _0 extends InjectableFactory { - public _0(Function creator, CacheProvider cacheProvider) { super(creator, cacheProvider); } + public _0(Function creator, ScopeProvider scopeProvider) { super(creator, scopeProvider); } public static _0 of(Supplier objectCreator) { return of(objectCreator, NoCache.get()); } - public static _0 of(Supplier objectCreator, CacheProvider cacheProvider) { return new _0<>((Function)(p -> objectCreator.get()), cacheProvider); } + public static _0 of(Supplier objectCreator, ScopeProvider scopeProvider) { return new _0<>((Function)(p -> objectCreator.get()), scopeProvider); } public I get() { return super.get(null); } } - public static class _2 extends InjectableFactory, I> { - public _2(Function, I> creator, CacheProvider cacheProvider) { super(creator, cacheProvider); } - public static _2 of(BiFunction objectCreator) { return of(objectCreator, NoCache.get()); } - public static _2 of(BiFunction objectCreator, CacheProvider cacheProvider) { return new _2<>((Function, I>)(p -> objectCreator.apply(p._1, p._2)), cacheProvider); } - public I get(P1 p1, P2 p2) { return super.get(Tuple.Pair.of(p1, p2)); } + public static class _2 extends InjectableFactory, I> { + public _2(Function, I> creator, ScopeProvider scopeProvider) { super(creator, scopeProvider); } + public static _2 of(BiFunction objectCreator) { return of(objectCreator, NoCache.get()); } + public static _2 of(BiFunction objectCreator, ScopeProvider scopeProvider) { return new _2<>((Function, I>)(p -> objectCreator.apply(p._0, p._1)), scopeProvider); } + public I get(P0 p0, P1 p1) { return super.get(Tuple.Pair.of(p0, p1)); } } - public static class _3 extends InjectableFactory, I> { - public _3(Function, I> creator, CacheProvider cacheProvider) { super(creator, cacheProvider); } - public static _3 of(Func._3 objectCreator) { return of(objectCreator, NoCache.get()); } - public static _3 of(Func._3 objectCreator, CacheProvider cacheProvider) { return new _3<>((Function, I>)(p -> objectCreator.apply(p._1, p._2, p._3)), cacheProvider); } - public I get(P1 p1, P2 p2, P3 p3) { return super.get(Tuple._3.of(p1, p2, p3)); } + public static class _3 extends InjectableFactory, I> { + public _3(Function, I> creator, ScopeProvider scopeProvider) { super(creator, scopeProvider); } + public static _3 of(Func._3 objectCreator) { return of(objectCreator, NoCache.get()); } + public static _3 of(Func._3 objectCreator, ScopeProvider scopeProvider) { return new _3<>((Function, I>)(p -> objectCreator.apply(p._0, p._1, p._2)), scopeProvider); } + public I get(P0 p0, P1 p1, P2 p2) { return super.get(Tuple.Tuple3.of(p0, p1, p2)); } } - public static class _4 extends InjectableFactory, I> { - public _4(Function, I> creator, CacheProvider cacheProvider) { super(creator, cacheProvider); } - public static _4 of(Func._4 objectCreator) { return of(objectCreator, NoCache.get()); } - public static _4 of(Func._4 objectCreator, CacheProvider cacheProvider) { return new _4<>((Function, I>)(p -> objectCreator.apply(p._1, p._2, p._3, p._4)), cacheProvider); } - public I get(P1 p1, P2 p2, P3 p3, P4 p4) { return super.get(Tuple._4.of(p1, p2, p3, p4)); } + public static class _4 extends InjectableFactory, I> { + public _4(Function, I> creator, ScopeProvider scopeProvider) { super(creator, scopeProvider); } + public static _4 of(Func._4 objectCreator) { return of(objectCreator, NoCache.get()); } + public static _4 of(Func._4 objectCreator, ScopeProvider scopeProvider) { return new _4<>((Function, I>)(p -> objectCreator.apply(p._0, p._1, p._2, p._3)), scopeProvider); } + public I get(P0 p0, P1 p1, P2 p2, P3 p3) { return super.get(Tuple.Tuple4.of(p0, p1, p2, p3)); } } - public static class _5 extends InjectableFactory, I> { - public _5(Function, I> creator, CacheProvider cacheProvider) { super(creator, cacheProvider); } - public static _5 of(Func._5 objectCreator) { return of(objectCreator, NoCache.get()); } - public static _5 of(Func._5 objectCreator, CacheProvider cacheProvider) { return new _5<>((Function, I>)(p -> objectCreator.apply(p._1, p._2, p._3, p._4, p._5)), cacheProvider); } - public I get(P1 p1, P2 p2, P3 p3, P4 p4, P5 p5) { return super.get(Tuple._5.of(p1, p2, p3, p4, p5)); } + public static class _5 extends InjectableFactory, I> { + public _5(Function, I> creator, ScopeProvider scopeProvider) { super(creator, scopeProvider); } + public static _5 of(Func._5 objectCreator) { return of(objectCreator, NoCache.get()); } + public static _5 of(Func._5 objectCreator, ScopeProvider scopeProvider) { return new _5<>((Function, I>)(p -> objectCreator.apply(p._0, p._1, p._2, p._3, p._4)), scopeProvider); } + public I get(P0 p0, P1 p1, P2 p2, P3 p3, P4 p4) { return super.get(Tuple.Tuple5.of(p0, p1, p2, p3, p4)); } } - public static class _6 extends InjectableFactory, I> { - public _6(Function, I> creator, CacheProvider cacheProvider) { super(creator, cacheProvider); } - public static _6 of(Func._6 objectCreator) { return of(objectCreator, NoCache.get()); } - public static _6 of(Func._6 objectCreator, CacheProvider cacheProvider) { return new _6<>((Function, I>)(p -> objectCreator.apply(p._1, p._2, p._3, p._4, p._5, p._6)), cacheProvider); } - public I get(P1 p1, P2 p2, P3 p3, P4 p4, P5 p5, P6 p6) { return super.get(Tuple._6.of(p1, p2, p3, p4, p5,p6)); } + public static class _6 extends InjectableFactory, I> { + public _6(Function, I> creator, ScopeProvider scopeProvider) { super(creator, scopeProvider); } + public static _6 of(Func._6 objectCreator) { return of(objectCreator, NoCache.get()); } + public static _6 of(Func._6 objectCreator, ScopeProvider scopeProvider) { return new _6<>((Function, I>)(p -> objectCreator.apply(p._0, p._1, p._2, p._3, p._4, p._5)), scopeProvider); } + public I get(P0 p0, P1 p1, P2 p2, P3 p3, P4 p4, P5 p5) { return super.get(Tuple.Tuple6.of(p0, p1, p2, p3, p4,p5)); } } - public static class _7 extends InjectableFactory, I> { - public _7(Function, I> creator, CacheProvider cacheProvider) { super(creator, cacheProvider); } - public static _7 of(Func._7 objectCreator) { return of(objectCreator, NoCache.get()); } - public static _7 of(Func._7 objectCreator, CacheProvider cacheProvider) { return new _7<>((Function, I>)(p -> objectCreator.apply(p._1, p._2, p._3, p._4, p._5, p._6, p._7)), cacheProvider); } - public I get(P1 p1, P2 p2, P3 p3, P4 p4, P5 p5, P6 p6, P7 p7) { return super.get(Tuple._7.of(p1, p2, p3, p4, p5, p6, p7)); } + public static class _7 extends InjectableFactory, I> { + public _7(Function, I> creator, ScopeProvider scopeProvider) { super(creator, scopeProvider); } + public static _7 of(Func._7 objectCreator) { return of(objectCreator, NoCache.get()); } + public static _7 of(Func._7 objectCreator, ScopeProvider scopeProvider) { return new _7<>((Function, I>)(p -> objectCreator.apply(p._0, p._1, p._2, p._3, p._4, p._5, p._6)), scopeProvider); } + public I get(P0 p0, P1 p1, P2 p2, P3 p3, P4 p4, P5 p5, P6 p6) { return super.get(Tuple.Tuple7.of(p0, p1, p2, p3, p4, p5, p6)); } } } diff --git a/core/src/main/java/org/jsonex/core/factory/ScopeGlobal.java b/core/src/main/java/org/jsonex/core/factory/ScopeGlobal.java new file mode 100644 index 0000000..a77b2f9 --- /dev/null +++ b/core/src/main/java/org/jsonex/core/factory/ScopeGlobal.java @@ -0,0 +1,15 @@ +package org.jsonex.core.factory; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class ScopeGlobal implements ScopeProvider { + public final static InjectableInstance it = InjectableInstance.of(ScopeGlobal.class); + public static ScopeGlobal get() { return it.get(); } + + private final Map cache = new ConcurrentHashMap<>(); + + @Override public Scope getCache(Object key) { + return cache.computeIfAbsent(key, (k) -> new ScopeMapImpl(new ConcurrentHashMap<>())); + } +} diff --git a/core/src/main/java/org/jsonex/core/factory/CacheProvider.java b/core/src/main/java/org/jsonex/core/factory/ScopeProvider.java similarity index 70% rename from core/src/main/java/org/jsonex/core/factory/CacheProvider.java rename to core/src/main/java/org/jsonex/core/factory/ScopeProvider.java index ebef4d2..7f9b789 100644 --- a/core/src/main/java/org/jsonex/core/factory/CacheProvider.java +++ b/core/src/main/java/org/jsonex/core/factory/ScopeProvider.java @@ -5,17 +5,17 @@ import java.util.Map; import java.util.function.Function; -public interface CacheProvider { - ObjectCache getCache(Object scope); +public interface ScopeProvider { + Scope getCache(Object scope); - interface ObjectCache { + interface Scope { V get(K k, Function creator); V put(K k, V v); void clear(); } @RequiredArgsConstructor - class ObjectCacheMapImpl implements ObjectCache { + class ScopeMapImpl implements Scope { private final Map map; @Override public V get(K k, Function creator) { return map.computeIfAbsent(k, creator); } @@ -23,16 +23,16 @@ class ObjectCacheMapImpl implements ObjectCache { @Override public void clear() { map.clear(); } } - class ObjectCachePassThrough implements ObjectCache { + class ScopePassThrough implements Scope { @Override public V get(K k, Function creator) { return creator.apply(k); } @Override public V put(K k, V v) { throw new UnsupportedOperationException("put can't be called for Pass Through cache"); } @Override public void clear() { } } - class NoCache implements CacheProvider { + class NoCache implements ScopeProvider { private static InjectableInstance it = InjectableInstance.of(NoCache.class); public static NoCache get() { return it.get(); } - @Override public ObjectCache getCache(Object scope) { return new ObjectCachePassThrough<>(); } + @Override public Scope getCache(Object scope) { return new ScopePassThrough<>(); } } } diff --git a/core/src/main/java/org/jsonex/core/factory/ScopeThreadLocal.java b/core/src/main/java/org/jsonex/core/factory/ScopeThreadLocal.java new file mode 100644 index 0000000..83564de --- /dev/null +++ b/core/src/main/java/org/jsonex/core/factory/ScopeThreadLocal.java @@ -0,0 +1,16 @@ +package org.jsonex.core.factory; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class ScopeThreadLocal implements ScopeProvider { + public final static InjectableInstance it = InjectableInstance.of(ScopeThreadLocal.class); + public static ScopeThreadLocal get() { return it.get(); } + + // Seems ThreadLocal.withInitial() is not synchronized when create initial, so potentially it could be called multiple times in race condition + private ThreadLocal> cache = ThreadLocal.withInitial(ConcurrentHashMap::new); + + @Override public Scope getCache(Object scope) { + return cache.get().computeIfAbsent(scope, (k) -> new ScopeMapImpl(new ConcurrentHashMap<>())); + } +} diff --git a/core/src/main/java/org/jsonex/core/factory/TimeProvider.java b/core/src/main/java/org/jsonex/core/factory/TimeProvider.java index 7cf0e54..b4a26da 100644 --- a/core/src/main/java/org/jsonex/core/factory/TimeProvider.java +++ b/core/src/main/java/org/jsonex/core/factory/TimeProvider.java @@ -13,7 +13,10 @@ import lombok.Setter; import lombok.experimental.Accessors; +import java.time.Clock; +import java.time.Instant; import java.util.Date; +import java.util.concurrent.TimeUnit; /** * A time provider to provide abstraction of system time. So that application logic can avoid direct dependency of system time @@ -23,18 +26,37 @@ public interface TimeProvider { InjectableInstance it = InjectableInstance.of(Impl.class); static TimeProvider get() { return it.get(); } + static long millis() { return get().getTimeMillis(); } + static long nano() { return get().getNanoTime(); } + static long duration(long since) { return get().getTimeMillis() - since; } + static long now(long offset) { return get().getTimeMillis() + offset; } + static long now(long offset, TimeUnit unit) { return get().getTimeMillis() + unit.toMillis(offset); } + class Impl implements TimeProvider { @Override public Date getDate() { return new Date(); } @Override public long getTimeMillis() { return System.currentTimeMillis(); } + @Override public long getNanoTime() { return systemEpochNanoTime(); } } @Accessors(chain = true) class Mock implements TimeProvider { - @Setter @Getter long timeMillis = System.currentTimeMillis(); - @Override public Date getDate() { return new Date(timeMillis); } - public void add(long offset) { timeMillis += offset; } + @Setter @Getter long nanoTime = systemEpochNanoTime(); + + @Override public long getTimeMillis() { return TimeUnit.NANOSECONDS.toMillis(nanoTime); } + public TimeProvider setTimeMillis(long ms) { nanoTime = TimeUnit.MILLISECONDS.toNanos(ms); return this; } + + @Override public Date getDate() { return new Date(getTimeMillis()); } + @Deprecated + public void add(long offset) { nanoTime += TimeUnit.MILLISECONDS.toNanos(offset); } + public void sleepMs(long offset) { nanoTime += TimeUnit.MILLISECONDS.toNanos(offset); } } Date getDate(); long getTimeMillis(); + long getNanoTime(); + + default long systemEpochNanoTime() { + Instant clock = Clock.systemDefaultZone().instant(); + return clock.getEpochSecond() * 1000_000_000L + clock.getNano(); + } } diff --git a/core/src/main/java/org/jsonex/core/type/Tuple.java b/core/src/main/java/org/jsonex/core/type/Tuple.java index eb7e85c..adbbe4a 100644 --- a/core/src/main/java/org/jsonex/core/type/Tuple.java +++ b/core/src/main/java/org/jsonex/core/type/Tuple.java @@ -2,19 +2,21 @@ import lombok.AllArgsConstructor; import lombok.Data; +import lombok.RequiredArgsConstructor; +/** Modeled as mutable and provide a default constructor so that simplify the subclasses logic */ public interface Tuple { - @Data @AllArgsConstructor(staticName = "of") class Pair implements Tuple { public T1 _1; public T2 _2; } - @Data @AllArgsConstructor(staticName = "of") class _3 implements Tuple { public T1 _1; public T2 _2; public T3 _3; } - @Data @AllArgsConstructor(staticName = "of") class _4 implements Tuple { public T1 _1; public T2 _2; public T3 _3; public T4 _4; } - @Data @AllArgsConstructor(staticName = "of") class _5 implements Tuple { public T1 _1; public T2 _2; public T3 _3; public T4 _4; public T5 _5; } - @Data @AllArgsConstructor(staticName = "of") class _6 implements Tuple { public T1 _1; public T2 _2; public T3 _3; public T4 _4; public T5 _5; public T6 _6; } - @Data @AllArgsConstructor(staticName = "of") class _7 implements Tuple { public T1 _1; public T2 _2; public T3 _3; public T4 _4; public T5 _5; public T6 _6; public T7 _7; } + @Data @AllArgsConstructor(staticName = "of") @RequiredArgsConstructor class Pair implements Tuple { public T0 _0; public T1 _1; } + @Data @AllArgsConstructor(staticName = "of") @RequiredArgsConstructor class Tuple3 implements Tuple { public T0 _0; public T1 _1; public T2 _2; } + @Data @AllArgsConstructor(staticName = "of") @RequiredArgsConstructor class Tuple4 implements Tuple { public T0 _0; public T1 _1; public T2 _2; public T3 _3; } + @Data @AllArgsConstructor(staticName = "of") @RequiredArgsConstructor class Tuple5 implements Tuple { public T0 _0; public T1 _1; public T2 _2; public T3 _3; public T4 _4; } + @Data @AllArgsConstructor(staticName = "of") @RequiredArgsConstructor class Tuple6 implements Tuple { public T0 _0; public T1 _1; public T2 _2; public T3 _3; public T4 _4; public T5 _5; } + @Data @AllArgsConstructor(staticName = "of") @RequiredArgsConstructor class Tuple7 implements Tuple { public T0 _0; public T1 _1; public T2 _2; public T3 _3; public T4 _4; public T5 _5; public T6 _6; } - static Pair of(T1 _1, T2 _2) { return Pair.of(_1, _2); } - static _3 of(T1 _1, T2 _2, T3 _3) { return Tuple._3.of(_1, _2, _3); } - static _4 of(T1 _1, T2 _2, T3 _3, T4 _4) { return Tuple._4.of(_1, _2, _3, _4); } - static _5 of(T1 _1, T2 _2, T3 _3, T4 _4, T5 _5) { return Tuple._5.of(_1, _2, _3, _4, _5); } - static _6 of(T1 _1, T2 _2, T3 _3, T4 _4, T5 _5, T6 _6) { return Tuple._6.of(_1, _2, _3, _4, _5, _6); } - static _7 of(T1 _1, T2 _2, T3 _3, T4 _4, T5 _5, T6 _6, T7 _7) { return Tuple._7.of(_1, _2, _3, _4, _5, _6, _7); } + static Pair of(T0 _0, T1 _1) { return Pair.of(_0, _1); } + static Tuple3 of(T0 _0, T1 _1, T2 _2) { return Tuple3.of(_0, _1, _2); } + static Tuple4 of(T0 _0, T1 _1, T2 _2, T3 _3) { return Tuple4.of(_0, _1, _2, _3); } + static Tuple5 of(T0 _0, T1 _1, T2 _2, T3 _3, T4 _4) { return Tuple5.of(_0, _1, _2, _3, _4); } + static Tuple6 of(T0 _0, T1 _1, T2 _2, T3 _3, T4 _4, T5 _5) { return Tuple6.of(_0, _1, _2, _3, _4, _5); } + static Tuple7 of(T0 _0, T1 _1, T2 _2, T3 _3, T4 _4, T5 _5, T6 _6) { return Tuple7.of(_0, _1, _2, _3, _4, _5, _6); } } diff --git a/core/src/main/java/org/jsonex/core/type/Union.java b/core/src/main/java/org/jsonex/core/type/Union.java new file mode 100644 index 0000000..3e02ad6 --- /dev/null +++ b/core/src/main/java/org/jsonex/core/type/Union.java @@ -0,0 +1,33 @@ +package org.jsonex.core.type; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Data; + +public interface Union { + @Data @AllArgsConstructor(access=AccessLevel.PROTECTED) class Union2 implements Union { + public final T0 _0; + public final T1 _1; + public Class getType() { + if (_0 != null) return _0.getClass(); + else return _1.getClass(); + } + + public static Union2 of_0(T0 _0) { return new Union2<>(_0, null); } + public static Union2 of_1(T1 _1) { return new Union2<>(null, _1); } + } + + @Data @AllArgsConstructor(access=AccessLevel.PROTECTED) class Union3 implements Union { + public final T0 _0; + public final T1 _1; + public final T2 _2; + public Class getType() { + if (_0 != null) return _0.getClass(); + if (_1 != null) return _1.getClass(); + else return _2.getClass(); + } + public static Union3 of_0(T0 _0) { return new Union3<>(_0, null, null); } + public static Union3 of_1(T1 _1) { return new Union3<>(null, _1, null); } + public static Union3 of_2(T2 _2) { return new Union3<>(null, null, _2); } + } +} diff --git a/core/src/main/java/org/jsonex/core/util/ArrayUtil.java b/core/src/main/java/org/jsonex/core/util/ArrayUtil.java index c5e4c5a..8b9836e 100644 --- a/core/src/main/java/org/jsonex/core/util/ArrayUtil.java +++ b/core/src/main/java/org/jsonex/core/util/ArrayUtil.java @@ -1,10 +1,100 @@ package org.jsonex.core.util; +import org.jsonex.core.type.Nullable; + +import java.lang.reflect.Array; +import java.util.Arrays; +import java.util.Optional; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Predicate; + public class ArrayUtil { - public static int indexOf(T[] array, T e) { + public static int indexOf(@Nullable T[] array, T e) { + if (array == null) + return -1; for (int i = 0; i < array.length; i++) if (array[i] == e) return i; return -1; } + + public static boolean contains(@Nullable T[] array, T e) { + return indexOf(array, e) >= 0; + } + + public static TDest[] map(@Nullable TSrc[] source, Function func, TDest[] dest) { + return mapWithIndex(source, (s, idx) -> func.apply(s), dest); + } + + public static TDest[] mapWithIndex( + @Nullable TSrc[] source, BiFunction func, TDest[] dest) { + if (source == null) + return null; + if (dest.length < source.length) + dest = (TDest[]) Array.newInstance(dest.getClass().getComponentType(), source.length); + for (int i = 0; i < source.length; i++) + dest[i] = func.apply(source[i], i); + return dest; + } + + public static Integer[] box(@Nullable int[] ints) { + if (ints == null) + return null; + Integer[] result = new Integer[ints.length]; + for (int i = 0; i < ints.length; i++) + result[i] = ints[i]; + return result; + } + + public static int[] unbox(@Nullable Integer[] ints) { + if (ints == null) + return null; + int[] result = new int[ints.length]; + for (int i = 0; i < ints.length; i++) + result[i] = ints[i]; + return result; + } + + public static T[] subArray(@Nullable T[] array, int start, int length) { + if (start < 0) + start = array.length + start; + return Arrays.copyOfRange(array, start, start + length); + } + + public static T[] subArray(@Nullable T[] array, int start) { + return subArray(array, start, (array.length - start) % array.length); + } + + public static Optional first(V[] source, Predicate pred) { + if (source != null) + for (V s : source) + if (pred.test(s)) + return Optional.of(s); + return Optional.empty(); + } + + public static int indexOf(V[] source, Predicate pred) { + if (source != null) + for (int i = 0; i < source.length; i++) + if (pred.test(source[i])) + return i; + return -1; + } + + public static T reduce(V[] source, T identity, BiFunction accumulate) { + T result = identity; + if (source != null) + for (V s : source) + result = accumulate.apply(result, s); + return result; + } + + public static T reduceTo(V[] source, T result, BiConsumer accumulate) { + if (source != null) + for (V s : source) + accumulate.accept(result, s); + return result; + } } diff --git a/core/src/main/java/org/jsonex/core/util/Assert.java b/core/src/main/java/org/jsonex/core/util/Assert.java index e68207f..64f4c18 100644 --- a/core/src/main/java/org/jsonex/core/util/Assert.java +++ b/core/src/main/java/org/jsonex/core/util/Assert.java @@ -7,10 +7,14 @@ @UtilityClass public class Assert { public void isTrue(boolean condition, Supplier error) { if (!condition) throw new AssertionError(error.get()); } - public void isNull(Object val, Supplier error) { if (val != null) throw new AssertionError(error.get()); } - public void isNotNull(Object val, Supplier error) { if (val == null) throw new AssertionError(error.get()); } - public void isTrue(boolean condition, String error) { if (!condition) throw new AssertionError(error); } + public void isTrue(boolean condition) { if (!condition) throw new AssertionError(); } + + public void isNull(Object val, Supplier error) { if (val != null) throw new AssertionError(error.get()); } public void isNull(Object val, String error) { if (val != null) throw new AssertionError(error); } + public void isNull(Object val) { if (val != null) throw new AssertionError(); } + + public void isNotNull(Object val, Supplier error) { if (val == null) throw new AssertionError(error.get()); } public void isNotNull(Object val, String error) { if (val == null) throw new AssertionError(error); } + public void isNotNull(Object val) { if (val == null) throw new AssertionError(); } } diff --git a/core/src/main/java/org/jsonex/core/util/BeanProperty.java b/core/src/main/java/org/jsonex/core/util/BeanProperty.java index 5294e24..4653ec1 100644 --- a/core/src/main/java/org/jsonex/core/util/BeanProperty.java +++ b/core/src/main/java/org/jsonex/core/util/BeanProperty.java @@ -15,8 +15,10 @@ import java.lang.annotation.Annotation; import java.lang.reflect.*; +import java.util.Optional; import java.util.function.Function; +import static org.jsonex.core.util.ArrayUtil.first; import static org.jsonex.core.util.LangUtil.getIfInstanceOf; /** @@ -29,13 +31,15 @@ public class BeanProperty { @Getter final String name; @Getter Method setter; @Getter Method getter; + @Getter Method hasChecker; @Getter Field field; public boolean isTransient() { if (field != null && Modifier.isTransient(field.getModifiers())) return true; - return getAnnotation(java.beans.Transient.class) != null; + // java.beans.Transient.class is not available in Android. we use name matching + return getAnnotation("Transient") != null; } public boolean isImmutable(boolean allowPrivate){return setter == null && !isFieldAccessible(allowPrivate); } @@ -65,6 +69,8 @@ public void set(Object obj, Object value){ public Object get(Object obj){ try { + if (hasChecker != null && Boolean.FALSE.equals(hasChecker.invoke(obj))) + return null; if (getter != null) { getter.setAccessible(true); return getter.invoke(obj); @@ -73,7 +79,7 @@ public Object get(Object obj){ return field.get(obj); } } catch(Exception e) { - throw new InvokeRuntimeException("error get value:" + name + ", class:" + obj.getClass(), e); + throw new InvokeRuntimeException("error get value:" + name + ", class:" + obj.getClass() + ";hasChecker:" + hasChecker, e); } throw new InvokeRuntimeException("field is not readable: " + name + ", class:" + obj.getClass()); } @@ -97,7 +103,28 @@ public T getAnnotation(Class cls) { return null; } - + + public Annotation getAnnotation(String name) { + Optional result; + if (getter != null) { + result = getAnnotation(getter.getAnnotations(), name); + if (result.isPresent()) + return result.get(); + } + if (field != null) { + result = getAnnotation(field.getAnnotations(), name); + if (result.isPresent()) + return result.get(); + } + if (setter != null) + return getAnnotation(setter.getAnnotations(), name).orElse(null); + return null; + } + + private Optional getAnnotation(Annotation[] annot, String name) { + return first(annot, a -> a.annotationType().getSimpleName().equals(name)); + } + public Type getGenericType(){ if (getter != null) return getter.getGenericReturnType(); diff --git a/core/src/main/java/org/jsonex/core/util/ClassUtil.java b/core/src/main/java/org/jsonex/core/util/ClassUtil.java index e3868be..2c975ca 100644 --- a/core/src/main/java/org/jsonex/core/util/ClassUtil.java +++ b/core/src/main/java/org/jsonex/core/util/ClassUtil.java @@ -21,8 +21,10 @@ import java.util.*; import java.util.function.Predicate; +import static org.jsonex.core.util.ArrayUtil.first; import static org.jsonex.core.util.ArrayUtil.indexOf; import static org.jsonex.core.util.ListUtil.listOf; +import static org.jsonex.core.util.StringUtil.lowerFirst; @SuppressWarnings("ALL") public class ClassUtil { @@ -88,7 +90,7 @@ public static Map getProperties(Class cls) { Map result = beanPropertyCache.get(cls); if (result == null) { synchronized(beanPropertyCache) { - result = _getProperties(cls); + result = getPropertiesNoCache(cls); beanPropertyCache.put(cls, result); } } @@ -97,55 +99,48 @@ public static Map getProperties(Class cls) { private static final WeakHashMap, Map> beanPropertyCache = new WeakHashMap<>(); - private static Map _getProperties(Class cls) { + private static Map getPropertiesNoCache(Class cls) { // Find all the getter/setter methods // Use TreeMap as method is not in stable order in JVM implementations, so we need to sort them to make stable order Map attributeMap = new TreeMap<>(); - for (Method m : getAllMethods(cls)) { + Method[] methods = getAllMethods(cls); + for (Method m : methods) { int mod = m.getModifiers(); if(!Modifier.isPublic(mod) || Modifier.isStatic(mod)) continue; //None public, or static, transient - String name = null; - boolean isSetter = false; - if (m.getName().startsWith("get")) - name = m.getName().substring(3); - else if(m.getName().startsWith("is")) { - if(m.getReturnType() != Boolean.class && m.getReturnType() != boolean.class) + for (BeanMethodType methodType: BeanMethodType.values()) { + String name = methodType.checkAndGetName(m); + if (name == null) continue; - name = m.getName().substring(2); - } else if(m.getName().startsWith("set")) { - isSetter = true; - name = m.getName().substring(3); - }else - continue; - - if (!isSetter && m.getParameterTypes().length != 0) - continue; - - if (isSetter && m.getParameterTypes().length != 1) - continue; - - if (name.length() == 0) // JSON doesn't allow empty key, we replace it with ^ - name = "^"; - name = StringUtil.lowerFirst(name); - if(name.equals("class")) - continue; + if (name.length() == 0) // JSON doesn't allow emptdy key, we replace it with ^ + name = "^"; - BeanProperty prop = attributeMap.get(name); - if(prop == null){ - prop = new BeanProperty(name); - attributeMap.put(name, prop); - } + name = lowerFirst(name); + if (name.equals("class")) + continue; - if(isSetter) - prop.setter = m; - else { - // For union type in certain framework, isXXX is to indicate if the attribute is available, we will override it - // with the actual getter method - if (prop.getter == null || prop.getter.getReturnType() == Boolean.TYPE) - prop.getter = m; + BeanProperty prop = attributeMap.computeIfAbsent(name, k -> new BeanProperty(k)); + switch (methodType) { + case has: prop.hasChecker = m; break; + case set: prop.setter = m; break; + case is: + if (prop.getter == null) + prop.getter = m; + else + prop.hasChecker = m ; + break; + case get: + if (prop.getter == null) + prop.getter = m; + else if (BeanMethodType.is.checkAndGetName(prop.getter) != null) { + // For union type in certain framework, isXXX is to indicate if the attribute is available, we will override it + // with the actual getter method + prop.hasChecker = prop.getter; + prop.getter = m; + } + } } } @@ -172,6 +167,14 @@ else if(m.getName().startsWith("is")) { BeanProperty prop = attributeMap.remove(name); if (prop == null) { prop = new BeanProperty(name); + final BeanProperty fProp = prop; + final String fName = name; + // For java 17 record feature, it didn't following javabean convention, + // the getter method name is the same as field name. + Optional method = first(methods, m -> + m.getName().equals(fName) && m.getParameterCount() == 0 + && Modifier.isPublic(m.getModifiers()) && m.getReturnType() == f.getType()); + method.ifPresent(m -> fProp.getter = m); } fieldMap.put(name, prop); prop.field = f; @@ -181,6 +184,24 @@ else if(m.getName().startsWith("is")) { return fieldMap; } + private static boolean isReturnBoolean(Method m) { + return m.getReturnType() != Boolean.class && m.getReturnType() != boolean.class; + } + + private enum BeanMethodType { + is { boolean valid(Method m) { return m.getParameterCount() == 0 && !isReturnBoolean(m); } }, + get { boolean valid(Method m) { return m.getParameterCount() == 0; } }, + has { boolean valid(Method m) { return m.getParameterCount() == 0 && !isReturnBoolean(m); } }, + set { boolean valid(Method m) { return m.getParameterCount() == 1; } } + ; + abstract boolean valid(Method m); + String checkAndGetName(Method m) { + if (!m.getName().startsWith(this.name())) + return null; + return valid(m) ? m.getName().substring(this.name().length()) : null; + } + } + /** * Get the object using a Object Path * Object path has following format @@ -228,7 +249,7 @@ public static Object getObjectByPath(@Nullable Class cls, @Nullable Object ob * It will try getter method first, then try field */ @SneakyThrows - public static @Nullable Object getPropertyValue(@Nullable Class cls, @Nullable Object obj, String propertyName) + public static @Nullable Object getPropertyValue(@Nullable Class cls, @Nullable Object obj, String propertyName) { if (cls == null && obj == null) return null; @@ -614,4 +635,43 @@ public static StackTraceElement findCallerStackTrace(Class calleeClass, Predi logger.error("Can't find the caller stack, return a dummy Stacktrace"); // Shouldn't happen return UNKNOWN_STACK_TRACE_ELEMENT; // Shouldn't reach here, return the DUMMY value just to avoid NPE for caller } + + static final int SYNTHETIC = 0x00001000; // Copied from Modifier as it's not accessible. + public static MethodWrapper findMethod(Class cls, String method, int numOfParam) { + return findMethod(cls, method, numOfParam, null); + } + + public static MethodWrapper findMethod(Class cls, String method, int numOfParam, @Nullable Class[] paramClasses) { + if(method.equals(MethodWrapper.METHOD_INIT)) { + for(Constructor c : cls.getConstructors()) { + if(c.getParameterTypes().length != numOfParam) + continue; + if(isParamCompatible(c.getParameterTypes(), paramClasses)) + return new MethodWrapper(c); + } + } else { + for (Method m : cls.getMethods()) { + if (m.getParameterTypes().length != numOfParam || !m.getName().equals(method)) + continue; + if ((SYNTHETIC & m.getModifiers()) != 0) + continue; + if (isParamCompatible(m.getParameterTypes(), paramClasses)) + return new MethodWrapper(m); + } + } + throw new IllegalArgumentException("Method Not Found: " + method + + " in class:" + cls.getName() + ", numOfParam:" + numOfParam + ", paramClasses:" + paramClasses); + } + + private static boolean isParamCompatible(Class[] paramClasses, Class[] formalClasses) { + if(formalClasses == null) + return true; + Assert.isTrue(paramClasses.length == formalClasses.length, + "arg.length:" + paramClasses.length + ",types: " + formalClasses.length); + for(int i=0; i R getIfInstanceOf( return obj != null && cls.isAssignableFrom(obj.getClass()) ? func.apply(cls.cast(obj)) : elseFunc.apply(obj); } - public static R getIfInstanceOfElseThrow( + @SneakyThrows + public static R getIfInstanceOfOrElse( + E obj, Class cls, Function func, Function elseFunc) { + if (obj != null && cls.isAssignableFrom(obj.getClass())) + return func.apply(cls.cast(obj)); + return elseFunc.apply(obj); + } + + public static R getIfInstanceOfOrElseThrow( Object obj, Class cls, Function func) { - return getIfInstanceOfElseThrow(obj, cls, func, () -> + return getIfInstanceOfOrElseThrow(obj, cls, func, () -> new IllegalStateException("Expect class: " + cls + ";got: " + safe(obj, Object::getClass))); } @SneakyThrows - public static R getIfInstanceOfElseThrow( + public static R getIfInstanceOfOrElseThrow( Object obj, Class cls, Function func, Supplier exp) { if (obj != null && cls.isAssignableFrom(obj.getClass())) return func.apply(cls.cast(obj)); throw exp.get(); } + @SneakyThrows + public static R getIfInstanceOfOrElseThrow( + E obj, Class cls, Function func, Function exp) { + if (obj != null && cls.isAssignableFrom(obj.getClass())) + return func.apply(cls.cast(obj)); + throw exp.apply(obj); + } + + public static T orElse(T1 value, T2 fullBack) { return value == null ? fullBack : value; } @@ -92,4 +110,31 @@ public static T orElse(T1 value, T2 fullBack) { public static T orElse(T1 value, Supplier fullBack) { return value == null ? fullBack.get() : value; } + + /** + * Simulate Javascript comma operator, run actions in sequence and return the last parameter + * This is useful to combine multiple statement into a single espresso, so that we can avoid code block in lambda + */ + public static T seq(Runnable action, T val) { + action.run(); + return val; + } + + public static T seq(Runnable action1, Runnable action2, T val) { + action1.run(); + action2.run(); + return val; + } + + /** + * Convert a block of code with local variables definitions into a single expression, it's useful to avoid code block + * in lambda statement. + */ + public static R with(T val, Function action) { + return action.apply(val); + } + + public static R with(T1 val1, T2 val2, BiFunction action) { + return action.apply(val1, val2); + } } diff --git a/core/src/main/java/org/jsonex/core/util/ListUtil.java b/core/src/main/java/org/jsonex/core/util/ListUtil.java index 2522a01..fe48c98 100644 --- a/core/src/main/java/org/jsonex/core/util/ListUtil.java +++ b/core/src/main/java/org/jsonex/core/util/ListUtil.java @@ -10,26 +10,38 @@ package org.jsonex.core.util; import org.jsonex.core.type.Identifiable; +import static org.jsonex.core.util.LangUtil.safe; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Predicate; -import static org.jsonex.core.util.LangUtil.safe; - /** - * A collection of utilities related to Collection classes. Many methods here are a better alternative + * A collection of utilities related to Collection classes. Many methods here are better alternatives * to Java8 stream with more concise expression * * Regarding null input, for most of the list transformation methods, it will keep as silent as possible. That - * means if you give null, I'll give back you null without NPE. The principle is I just don't make it worse, + * means if you give a null, I'll give back you null without NPE. The principle is I just don't make it worse, * but won't complain it's already bad, I'm just a transformer but not a validator. */ public class ListUtil { /** Map with index. Have to use a different name as Java type inference has difficult to distinguish BiFuncion and Function */ public static , TSrc, TDest> C mapWithIndex( - Collection source, BiFunction func, C dest) { + Iterable source, BiFunction func, C dest) { if (source != null) { int idx = 0; for (TSrc src : source) @@ -39,21 +51,21 @@ public static , TSrc, TDest> C mapWithIndex( } public static , TSrc, TDest> C map( - Collection source, Function func, C dest) { + Iterable source, Function func, C dest) { return mapWithIndex(source, (e, i) -> func.apply(e), dest); } public static List mapWithIndex( - Collection source, BiFunction func) { + Iterable source, BiFunction func) { return source == null ? null : mapWithIndex(source, func, new ArrayList<>()); } public static List map( - Collection source, Function func) { + Iterable source, Function func) { return source == null ? null : map(source, func, new ArrayList<>()); } - public static , TSrc, TDest> C flatMapWithIndex(Collection source, + public static , TSrc, TDest> C flatMapWithIndex(Iterable source, BiFunction> func, C dest) { if (source != null) { int idx = 0; @@ -67,22 +79,22 @@ public static , TSrc, TDest> C flatMapWithIn } public static , TSrc, TDest> C flatMap( - Collection source, Function> func, C dest) { + Iterable source, Function> func, C dest) { return flatMapWithIndex(source, (e, i) -> func.apply(e), dest); } - public static List flatMapWithIndex(Collection source, + public static List flatMapWithIndex(Iterable source, BiFunction> func) { return source == null ? null : flatMapWithIndex(source, func, new ArrayList<>()); } public static List flatMap( - Collection source, Function> func) { + Iterable source, Function> func) { return source == null ? null : flatMap(source, func, new ArrayList<>()); } public static Set unique( - Collection source, Function func) { + Iterable source, Function func) { return source == null ? null : map(source, func, new HashSet<>()); } @@ -91,7 +103,7 @@ public static List getIds(Collection> ide } public static Map> groupBy( - Collection source, Function classifier) { + Iterable source, Function classifier) { if (source == null) return null; Map> result = new LinkedHashMap<>(); @@ -100,13 +112,20 @@ public static Map> groupBy( return result; } + /** + * convert a Map to another Map by applying keyFunc and valFunc to convert key and values. If converted key is null + * the entry will be removed + */ public static Map map(Map source, Function keyFunc, Function valFunc) { if (source == null) return null; Map result = new HashMap<>(); - for (Map.Entry entry : source.entrySet()) - result.put(safe(entry.getKey(), keyFunc), safe(entry.getValue(), valFunc)); + for (Map.Entry entry : source.entrySet()) { + TK key = safe(entry.getKey(), keyFunc); + if (key != null) + result.put(key, safe(entry.getValue(), valFunc)); + } return result; } @@ -121,7 +140,7 @@ public static Map mapKeys( } /** - * The list should contain Long values. Otherwise ClassCastException will be thrown. + * The list should contain Long values, otherwise ClassCastException will be thrown. */ public static long[] toLongArray(Collection list) { if (list == null) @@ -143,23 +162,23 @@ else if (e instanceof Number) return result; } - public static Map toMap(Collection source, Function keyFunc) { + public static Map toMap(Iterable source, Function keyFunc) { return toMap(source, keyFunc, Function.identity()); } public static Map toMap( - Collection source, Function keyFunc, Function valFunc) { + Iterable source, Function keyFunc, Function valFunc) { return toMapInto(source, keyFunc, valFunc, new LinkedHashMap<>()); } // Have to use different name, as Java compile will confuse the overloaded methods with generics. public static > D toMapInto( - Collection source, Function keyFunc, D dest) { + Iterable source, Function keyFunc, D dest) { return toMapInto(source, keyFunc, i -> i, dest); } public static > D toMapInto( - Collection source, Function keyFunc, Function valFunc, D dest) { + Iterable source, Function keyFunc, Function valFunc, D dest) { if (source == null) return null; for (S s : source) @@ -167,7 +186,7 @@ public static Map toMap( return dest; } - public static , D extends Collection> D filter( + public static , D extends Collection> D filter( S source, Predicate pred, D dest) { if (source != null) for (V s : source) @@ -176,7 +195,7 @@ public static , D extends Collection> List filter(C source, Predicate pred) { + public static > List filter(C source, Predicate pred) { return source == null ? null : filter(source, pred, new ArrayList<>()); } @@ -233,19 +252,32 @@ public static boolean isIn(T match, T... values) { public static String join(T[] list, String delimiter) { return join(Arrays.asList(list), delimiter); } public static String join(Collection list, String delimiter) { StringBuilder sb = new StringBuilder(); + int i = 0; + for(Object obj : list) { + if(i++ > 0) + sb.append(delimiter); + sb.append(obj); + } + return sb.toString(); + } + + public static String join(T[] list, char delimiter) { return join(Arrays.asList(list), delimiter); } + public static String join(Collection list, char delimiter) { + StringBuilder sb = new StringBuilder(); + int i = 0; for(Object obj : list) { - if(sb.length() > 0) + if(i++ > 0) sb.append(delimiter); sb.append(obj); } return sb.toString(); } - public static > boolean exists(C source, Predicate pred) { + public static > boolean exists(C source, Predicate pred) { return source == null ? false : first(source, pred).isPresent(); } - public static > Optional first(C source, Predicate pred) { + public static > Optional first(C source, Predicate pred) { if (source != null) for (V s : source) if (pred.test(s)) @@ -261,7 +293,7 @@ public static > int indexOf(C source, Predicate return -1; } - public static , D extends Collection> D takeBetween( + public static , D extends Collection> D takeBetween( S source, Predicate dropPred, Predicate whilePred, D dest) { if (source != null) { boolean startTaking = false; @@ -278,26 +310,26 @@ public static , D extends Collection> List takeBetween( + public static > List takeBetween( S source, Predicate dropPred, Predicate whilePred) { return source == null ? null : takeBetween(source, dropPred, whilePred, new ArrayList<>()); } - public static , D extends Collection> D takeWhile( + public static , D extends Collection> D takeWhile( S source, Predicate pred, D dest) { return takeBetween(source, (v) -> false, pred, dest); } - public static > List takeWhile(S source, Predicate pred) { + public static > List takeWhile(S source, Predicate pred) { return source == null ? null : takeWhile(source, pred, new ArrayList<>()); } - public static , D extends Collection> D dropWhile( + public static , D extends Collection> D dropWhile( S source, Predicate pred, D dest) { return takeBetween(source, pred, v -> true, dest); } - public static > List dropWhile(S source, Predicate pred) { + public static > List dropWhile(S source, Predicate pred) { return dropWhile(source, pred, new ArrayList<>()); } @@ -307,16 +339,17 @@ public static Optional last(List list) { return list.isEmpty() ? Optional.empty() : Optional.of(list.get(list.size() - 1)); } - public static Optional first(Collection list) { + public static Optional first(Iterable list) { if (list == null) return Optional.empty(); - return list.isEmpty() ? Optional.empty() : Optional.of(list.iterator().next()); + Iterator it = list.iterator(); + return it.hasNext() ? Optional.of(list.iterator().next()) : Optional.empty(); } - public static boolean containsAny(Collection list, Collection elements) { + public static boolean containsAny(Iterable list, Collection elements) { if (list != null) - for (T e : elements) - if (list.contains(e)) + for (T e : list) + if (elements.contains(e)) return true; return false; } @@ -351,7 +384,11 @@ public static > L mutateAt(L list, int idx, T defaultVal, F return setAt(list, idx, mutator.apply(getOrDefault(list, idx, defaultVal))); } - /** build a copy of mutable Set whose content will be independent with original array once created */ + /** + * build a copy of mutable Set whose content will be independent with original array once created + * @Deprecated use {@link SetUtil#setOf(Object...)} instead + */ + @Deprecated public static Set setOf(T... e) { return e == null ? null : new LinkedHashSet<>(Arrays.asList(e)); } /** build a copy of mutable list whose content will be independent with original array once created */ @@ -370,4 +407,19 @@ public static , M extends Map> M mergeWith( return l1; }); } + + public static T reduce(Iterable source, T identity, BiFunction accumulate) { + T result = identity; + if (source != null) + for (V s : source) + result = accumulate.apply(result, s); + return result; + } + + public static T reduceTo(Iterable source, T result, BiConsumer accumulate) { + if (source != null) + for (V s : source) + accumulate.accept(result, s); + return result; + } } diff --git a/core/src/main/java/org/jsonex/core/util/MapBuilder.java b/core/src/main/java/org/jsonex/core/util/MapBuilder.java index 303a411..ab4d6bd 100644 --- a/core/src/main/java/org/jsonex/core/util/MapBuilder.java +++ b/core/src/main/java/org/jsonex/core/util/MapBuilder.java @@ -14,6 +14,10 @@ import java.util.LinkedHashMap; import java.util.Map; +/** + * A simple wrapper of Map to provide chainable put() method to support fluent coding style. This class is more of a wrapper + * instead of Builder + */ public class MapBuilder { @Getter final Map map = new LinkedHashMap<>(); public MapBuilder put(K key, V val) { @@ -22,4 +26,13 @@ public MapBuilder put(K key, V val) { } public MapBuilder() {} public MapBuilder(K key, V val) { put(key, val); } + public Map build() { return map; } + + @Deprecated // Use of instead + public static MapBuilder mapOf(K key, V val) { + return new MapBuilder<>(key, val); + } + public static MapBuilder of(K key, V val) { + return new MapBuilder<>(key, val); + } } \ No newline at end of file diff --git a/core/src/main/java/org/jsonex/core/util/MethodWrapper.java b/core/src/main/java/org/jsonex/core/util/MethodWrapper.java new file mode 100644 index 0000000..3b0e228 --- /dev/null +++ b/core/src/main/java/org/jsonex/core/util/MethodWrapper.java @@ -0,0 +1,90 @@ +package org.jsonex.core.util; + +import lombok.Getter; +import lombok.SneakyThrows; +// import org.springframework.core.LocalVariableTableParameterNameDiscoverer; + +import java.lang.reflect.*; + +/** + * Method wrapper to wrapper constructors and methods, so client will have consistent interface when invoke constructors or methods. + * It also collects the method parameter names. + */ +public class MethodWrapper { + public final static String METHOD_INIT = ""; + + // private static final LocalVariableTableParameterNameDiscoverer pnd = new LocalVariableTableParameterNameDiscoverer(); + @Getter final Method method; + @Getter final Constructor constructor; + @Getter final Class declaringClass; + @Getter final String name; + @Getter final Class returnClass; + @Getter final Type[] paramTypes; + @Getter final Class[] paramClasses; + @Getter final String[] paramNames; + @Getter final boolean isStatic; + + public MethodWrapper(Method _method){ + method = _method; + method.setAccessible(true); + constructor = null; + declaringClass = method.getDeclaringClass(); + name = method.getName(); + returnClass = method.getReturnType(); + paramTypes = method.getGenericParameterTypes(); + paramClasses = method.getParameterTypes(); + paramNames = getParameterNames(method.getParameters()); // pnd.getParameterNames(method); + isStatic = (method.getModifiers() & Modifier.STATIC) != 0; + } + + public MethodWrapper(Constructor _constructor) { + constructor = _constructor; + constructor.setAccessible(true); + method = null; + name = ""; + declaringClass = constructor.getDeclaringClass(); + returnClass = null; + paramTypes = constructor.getGenericParameterTypes(); + paramClasses = constructor.getParameterTypes(); + paramNames = getParameterNames(constructor.getParameters()); // pnd.getParameterNames(constructor); + isStatic = true; + } + + private String[] getParameterNames(Parameter[] parameters) { + return ArrayUtil.map(parameters, Parameter::getName, new String[0]); + } + + public String getSignature(boolean includeClassName) { + StringBuilder sb = new StringBuilder(); + if (returnClass != null) + sb.append(returnClass.getSimpleName() + " "); + if (includeClassName) + sb.append(declaringClass.getName() + "/"); + sb.append(name + "("); + for (int i=0; i 0) + sb.append(", "); + String paramName = paramNames != null ? paramNames[i] : ("arg" + i); + sb.append(paramClasses[i].getSimpleName() + " " + paramName); + } + sb.append(")"); + return sb.toString(); + } + + public String toString() { + return getSignature(true); + } + + public String getCanonicalName() { + return declaringClass.getCanonicalName() + "/" + name; + } + + @SneakyThrows + public Object invoke(Object obj, Object[] args) { + if (method != null) + return method.invoke(obj, args); + else + return constructor.newInstance(args); + } +} + diff --git a/core/src/main/java/org/jsonex/core/util/SetUtil.java b/core/src/main/java/org/jsonex/core/util/SetUtil.java new file mode 100644 index 0000000..917c2b2 --- /dev/null +++ b/core/src/main/java/org/jsonex/core/util/SetUtil.java @@ -0,0 +1,42 @@ +package org.jsonex.core.util; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Java building set functions didn't provide functional API that operation on set without side effect. + * This util class provide functional API for standard set operation without won't mutate the input set. + */ +public class SetUtil { + /** build a copy of mutable Set whose content will be independent with original array once created */ + public static Set setOf(T... e) { return e == null ? null : new LinkedHashSet<>(Arrays.asList(e)); } + + public static Set union(Set set1, Set set2) { + Set result = new HashSet<>(set1); + result.addAll(set2); + return result; + } + + public static Set difference(Set set1, Set set2) { + Set result = new HashSet<>(set1); + result.removeAll(set2); + return result; + } + + public static Set intersection(Set set1, Set set2) { + Set result = new HashSet<>(set1); + result.retainAll(set2); + return result; + } + + public static Set symmetricDifference(Set set1, Set set2) { + Set result = new HashSet<>(set1); + result.addAll(set2); + Set tmp = new HashSet<>(set1); + tmp.retainAll(set2); + result.removeAll(tmp); + return result; + } +} diff --git a/core/src/main/java/org/jsonex/core/util/StringUtil.java b/core/src/main/java/org/jsonex/core/util/StringUtil.java index 79a8a55..fb2fc7b 100644 --- a/core/src/main/java/org/jsonex/core/util/StringUtil.java +++ b/core/src/main/java/org/jsonex/core/util/StringUtil.java @@ -24,7 +24,21 @@ public class StringUtil { * Truncates the rightmost characters of a String to a desired length */ public static String getLeft(String src, int length) { return src == null || length > src.length() ? src : src.substring(0, length); } - + + public static String getLeft(String src, char sep) { + if (src == null) + return src; + int p = src.indexOf(sep); + return p < 0 ? src : src.substring(0, p); + } + + public static String getRight(String src, char sep) { + if (src == null) + return src; + int p = src.lastIndexOf(sep); + return p < 0 ? src : src.substring(p+1); + } + /** * Truncates the leftmost characters of a String to a desired length */ @@ -74,6 +88,14 @@ public static String fillString(String str, int length, char fillChar, boolean f return str + sb.toString(); } + public static String fillZero(String str,int length) { + return fillString(str,length,'0',true); + } + + public static String fillSpace(String str,int length) { + return fillString(str,length,' ',false); + } + private final static String C_ESC_CHAR = "'\"`\\\b\f\n\r\t"; private final static String C_ESC_SYMB = "'\"`\\bfnrt"; private final static char MIN_PRINTABLE_CHAR = ' '; @@ -200,5 +222,13 @@ public static StringBuilder appendRepeatedly(StringBuilder result, int times, ch public static String toTrimmedStr(Object o, int len) { return StringUtil.getLeft(String.valueOf(o), len); } public static String noNull(Object o) { return o == null ? "" : o.toString(); } + + public static int indexOfAnyChar(String str, String toMatch) { + for (int i = 0; i < str.length(); i++) { + if (toMatch.indexOf(str.charAt(i)) >= 0) + return i; + } + return -1; + } } diff --git a/core/src/main/java/org/jsonex/core/util/TwoWayMap.java b/core/src/main/java/org/jsonex/core/util/TwoWayMap.java new file mode 100644 index 0000000..0a68f22 --- /dev/null +++ b/core/src/main/java/org/jsonex/core/util/TwoWayMap.java @@ -0,0 +1,28 @@ +package org.jsonex.core.util; + +import lombok.Setter; + +import java.util.HashMap; +import java.util.Map; + +/** + * A two-way map which allow map values from key to value or reverse. This class provides default value if no match found + * and support chained put method. + */ +public class TwoWayMap { + private final Map map = new HashMap<>(); + private final Map reverseMap = new HashMap<>(); + @Setter private K defaultKey; + @Setter private V defaultValue; + + public static TwoWayMap of(K key, V value) { return new TwoWayMap().put(key, value); } + + public TwoWayMap put(K key, V value) { + map.put(key, value); + reverseMap.put(value, key); + return this; + } + + public V get(K key) { return key == null ? null : map.getOrDefault(key, defaultValue); } + public K getKey(V value) { return value == null ? null : reverseMap.getOrDefault(value, defaultKey); } +} \ No newline at end of file diff --git a/core/src/test/java/org/jsonex/core/charsource/BaseCharSourceTest.java b/core/src/test/java/org/jsonex/core/charsource/BaseCharSourceTest.java index cbc9a6a..7145e87 100644 --- a/core/src/test/java/org/jsonex/core/charsource/BaseCharSourceTest.java +++ b/core/src/test/java/org/jsonex/core/charsource/BaseCharSourceTest.java @@ -33,7 +33,7 @@ public abstract class BaseCharSourceTest { assertEquals(2, cs.getPos()); StringBuilder target = new StringBuilder(); - assertTrue("should match /*", cs.readUntilMatch("/*", false, target, 0, 1000)); + assertTrue("should match /*", cs.readUntilMatch(target, "/*", false, 0, 1000)); assertEquals("Text before ", target.toString()); assertTrue("should start with /*", cs.startsWith("/*")); cs.skip(2); // skip /* diff --git a/core/src/test/java/org/jsonex/core/factory/InjectableFactoryTest.java b/core/src/test/java/org/jsonex/core/factory/InjectableFactoryTest.java index 88ae8f0..19548e2 100644 --- a/core/src/test/java/org/jsonex/core/factory/InjectableFactoryTest.java +++ b/core/src/test/java/org/jsonex/core/factory/InjectableFactoryTest.java @@ -46,7 +46,7 @@ public static class P3 {} assertEquals(C.class, i1Fact.get().getClass()); assertNotSame(i1Fact.get(), i1Fact.get()); - i1Fact = InjectableFactory._0.of(C0::new, CacheGlobal.get()); + i1Fact = InjectableFactory._0.of(C0::new, ScopeGlobal.get()); assertEquals(C0.class, i1Fact.get().getClass()); assertSame(i1Fact.get(), i1Fact.get()); @@ -67,7 +67,7 @@ public static class P3 {} } @Test public void testThreadLocalCache() throws InterruptedException { - final InjectableFactory._0 fact = InjectableFactory._0.of(C::new, CacheThreadLocal.get()); + final InjectableFactory._0 fact = InjectableFactory._0.of(C::new, ScopeThreadLocal.get()); final I1[] objs = new I1[3]; objs[0] = fact.get(); Thread thread = new Thread(() -> { @@ -81,7 +81,7 @@ public static class P3 {} } @Test public void testSetInstance() { - InjectableFactory i1Fact = InjectableFactory.of((p) -> new C0(), CacheGlobal.get()); + InjectableFactory i1Fact = InjectableFactory.of((p) -> new C0(), ScopeGlobal.get()); I1 inst = new I1() {}; i1Fact.setInstance(inst); assertEquals(inst, i1Fact.get()); diff --git a/core/src/test/java/org/jsonex/core/type/TupleTest.java b/core/src/test/java/org/jsonex/core/type/TupleTest.java new file mode 100644 index 0000000..da49a78 --- /dev/null +++ b/core/src/test/java/org/jsonex/core/type/TupleTest.java @@ -0,0 +1,14 @@ +package org.jsonex.core.type; + +import org.junit.Test; + +import java.util.Date; +import static org.junit.Assert.assertEquals; + +public class TupleTest { + @Test + public void testTuple() { + Tuple.Tuple3 tpl = Tuple.of("string", new Date(), 100); + assertEquals("string", tpl._0); + } +} diff --git a/core/src/test/java/org/jsonex/core/type/UnionTest.java b/core/src/test/java/org/jsonex/core/type/UnionTest.java new file mode 100644 index 0000000..6390929 --- /dev/null +++ b/core/src/test/java/org/jsonex/core/type/UnionTest.java @@ -0,0 +1,17 @@ +package org.jsonex.core.type; + +import org.junit.Assert; +import org.junit.Test; + +public class UnionTest { + @Test + public void testUnion() { + Union.Union2 str = Union.Union2.of_0("string"); + Assert.assertEquals(String.class, str.getType()); + Assert.assertEquals("string", str._0); + + Union.Union2 num = Union.Union2.of_1(1); + Assert.assertEquals(Integer.class, num.getType()); + Assert.assertEquals((Integer)1, num._1); + } +} diff --git a/core/src/test/java/org/jsonex/core/util/ArrayUtilTest.java b/core/src/test/java/org/jsonex/core/util/ArrayUtilTest.java new file mode 100644 index 0000000..ec590b9 --- /dev/null +++ b/core/src/test/java/org/jsonex/core/util/ArrayUtilTest.java @@ -0,0 +1,52 @@ +package org.jsonex.core.util; + +import org.jsonex.core.util.ListUtilTest.TestCls; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Optional; + +import static org.jsonex.core.type.Operator.eq; +import static org.jsonex.core.util.ListUtilTest.TestCls.F_TYPE; +import static org.junit.Assert.*; + +public class ArrayUtilTest { + private TestCls[] buildArray() { + return new TestCls[]{ + new TestCls(0, null, 0, Arrays.asList("a", "b")), + new TestCls(1, "name1", 1, Arrays.asList("c", "d", "e")), + new TestCls(2, null, 2, Arrays.asList()), + new TestCls(3, "name3", 2, null), + }; + } + + @Test public void testMap() { + assertArrayEquals(new String[]{"1", "2", "3"}, ArrayUtil.map(new Integer[]{ 1, 2, 3 }, i -> i + "", new String[0])); + } + + @Test public void testBox() { + assertArrayEquals(new Integer[]{1, 2, 3}, ArrayUtil.box(new int[]{ 1, 2, 3 })); + assertArrayEquals(new int[]{1, 2, 3}, ArrayUtil.unbox(new Integer[]{ 1, 2, 3 })); + } + + @Test public void testSubArray() { + assertArrayEquals(new Integer[]{2, 3}, ArrayUtil.subArray(new Integer[]{ 1, 2, 3 }, -2)); + assertArrayEquals(new Integer[]{2}, ArrayUtil.subArray(new Integer[]{ 1, 2, 3 }, 1, 1)); + } + + @Test public void testFirstLastIndexOf() { + TestCls[] list = buildArray(); + assertEquals(list[2], ArrayUtil.first(list, eq(F_TYPE, 2)).get()); + assertEquals(2, ArrayUtil.indexOf(list, eq(F_TYPE, 2))); + + assertEquals(Optional.empty(), ArrayUtil.first(list, eq(F_TYPE, 3))); + assertEquals(Optional.empty(), ArrayUtil.first(null, eq(F_TYPE, 3))); + assertEquals(-1, ArrayUtil.indexOf(null, eq(F_TYPE, 2))); + assertTrue(ArrayUtil.contains(list, list[2])); + } + + @Test public void testReduce() { + String[] str = {"Hello", "world"}; + assertEquals("Hello world ", ArrayUtil.reduce(str, "", (sum, item) -> sum + item + " ")); + } +} diff --git a/core/src/test/java/org/jsonex/core/util/ClassUtilTest.java b/core/src/test/java/org/jsonex/core/util/ClassUtilTest.java index 5bf03ad..50fc5a5 100644 --- a/core/src/test/java/org/jsonex/core/util/ClassUtilTest.java +++ b/core/src/test/java/org/jsonex/core/util/ClassUtilTest.java @@ -25,13 +25,7 @@ import java.lang.reflect.Modifier; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; -import java.util.Arrays; -import java.util.Collection; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.TimeUnit; import static org.junit.Assert.*; @@ -64,6 +58,17 @@ public static class B extends A { @Nullable @Transient public void setWriteOnly(String str) { fieldB5 = str; } public String getReadOnly() { return fieldB1; } + + public boolean hasFieldWithHasCheck() { return false; }; + public String getFieldWithHasCheck() { throw new IllegalStateException("fieldWithHasCheck is not available"); } + + public boolean isFieldWithUnionCheck() { return false; }; + public String getFieldWithUnionCheck() { throw new IllegalStateException("fieldWithUnionCheck is not available"); } + + // To support Java 17 Record feature + private String fieldWithSameGetterMethodName; + public String fieldWithSameGetterMethodName() { return fieldWithSameGetterMethodName; } + } @SuppressWarnings({"CanBeFinal", "SameReturnValue", "WeakerAccess"}) @@ -80,9 +85,11 @@ public void setMethodSetOnly(String str) { @Test public void testGetProperties() { Map properties = ClassUtil.getProperties(B.class); log.info("Properties.keySet():" + properties.keySet()); - String[] exp = {"fieldA1", "fieldA2", "fieldA3", "fieldA4", "fieldB1", "fieldB2", "fieldB3", "fieldB4", "fieldB5", "fieldB6", "readOnly", "writeOnly"}; - // Java compiler will mass up the order of the getter methods, field order is preserved in most of the java versions - assertArrayEquals(exp, properties.keySet().toArray()); + String[] exp = {"fieldA1", "fieldA2", "fieldA3", "fieldA4", "fieldB1", "fieldB2", "fieldB3", "fieldB4", "fieldB5", "fieldB6", + "fieldWithSameGetterMethodName", "fieldWithHasCheck", "fieldWithUnionCheck", "readOnly", "writeOnly"}; + // Java compiler will mess up the order of the getter methods, field order is preserved in most of the java versions + // assertArrayEquals(exp, properties.keySet().toArray()); // This will fail + assertEquals(ListUtil.setOf(exp), properties.keySet()); // Field with setter/getters BeanProperty prop = properties.get("fieldB6"); @@ -125,6 +132,14 @@ public void setMethodSetOnly(String str) { assertNull(prop.getAnnotation(DefaultEnum.class)); assertEquals(Modifier.PUBLIC, prop.getModifier()); assertTrue(prop.isTransient()); + + // Has checker + assertNull(properties.get("fieldWithHasCheck").get(b)); + assertNull(properties.get("fieldWithUnionCheck").get(b)); + + // Support getter with the same name as field name to support java 17 Record pattern + prop = properties.get("fieldWithSameGetterMethodName"); + assertEquals("fieldWithSameGetterMethodName", prop.getter.getName()); } @Test public void testGetPropertiesWithExceptions () { @@ -278,7 +293,18 @@ static class GenericTypeTestCls { } @Test public void testObjectToSimpleType() { - assertEquals((Integer)100, ClassUtil.objectToSimpleType(100, Integer.class)); - assertEquals((Float)(float)100.1, ClassUtil.objectToSimpleType(100.1, float.class)); + assertEquals(100, ClassUtil.objectToSimpleType(100, Integer.class)); + assertEquals(100.1f, ClassUtil.objectToSimpleType(100.1, float.class)); + } + + @SneakyThrows + @Test public void testFindMethod() { + MethodWrapper mw = ClassUtil.findMethod(HashMap.class, "", 2, null); + assertEquals(HashMap.class.getConstructor(int.class, float.class), mw.getConstructor()); + assertEquals(new HashMap<>(), mw.invoke(null, new Object[]{1, 0.7f})); + assertEquals("java.util.HashMap/(int arg0, float arg1)", mw.toString()); + + mw = ClassUtil.findMethod(HashMap.class, "", 1, new Class[]{TreeMap.class}); + assertEquals(HashMap.class.getConstructor(Map.class), mw.getConstructor()); } } diff --git a/core/src/test/java/org/jsonex/core/util/LangUtilTest.java b/core/src/test/java/org/jsonex/core/util/LangUtilTest.java index 09a559a..588221e 100644 --- a/core/src/test/java/org/jsonex/core/util/LangUtilTest.java +++ b/core/src/test/java/org/jsonex/core/util/LangUtilTest.java @@ -57,4 +57,17 @@ static class C { LangUtil.doIfInstanceOf(col, List.class, list -> list.set(2, 0)); assertEquals(ListUtil.setOf(1,2,3), col1); } + + @Test public void testSeq() { + final int[] i = new int[]{0}; + Integer val = 123; + assertEquals(val, LangUtil.seq(() -> i[0] += 1, val)); + assertEquals(val, LangUtil.seq(() -> i[0] += 1, () -> i[0] += 1, val)); + assertEquals(3, i[0]); + } + + @Test public void testWith() { + assertEquals("abcd", LangUtil.with("abc", a -> a + "d")); + assertEquals("abcd", LangUtil.with("abc", "d", (a, b) -> a + b)); + } } diff --git a/core/src/test/java/org/jsonex/core/util/ListUtilTest.java b/core/src/test/java/org/jsonex/core/util/ListUtilTest.java index 5ec46be..b816a6a 100644 --- a/core/src/test/java/org/jsonex/core/util/ListUtilTest.java +++ b/core/src/test/java/org/jsonex/core/util/ListUtilTest.java @@ -9,22 +9,39 @@ package org.jsonex.core.util; -import org.jsonex.core.type.BeanField; -import org.jsonex.core.type.Identifiable; -import org.jsonex.core.type.Operator; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; import lombok.experimental.ExtensionMethod; +import org.jsonex.core.type.BeanField; +import org.jsonex.core.type.Identifiable; +import org.jsonex.core.type.Operator; +import static org.jsonex.core.type.Operator.eq; +import static org.jsonex.core.type.Operator.ge; +import static org.jsonex.core.type.Operator.in; +import static org.jsonex.core.type.Operator.not; +import static org.jsonex.core.type.Operator.safeOf; +import static org.jsonex.core.util.ListUtil.listOf; +import static org.jsonex.core.util.ListUtilTest.TestCls.F_ID; +import static org.jsonex.core.util.ListUtilTest.TestCls.F_NAME; +import static org.jsonex.core.util.ListUtilTest.TestCls.F_TAGS; +import static org.jsonex.core.util.ListUtilTest.TestCls.F_TYPE; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; import org.junit.Test; -import java.util.*; - -import static org.jsonex.core.type.Operator.*; -import static org.jsonex.core.util.ListUtil.listOf; -import static org.jsonex.core.util.ListUtilTest.TestCls.*; -import static org.junit.Assert.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; @ExtensionMethod({Operator.class, ListUtil.class}) public class ListUtilTest { @@ -198,7 +215,11 @@ private List buildList() { assertEquals(-1, ListUtil.indexOf(null, eq(F_TYPE, 2))); } - @Test public void testJoin() { assertEquals("1,2,3", ListUtil.join(new Integer[]{1,2,3}, ",")); } + @Test public void testJoin() { + assertEquals("1,2,3", ListUtil.join(new Integer[]{1,2,3}, ",")); + assertEquals(",,3", ListUtil.join(new Object[]{"","",3}, ",")); + assertEquals(",,3", ListUtil.join(new Object[]{"","",3}, ',')); + } @Test public void testRemoveLast() { List list = new ArrayList<>(asList(1,2,3)); @@ -252,4 +273,10 @@ private List buildList() { assertEquals(result, ListUtil.mergeWith(target, source)); } + + @Test public void testReduce() { + List str = ListUtil.listOf("Hello", "world"); + assertEquals("Hello world ", ListUtil.reduce(str, "", (sum, item) -> sum + item + " ")); + assertEquals("Hello world ", ListUtil.reduceTo(str, new StringBuilder(), (sum, item) -> sum.append(item + " ")).toString()); + } } diff --git a/core/src/test/java/org/jsonex/core/util/SetUtilTest.java b/core/src/test/java/org/jsonex/core/util/SetUtilTest.java new file mode 100644 index 0000000..ced6cc9 --- /dev/null +++ b/core/src/test/java/org/jsonex/core/util/SetUtilTest.java @@ -0,0 +1,37 @@ +package org.jsonex.core.util; + +import static org.jsonex.core.util.ListUtil.listOf; +import static org.jsonex.core.util.SetUtil.setOf; +import static org.jsonex.core.util.SetUtil.difference; +import static org.jsonex.core.util.SetUtil.intersection; +import static org.jsonex.core.util.SetUtil.symmetricDifference; +import static org.jsonex.core.util.SetUtil.union; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import org.junit.Test; + +import java.util.Set; + +public class SetUtilTest { + private static Set SET1 = setOf(1,2,3); + private static Set SET2 = setOf(2,3,4); + + @Test public void testSetOf() { + assertTrue("set should contain all the elements", setOf(1,2,3).containsAll(listOf(1,2,3))); } + + @Test public void testUnion() { + assertEquals(union(SET1, SET2), setOf(1,2,3,4)); + } + + @Test public void testDifference() { + assertEquals(difference(SET1, SET2), setOf(1)); + } + + @Test public void testIntersection() { + assertEquals(intersection(SET1, SET2), setOf(2, 3)); + } + + @Test public void testSymmetricDifference() { + assertEquals(symmetricDifference(SET1, SET2), setOf(1, 4)); + } +} diff --git a/core/src/test/java/org/jsonex/core/util/StringUtilTest.java b/core/src/test/java/org/jsonex/core/util/StringUtilTest.java index 79e424f..f541a14 100644 --- a/core/src/test/java/org/jsonex/core/util/StringUtilTest.java +++ b/core/src/test/java/org/jsonex/core/util/StringUtilTest.java @@ -30,6 +30,9 @@ public class StringUtilTest { assertEquals("234", StringUtil.getRight("1234", 3)); assertEquals("1234", StringUtil.getRight("1234", 5)); + + assertEquals("abc", StringUtil.getLeft("abc=def=ghi", '=')); + assertEquals("ghi", StringUtil.getRight("abc=def=ghi", '=')); } @Test public void testIsDigitOnly() { @@ -45,6 +48,8 @@ public class StringUtilTest { @Test public void testFillString() { assertEquals("12 ", StringUtil.fillString("12", 4, ' ', false)); assertEquals(" 12", StringUtil.fillString("12", 4, ' ', true)); + assertEquals("12 ", StringUtil.fillSpace("12", 4)); + assertEquals("0012", StringUtil.fillZero("12", 4)); } @Test public void testCEscape() { @@ -90,4 +95,9 @@ public class StringUtilTest { @Test public void testToTrimmedStr() { assertEquals("12", StringUtil.toTrimmedStr("1234", 2)); } + + @Test public void testIndexOfAnyChar() { + assertEquals(1, StringUtil.indexOfAnyChar("1234", "234")); + assertEquals(-1, StringUtil.indexOfAnyChar("1234", "567")); + } } diff --git a/core/src/test/java/org/jsonex/core/util/TestClass.java b/core/src/test/java/org/jsonex/core/util/TestClass.java index d8fc300..0d23f33 100644 --- a/core/src/test/java/org/jsonex/core/util/TestClass.java +++ b/core/src/test/java/org/jsonex/core/util/TestClass.java @@ -16,7 +16,6 @@ public class TestClass extends ArrayList implements Comparable { public static class TestSubClass extends TestClass { - } Map stringIntMap; diff --git a/core/src/test/java/org/jsonex/core/util/TimeProviderTest.java b/core/src/test/java/org/jsonex/core/util/TimeProviderTest.java index 35b3f33..6adbab6 100644 --- a/core/src/test/java/org/jsonex/core/util/TimeProviderTest.java +++ b/core/src/test/java/org/jsonex/core/util/TimeProviderTest.java @@ -10,18 +10,27 @@ package org.jsonex.core.util; import org.jsonex.core.factory.TimeProvider; +import org.junit.After; import org.junit.Test; import java.util.Date; +import java.util.concurrent.TimeUnit; import static junit.framework.Assert.assertEquals; public class TimeProviderTest { @Test public void testTimeProvider() { TimeProvider.get().getDate(); // Warm up, so that following test could pass - // Flaky testing, but not likely fail. + TimeProvider.get().getNanoTime(); // Warm up Clock Instance as the class loader will take 20ms which will break test + // Flaky testing only if CPU is super slow assertEquals(new Date(), TimeProvider.get().getDate()); assertEquals(System.currentTimeMillis(), TimeProvider.get().getTimeMillis()); + assertEquals(System.currentTimeMillis(), TimeProvider.get().getNanoTime() / 1_000_000); + assertEquals(System.currentTimeMillis() + 1000, TimeProvider.now(1, TimeUnit.SECONDS)); + assertEquals(System.currentTimeMillis() - 2000, TimeProvider.now(-2000)); + assertEquals(TimeProvider.get().getTimeMillis(), TimeProvider.millis()); + // assertEquals(TimeProvider.get().getNanoTime(), TimeProvider.nano()); // Won't equals + assertEquals(TimeProvider.get().getTimeMillis() - 1000, TimeProvider.duration(1000)); } @Test public void testMock() { @@ -30,9 +39,12 @@ public class TimeProviderTest { mock.setTimeMillis(1000); assertEquals(1000, TimeProvider.get().getTimeMillis()); - mock.add(1000); + mock.sleepMs(1000); assertEquals(2000, TimeProvider.get().getTimeMillis()); assertEquals(2000, TimeProvider.get().getDate().getTime()); + } + + @After public void after() { TimeProvider.it.reset(); } } diff --git a/core/src/test/java/org/jsonex/core/util/TwoWayMapTest.java b/core/src/test/java/org/jsonex/core/util/TwoWayMapTest.java new file mode 100644 index 0000000..432c970 --- /dev/null +++ b/core/src/test/java/org/jsonex/core/util/TwoWayMapTest.java @@ -0,0 +1,17 @@ +package org.jsonex.core.util; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class TwoWayMapTest { + @Test public void testGet() { + TwoWayMap map = TwoWayMap.of("a", "A").put("b", "B").setDefaultKey("+").setDefaultValue("-"); + assertEquals("A", map.get("a")); + assertEquals("b", map.getKey("B")); + assertNull(map.getKey(null)); + assertEquals("+", map.getKey("C")); + assertEquals("-", map.get("c")); + } +} diff --git a/csv/pom.xml b/csv/pom.xml index 1fdf963..cb39418 100644 --- a/csv/pom.xml +++ b/csv/pom.xml @@ -6,7 +6,7 @@ org.jsonex jcParent - 0.1.21 + 0.1.27 ../pom.xml csv diff --git a/csv/src/main/java/org/jsonex/csv/CSVOption.java b/csv/src/main/java/org/jsonex/csv/CSVOption.java index 7b658f5..6f599f6 100644 --- a/csv/src/main/java/org/jsonex/csv/CSVOption.java +++ b/csv/src/main/java/org/jsonex/csv/CSVOption.java @@ -1,17 +1,45 @@ package org.jsonex.csv; +import lombok.AccessLevel; import lombok.Data; import lombok.Getter; +import lombok.Setter; import lombok.experimental.Accessors; @Data @Accessors(chain = true) public class CSVOption { + boolean includeHeader = true; char fieldSep = ','; char recordSep = '\n'; char quoteChar = '"'; - @Getter(lazy = true) private final String fieldAndRecord = "" + fieldSep + recordSep; - @Getter(lazy = true) private final String fieldSepStr = "" + fieldSep; - @Getter(lazy = true) private final String quoteCharStr = "" + quoteChar; - @Getter(lazy = true) private final String recordSepStr = "" + recordSep; + @Getter(value= AccessLevel.NONE) @Setter(value=AccessLevel.NONE) String _fieldAndRecord; + @Getter(value= AccessLevel.NONE) @Setter(value=AccessLevel.NONE) String _fieldSepStr; + @Getter(value= AccessLevel.NONE) @Setter(value=AccessLevel.NONE) String _quoteCharStr; + @Getter(value= AccessLevel.NONE) @Setter(value=AccessLevel.NONE) String _recordSepStr; + + { buildTerms(); } + + public CSVOption setFieldSep(char fieldSep) { + this.fieldSep = fieldSep; + return buildTerms(); + } + + public CSVOption setRecordSep(char recordSep) { + this.recordSep = recordSep; + return buildTerms(); + } + + public CSVOption setQuoteChar(char quoteChar) { + this.quoteChar = quoteChar; + return buildTerms(); + } + + private CSVOption buildTerms() { + _fieldAndRecord = "" + fieldSep + recordSep; + _fieldSepStr = "" + fieldSep; + _quoteCharStr = "" + quoteChar; + _recordSepStr = "" + recordSep; + return this; + } } diff --git a/csv/src/main/java/org/jsonex/csv/CSVParser.java b/csv/src/main/java/org/jsonex/csv/CSVParser.java index 0dbbec5..981309f 100644 --- a/csv/src/main/java/org/jsonex/csv/CSVParser.java +++ b/csv/src/main/java/org/jsonex/csv/CSVParser.java @@ -1,12 +1,19 @@ package org.jsonex.csv; -import org.jsonex.core.charsource.*; +import org.jsonex.core.charsource.ArrayCharSource; +import org.jsonex.core.charsource.Bookmark; +import org.jsonex.core.charsource.CharSource; +import org.jsonex.core.charsource.ReaderCharSource; import org.jsonex.core.factory.InjectableInstance; import org.jsonex.core.util.ClassUtil; +import static org.jsonex.core.util.ListUtil.map; import org.jsonex.treedoc.TDNode; import org.jsonex.treedoc.TreeDoc; import java.io.Reader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; public class CSVParser { private static String SPACE_CHARS = " \r"; @@ -20,23 +27,44 @@ public class CSVParser { public TDNode parse(String str, CSVOption opt) { return parse(new ArrayCharSource(str), opt); } public TDNode parse(CharSource src, CSVOption opt) { return parse(src, opt, new TreeDoc(null).getRoot()); } public TDNode parse(CharSource src, CSVOption opt, TDNode root) { + List fields = null; root.setType(TDNode.Type.ARRAY); + if (opt.includeHeader) { + fields = map(readNonEmptyRecord(src, opt), Object::toString); + if (fields.isEmpty()) + return root; + if (fields.get(0).equals(TDNode.COLUMN_KEY)) + root.setType(TDNode.Type.MAP); + } + while (!src.isEof()) { if (!src.skipChars(SPACE_CHARS)) break; - readRecord(src, opt, root); + readRecord(src, opt, root, fields); } return root; } - void readRecord(CharSource src, CSVOption opt, TDNode root) { - TDNode row = new TDNode(root.getDoc(), null).setType(TDNode.Type.ARRAY); + void readRecord(CharSource src, CSVOption opt, TDNode root, List fields) { + TDNode row = new TDNode(root.getDoc(), null).setType(fields == null ? TDNode.Type.ARRAY: TDNode.Type.MAP); row.setStart(src.getBookmark()); + int i = 0; while (!src.isEof() && src.peek() != opt.recordSep) { if (!src.skipChars(SPACE_CHARS)) break; Bookmark start = src.getBookmark(); - TDNode field = row.createChild().setValue(readField(src, opt)); + Object val = readField(src, opt); + String key = null; + if (fields != null) { + if (i >= fields.size()) + throw src.createParseRuntimeException("The row has more columns than headers"); + key = fields.get(i++); + if (key.equals(TDNode.COLUMN_KEY)) { + row.setKey(val.toString()); + continue; + } + } + TDNode field = row.createChild(key).setValue(val); field.setStart(start).setEnd(src.getBookmark()); } row.setEnd(src.getBookmark()); @@ -44,37 +72,63 @@ void readRecord(CharSource src, CSVOption opt, TDNode root) { root.addChild(row); if (!src.isEof()) src.read(); // Skip the recordSep + } + public List readNonEmptyRecord(CharSource src, CSVOption opt) { + while(!src.isEof()) { + List res = readRecord(src, opt); + if (!res.isEmpty()) + return res; + } + return Collections.emptyList(); + } + + public List readRecord(CharSource src, CSVOption opt) { + List result = new ArrayList<>(); + while (!src.isEof() && src.peek() != opt.recordSep) { + if (!src.skipChars(SPACE_CHARS)) + break; + result.add(readField(src, opt)); + } + if (!src.isEof()) + src.read(); // Skip the recordSep + return result; } Object readField(CharSource src, CSVOption opt) { StringBuilder sb = new StringBuilder(); - boolean previousQuoted = false; - boolean isString = false; - while (!src.isEof() && src.peek() != opt.fieldSep && src.peek() != opt.recordSep) { - if (src.peek() == opt.quoteChar) { - isString = true; - if (previousQuoted) - sb.append(opt.quoteChar); + if (src.isEof()) + return sb.toString(); + boolean isString = false; + if (src.peek() != opt.quoteChar) { // Read non-quoted string + sb.append(src.readUntil(opt._fieldAndRecord).trim()); + } else { // Read quoted string + isString = true; + src.skip(); + while (!src.isEof()) { // Not calling getBookmark() to avoid clone an object int pos = src.bookmark.getPos(); int line = src.bookmark.getLine(); int col = src.bookmark.getCol(); - src.skip(); // for "", we will keep one quote - src.readUntil(opt.getQuoteCharStr(), sb); + src.readUntil(sb, opt._quoteCharStr); + if (src.isEof()) + throw src.createParseRuntimeException("Can't find matching quote at position:" + pos + ";line:" + line + ";col:" + col); + src.skip(); if (src.isEof()) - throw new EOFRuntimeException("Can't find matching quote at position:" + pos + ";line:" + line + ";col:" + col); - if (src.peek() == opt.quoteChar) + break; + if (src.peek() == opt.quoteChar) { + sb.append(opt.quoteChar); src.skip(); - previousQuoted = true; - } else { - sb.append(src.readUntil(opt.getFieldAndRecord()).trim()); - previousQuoted = false; + } else { + break; + } } + src.skipChars(" \t"); } + if (!src.isEof() && src.peek() == opt.fieldSep) src.skip(); // Skip fieldSep diff --git a/csv/src/main/java/org/jsonex/csv/CSVWriter.java b/csv/src/main/java/org/jsonex/csv/CSVWriter.java index f3a4c4a..043a726 100644 --- a/csv/src/main/java/org/jsonex/csv/CSVWriter.java +++ b/csv/src/main/java/org/jsonex/csv/CSVWriter.java @@ -1,9 +1,14 @@ package org.jsonex.csv; +import lombok.SneakyThrows; import org.jsonex.core.factory.InjectableInstance; import org.jsonex.core.util.ClassUtil; +import static org.jsonex.core.util.ListUtil.join; +import static org.jsonex.core.util.ListUtil.map; import org.jsonex.treedoc.TDNode; -import lombok.SneakyThrows; + +import java.util.Collection; +import java.util.List; public class CSVWriter { public final static InjectableInstance instance = InjectableInstance.of(CSVWriter.class); @@ -12,41 +17,55 @@ public class CSVWriter { public String writeAsString(TDNode node) { return writeAsString(node, new CSVOption()); } public String writeAsString(TDNode node, CSVOption opt) { return write(new StringBuilder(), node, opt).toString(); } - @SneakyThrows public T write(T out, TDNode node, CSVOption opt) { - if (node.getChildren() != null) { - for (TDNode row : node.getChildren()) { - if (row.getChildren() != null) { - for (TDNode field : row.getChildren()) { - writeField(out, field, opt); - out.append(opt.getFieldSepStr()); - } - out.append(opt.getRecordSepStr()); - } - } + if (!opt.includeHeader) { + writeRecords(out, node.childrenValueAsListOfList(), opt); + } else { + List keys = node.getChildrenKeys(); + if (node.getType() == TDNode.Type.ARRAY && !keys.isEmpty()) + keys.remove( 0); // Remove array index key + append(out, encodeRecord(keys, opt), opt._recordSepStr); + writeRecords(out, node.childrenValueAsListOfList(keys), opt); } return out; } + public > T writeRecords(T out, Collection records, CSVOption opt) { + records.forEach(r -> append(out, encodeRecord(r, opt), opt._recordSepStr)); + return out; + } + @SneakyThrows - private T writeField(T out, TDNode field, CSVOption opt) { - String quote = opt.getQuoteCharStr(); - String str = "" + field.getValue(); - if (needQuote(field, opt)) { + private void append(T out, String... strs) { + for (String s : strs) + out.append(s); + } + + public String encodeRecord(Collection fields, CSVOption opt) { + return join(map(fields, f -> encodeField(f, opt)), opt.fieldSep); + } + + private String encodeField(Object field, CSVOption opt) { + if (field == null) + return ""; + String quote = opt._quoteCharStr; + String str = String.valueOf(field); + if (str.isEmpty()) + return str; + if (needQuote(field, str, opt)) { if (str.contains(quote)) str = str.replace(quote, quote + quote); - return (T) out.append(quote).append(str).append(quote); + return quote + str + quote; } - return (T) out.append(str); + return str; } - private static boolean needQuote(TDNode field, CSVOption opt) { - if (!(field.getValue() instanceof String)) + private static boolean needQuote(Object field, String str, CSVOption opt) { + if (str.isEmpty()) return false; - String str = (String)field.getValue(); - return str.contains(opt.getQuoteCharStr()) - || str.contains(opt.getFieldSepStr()) - || str.contains(opt.getRecordSepStr()) - || ClassUtil.toSimpleObject(str) != str; + return str.charAt(0) == opt.getQuoteChar() + || str.contains(opt._fieldSepStr) + || str.contains(opt._recordSepStr) + || (field instanceof String && !(ClassUtil.toSimpleObject(str) instanceof String)); } } diff --git a/csv/src/test/java/org/jsonex/csv/CSVTest.java b/csv/src/test/java/org/jsonex/csv/CSVTest.java index 2e87abb..a2a72eb 100644 --- a/csv/src/test/java/org/jsonex/csv/CSVTest.java +++ b/csv/src/test/java/org/jsonex/csv/CSVTest.java @@ -2,9 +2,10 @@ import lombok.extern.slf4j.Slf4j; import org.jsonex.core.charsource.ArrayCharSource; -import org.jsonex.core.charsource.EOFRuntimeException; +import org.jsonex.core.charsource.ParseRuntimeException; import org.jsonex.core.util.FileUtil; import org.jsonex.treedoc.TDNode; +import org.jsonex.treedoc.json.TDJSONParser; import org.junit.Test; import static org.jsonex.snapshottest.Snapshot.assertMatchesSnapshot; @@ -13,17 +14,37 @@ @Slf4j public class CSVTest { - @Test public void testParseAndWriter() { - TDNode node = CSVParser.get().parse(FileUtil.loadResource(CSVTest.class, "test.csv")); + private void testParseAndWrite(CSVOption opt, String file) { + TDNode node = CSVParser.get().parse(FileUtil.loadResource(CSVTest.class, file), opt); assertMatchesSnapshot("parsed", node.toString()); - - CSVOption opt = new CSVOption().setFieldSep('|'); - String str = CSVWriter.get().writeAsString(node, opt); + String str = CSVWriter.get().writeAsString(node, opt.setFieldSep('|')); assertMatchesSnapshot("asString", str); TDNode node1 = CSVParser.get().parse(str, opt); assertEquals(node, node1); } + @Test public void testParseAndWriteWithoutHeader() { + testParseAndWrite(new CSVOption().setIncludeHeader(false), "test.csv"); + } + + @Test public void testParseAndWriteWithHeader() { + testParseAndWrite(new CSVOption(), "test.csv"); + } + + @Test public void testParseAndWriteObj() { + testParseAndWrite(new CSVOption(), "testObj.csv"); + } + + @Test public void testParseAndWriteJson() { + testParseAndWrite(new CSVOption(), "csv_with_json.csv"); + } + + + @Test public void testJSONValue() { + String json = "[{f1: v1, f2: {a: 1, b: 2}}, {f2:'', f3: 3}]"; + assertMatchesSnapshot(CSVWriter.get().writeAsString(TDJSONParser.get().parse(json))); + } + @Test public void testReadField() { assertEquals("ab'cd", CSVParser.get().readField(new ArrayCharSource("'ab''cd'"), new CSVOption().setQuoteChar('\''))); @@ -33,9 +54,9 @@ public class CSVTest { String error = ""; try { CSVParser.get().readField(new ArrayCharSource("'ab''cd"), new CSVOption().setQuoteChar('\'')); - } catch (EOFRuntimeException e) { + } catch (ParseRuntimeException e) { error = e.getMessage(); } - assertEquals("Can't find matching quote at position:4;line:0;col:4", error); + assertEquals("Can't find matching quote at position:5;line:0;col:5, Bookmark(line=0, col=7, pos=7), digest:", error); } } diff --git a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testJSONValue.txt b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testJSONValue.txt new file mode 100644 index 0000000..dfc65c3 --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testJSONValue.txt @@ -0,0 +1,3 @@ +f1,f2,f3 +v1,"{a: 1, b: 2}", +,,3 diff --git a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteJson_asString.txt b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteJson_asString.txt new file mode 100644 index 0000000..a13b863 --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteJson_asString.txt @@ -0,0 +1,3 @@ +val|count|percent +[{"kind":"exact","field_path":"k8s_environment","value":"production"}]|73|0.24333333333333335 +[{"kind":"exact","field_path":"k8s_environment","value":"production"},{"kind":"regex","field_path":"k8s_namespace"}]|61|0.20333333333333334 diff --git a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteJson_parsed.txt b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteJson_parsed.txt new file mode 100644 index 0000000..198e7bd --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteJson_parsed.txt @@ -0,0 +1 @@ +[{val: '[{"kind":"exact","field_path":"k8s_environment","value":"production"}]', count: 73, percent: 0.24333333333333335}, {val: '[{"kind":"exact","field_path":"k8s_environment","value":"production"},{"kind":"regex","field_path":"k8s_namespace"}]', count: 61, percent: 0.20333333333333334}] \ No newline at end of file diff --git a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteObj_asString.txt b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteObj_asString.txt new file mode 100644 index 0000000..9a255c4 --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteObj_asString.txt @@ -0,0 +1,3 @@ +@key|field1|field2 +k1|v11|v12 +k2|v21|v22 diff --git a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteObj_parsed.txt b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteObj_parsed.txt new file mode 100644 index 0000000..b0b9ae7 --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteObj_parsed.txt @@ -0,0 +1 @@ +{k1: {field1: 'v11', field2: 'v12'}, k2: {field1: 'v21', field2: 'v22'}} \ No newline at end of file diff --git a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteWithHeader_asString.txt b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteWithHeader_asString.txt new file mode 100644 index 0000000..d5964d5 --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteWithHeader_asString.txt @@ -0,0 +1,5 @@ +field1|field2|field3|field4 +v11|v12|v13|1 +v21|"v2l1 +V2l2"|v23|true +v31"v31|v32""v32|v33|"3" diff --git a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteWithHeader_parsed.txt b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteWithHeader_parsed.txt new file mode 100644 index 0000000..532a1d4 --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteWithHeader_parsed.txt @@ -0,0 +1 @@ +[{field1: 'v11', field2: 'v12', field3: 'v13', field4: 1}, {field1: 'v21', field2: 'v2l1\nV2l2', field3: 'v23', field4: true}, {field1: 'v31"v31', field2: 'v32""v32', field3: 'v33', field4: '3'}] \ No newline at end of file diff --git a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteWithoutHeader_asString.txt b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteWithoutHeader_asString.txt new file mode 100644 index 0000000..d5964d5 --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteWithoutHeader_asString.txt @@ -0,0 +1,5 @@ +field1|field2|field3|field4 +v11|v12|v13|1 +v21|"v2l1 +V2l2"|v23|true +v31"v31|v32""v32|v33|"3" diff --git a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteWithoutHeader_parsed.txt b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteWithoutHeader_parsed.txt new file mode 100644 index 0000000..73c9f9c --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriteWithoutHeader_parsed.txt @@ -0,0 +1 @@ +[['field1', 'field2', 'field3', 'field4'], ['v11', 'v12', 'v13', 1], ['v21', 'v2l1\nV2l2', 'v23', true], ['v31"v31', 'v32""v32', 'v33', '3']] \ No newline at end of file diff --git a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriterWithHeader_asString.txt b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriterWithHeader_asString.txt new file mode 100644 index 0000000..d5964d5 --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriterWithHeader_asString.txt @@ -0,0 +1,5 @@ +field1|field2|field3|field4 +v11|v12|v13|1 +v21|"v2l1 +V2l2"|v23|true +v31"v31|v32""v32|v33|"3" diff --git a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriterWithHeader_parsed.txt b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriterWithHeader_parsed.txt new file mode 100644 index 0000000..532a1d4 --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriterWithHeader_parsed.txt @@ -0,0 +1 @@ +[{field1: 'v11', field2: 'v12', field3: 'v13', field4: 1}, {field1: 'v21', field2: 'v2l1\nV2l2', field3: 'v23', field4: true}, {field1: 'v31"v31', field2: 'v32""v32', field3: 'v33', field4: '3'}] \ No newline at end of file diff --git a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriterWithoutHeader_asString.txt b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriterWithoutHeader_asString.txt new file mode 100644 index 0000000..d5964d5 --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriterWithoutHeader_asString.txt @@ -0,0 +1,5 @@ +field1|field2|field3|field4 +v11|v12|v13|1 +v21|"v2l1 +V2l2"|v23|true +v31"v31|v32""v32|v33|"3" diff --git a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriterWithoutHeader_parsed.txt b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriterWithoutHeader_parsed.txt new file mode 100644 index 0000000..73c9f9c --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriterWithoutHeader_parsed.txt @@ -0,0 +1 @@ +[['field1', 'field2', 'field3', 'field4'], ['v11', 'v12', 'v13', 1], ['v21', 'v2l1\nV2l2', 'v23', true], ['v31"v31', 'v32""v32', 'v33', '3']] \ No newline at end of file diff --git a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriter_asString.txt b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriter_asString.txt new file mode 100644 index 0000000..d5964d5 --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriter_asString.txt @@ -0,0 +1,5 @@ +field1|field2|field3|field4 +v11|v12|v13|1 +v21|"v2l1 +V2l2"|v23|true +v31"v31|v32""v32|v33|"3" diff --git a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriter_asstring.txt b/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriter_asstring.txt deleted file mode 100644 index 11ece4b..0000000 --- a/csv/src/test/resources/org/jsonex/csv/__snapshot__/CSVTest_testParseAndWriter_asstring.txt +++ /dev/null @@ -1,5 +0,0 @@ -field1|field2|field3|field4| -v11|v12|v13|1| -v21|"v2l1 -V2l2"|v23|true| -"v31""v31"|"v32""""v32"|v33|"3"| diff --git a/csv/src/test/resources/org/jsonex/csv/csv_with_json.csv b/csv/src/test/resources/org/jsonex/csv/csv_with_json.csv new file mode 100644 index 0000000..e0d231b --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/csv_with_json.csv @@ -0,0 +1,3 @@ +val,count,percent +"[{""kind"":""exact"",""field_path"":""k8s_environment"",""value"":""production""}]",73,0.24333333333333335 +"[{""kind"":""exact"",""field_path"":""k8s_environment"",""value"":""production""},{""kind"":""regex"",""field_path"":""k8s_namespace""}]",61,0.20333333333333334 diff --git a/csv/src/test/resources/org/jsonex/csv/testObj.csv b/csv/src/test/resources/org/jsonex/csv/testObj.csv new file mode 100644 index 0000000..c9eae28 --- /dev/null +++ b/csv/src/test/resources/org/jsonex/csv/testObj.csv @@ -0,0 +1,3 @@ +"@key","field1","field2" +k1,v11,v12 +k2,v21,v22 diff --git a/pom.xml b/pom.xml index 524aeeb..e05bb5d 100755 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 org.jsonex jcParent - 0.1.21 + 0.1.27 pom JSONCoder Parent JSONCoder Parent @@ -17,6 +17,7 @@ csv CliArg SnapshotTest + HiveUDF @@ -76,10 +77,15 @@ SnapshotTest ${project.version} + + ${project.groupId} + csv + ${project.version} + org.projectlombok lombok - 1.18.16 + 1.18.24 provided @@ -118,7 +124,7 @@ org.projectlombok lombok - 1.18.2 + 1.18.24 @@ -136,11 +142,18 @@ jacoco-site - post-integration-test + package report + + jacoco-site-aggregate + post-site + + report-aggregate + + @@ -176,7 +189,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.21.0 + 2.12.4 true @@ -212,7 +225,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.1.0 + 3.4.1 attach-javadocs diff --git a/treedoc/pom.xml b/treedoc/pom.xml index bf74e90..6a89f1f 100644 --- a/treedoc/pom.xml +++ b/treedoc/pom.xml @@ -6,7 +6,7 @@ org.jsonex jcParent - 0.1.21 + 0.1.27 ../pom.xml treedoc diff --git a/treedoc/src/main/java/org/jsonex/treedoc/TDNode.java b/treedoc/src/main/java/org/jsonex/treedoc/TDNode.java index c141768..0068fde 100644 --- a/treedoc/src/main/java/org/jsonex/treedoc/TDNode.java +++ b/treedoc/src/main/java/org/jsonex/treedoc/TDNode.java @@ -15,26 +15,37 @@ import lombok.experimental.Accessors; import org.jsonex.core.charsource.Bookmark; import org.jsonex.core.type.Lazy; +import static org.jsonex.core.util.LangUtil.orElse; +import static org.jsonex.core.util.LangUtil.safe; import org.jsonex.core.util.ListUtil; +import static org.jsonex.core.util.ListUtil.last; +import static org.jsonex.core.util.ListUtil.listOf; +import static org.jsonex.core.util.ListUtil.map; import org.jsonex.core.util.StringUtil; import org.jsonex.treedoc.TDPath.Part; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.Consumer; -import static org.jsonex.core.util.LangUtil.orElse; -import static org.jsonex.core.util.ListUtil.last; - /** A Node in TreeDoc */ @RequiredArgsConstructor // @Getter @Setter @Accessors(chain = true) public class TDNode { + private final static int SIZE_TO_INIT_NAME_INDEX = 64; public final static String ID_KEY = "$id"; public final static String REF_KEY = "$ref"; + public final static String COLUMN_KEY = "@key"; + public final static String COLUMN_VALUE = "@value"; + public enum Type { MAP, ARRAY, SIMPLE } @Getter TreeDoc doc; @Getter @Setter TDNode parent; @@ -53,6 +64,7 @@ public enum Type { MAP, ARRAY, SIMPLE } transient private boolean deduped; transient private final Lazy hash = new Lazy<>(); transient private final Lazy str = new Lazy<>(); + transient private Map nameIndex; // Will only initialize when size is big enough public TDNode(TDNode parent, String key) { this.doc = parent.doc; this.parent = parent; this.key = key; } public TDNode(TreeDoc doc, String key) { this.doc = doc; this.key = key; } @@ -72,7 +84,7 @@ public TDNode createChild(String name) { return cn; } - // special handling for textproto due to it's bad design that allows duplicated keys + // special handling for textproto due to its bad design that allows duplicated keys TDNode existNode = children.get(childIndex); if (!existNode.deduped) { TDNode listNode = new TDNode(this, name).setType(Type.ARRAY); @@ -96,9 +108,22 @@ public TDNode addChild(TDNode node) { if (node.key == null) // Assume it's array element node.key = "" + getChildrenSize(); children.add(node); + if (nameIndex != null) + nameIndex.put(node.key, children.size() - 1); + else if (children.size() > SIZE_TO_INIT_NAME_INDEX) + initNameIndex(); return touch(); } + private void initNameIndex() { + nameIndex = new HashMap<>(); + for (int i = 0; i< children.size(); i++) { + TDNode child = children.get(i); + if (child.key != null) + nameIndex.put(child.key, i); + } + } + public void swapWith(TDNode to) { if (this.parent == null || to.parent == null) throw new IllegalArgumentException("Can't swap root node"); @@ -124,8 +149,12 @@ public TDNode getChild(String name) { return idx < 0 ? null : children.get(idx); } - int indexOf(TDNode node) { return ListUtil.indexOf(children, n -> n == node); } - int indexOf(String name) { return ListUtil.indexOf(children, n -> n.getKey().equals(name)); } + int indexOf(TDNode node) { + return nameIndex != null ? indexOf(node.key) : ListUtil.indexOf(children, n -> n == node); + } + int indexOf(String name) { + return nameIndex != null ? orElse(nameIndex.get(name), -1) : ListUtil.indexOf(children, n -> n.getKey().equals(name)); + } int index() { return parent == null ? 0 : parent.indexOf(this); } public Object getChildValue(String name) { @@ -201,15 +230,15 @@ public List getPath() { public boolean isLeaf() { return getChildrenSize() == 0; } private TDNode touch() { - hash.clear();; - str.clear();; + hash.clear(); + str.clear(); if (parent != null) parent.touch(); return this; } @Override public String toString() { - return str.getOrCompute(() -> toString(new StringBuilder(), true, true, 100000).toString()); + return str.getOrCompute(() -> toString(new StringBuilder(), false, true, 100000).toString()); } public StringBuilder toString(StringBuilder sb, boolean includeRootKey, boolean includeReservedKeys, int limit) { @@ -217,7 +246,8 @@ public StringBuilder toString(StringBuilder sb, boolean includeRootKey, boolean sb.append(key + ": "); if (value != null) { - if (!(value instanceof String)) { + // Don't quote if not inclueRootKey + if (!(value instanceof String) || !includeRootKey) { sb.append(value); } else { String str = StringUtil.cEscape((String) value, '\''); @@ -247,6 +277,58 @@ public StringBuilder toString(StringBuilder sb, boolean includeRootKey, boolean return sb; } + public List childrenValueAsList() { + return getChildren() == null ? Collections.emptyList() : map(getChildren(), c -> orElse(c.getValue(), c)); + } + + public List> childrenValueAsListOfList() { + return getChildren() == null ? Collections.emptyList() : map(getChildren(), TDNode::childrenValueAsList); + } + + public List childrenValueAsList(List keys, List target) { + for (String k : keys) { + if (k.equals(COLUMN_KEY)) + continue; + if (k.equals(COLUMN_VALUE)) + target.add(value); + else { + TDNode c = getChild(k); + target.add(safe(c, ignore -> orElse(c.value, c))); + } + } + return target; + } + + public List> childrenValueAsListOfList(List keys) { + return getChildren() == null ? Collections.emptyList() : + map(getChildren(), c -> c.childrenValueAsList(keys, keys.get(0).equals(COLUMN_KEY) ? listOf(c.key) : listOf())); + } + + /** Get union of keys for all children, it's used for represent children in a table view */ + public List getChildrenKeys() { + List result = new ArrayList<>(); + if (this.type == Type.SIMPLE || children == null) + return result; + // Add the key column + Set keySet = new HashSet<>(); + result.add(COLUMN_KEY); + keySet.add(COLUMN_KEY); + boolean hasValue = false; + for (TDNode c : children) { + if (c.value != null) + hasValue = true; + if (c.children != null) + for (TDNode cc : c.getChildren()) + if (!keySet.contains(cc.key)) { + result.add(cc.key); + keySet.add(cc.key); + } + } + if (hasValue) + result.add(1, COLUMN_VALUE); + return result; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -258,7 +340,8 @@ public StringBuilder toString(StringBuilder sb, boolean includeRootKey, boolean return Objects.equals(key, tdNode.key) && Objects.equals(value, tdNode.value) && Objects.equals(children, tdNode.children); } + /** Hash code of value and children, key is not included */ @Override public int hashCode() { - return hash.getOrCompute(() -> Objects.hash(key, value, children)); + return hash.getOrCompute(() -> Objects.hash(value, children, map(children, TDNode::getKey))); } } diff --git a/treedoc/src/main/java/org/jsonex/treedoc/TDPath.java b/treedoc/src/main/java/org/jsonex/treedoc/TDPath.java index d6cbc70..4d09c13 100644 --- a/treedoc/src/main/java/org/jsonex/treedoc/TDPath.java +++ b/treedoc/src/main/java/org/jsonex/treedoc/TDPath.java @@ -33,6 +33,7 @@ public static class Part { /** The path parts */ final List parts = new ArrayList<>(); public TDPath addParts(Part... part) { parts.addAll(Arrays.asList(part)); return this; } + public static TDPath ofParts(Part... part) { return new TDPath().addParts(part); } public static TDPath parse(String str) { return parse(str.split("/")); diff --git a/treedoc/src/main/java/org/jsonex/treedoc/TreeDoc.java b/treedoc/src/main/java/org/jsonex/treedoc/TreeDoc.java index c8ed41c..d0f2818 100644 --- a/treedoc/src/main/java/org/jsonex/treedoc/TreeDoc.java +++ b/treedoc/src/main/java/org/jsonex/treedoc/TreeDoc.java @@ -6,10 +6,9 @@ import lombok.experimental.Accessors; import java.net.URI; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; +import java.util.*; +import static java.lang.String.format; import static org.jsonex.core.util.LangUtil.doIfNotNull; import static org.jsonex.core.util.ListUtil.mapKeys; @@ -64,4 +63,32 @@ public static TreeDoc merge(Collection nodes) { return result; } + + /** + * Dedupe the nodes that are with the same contends by comparing the hash + */ + public void dedupeNodes() { + Map hashToNodeMap = new LinkedHashMap<>(); + Queue queue = new LinkedList<>(); + queue.add(root); + while(!queue.isEmpty()) { + TDNode n = queue.poll(); + if (n.isLeaf()) + continue; // Don't dedupe leaf node and array node + int hash = n.hashCode(); + TDNode refNode = hashToNodeMap.get(hash); + if ( refNode != null && refNode.getType() == TDNode.Type.MAP) { + if (refNode.getChild(TDNode.ID_KEY) == null) + refNode.createChild(TDNode.ID_KEY).setValue(format("%x", hash)); + n.children.clear();; + n.setType(TDNode.Type.MAP).createChild(TDNode.REF_KEY).setValue(format("#%x", hash)); + } else { + hashToNodeMap.put(hash, n); + if (n.hasChildren()) + for (TDNode cn : n.children) { + queue.add(cn); + } + } + } + } } diff --git a/treedoc/src/main/java/org/jsonex/treedoc/json/TDJSONOption.java b/treedoc/src/main/java/org/jsonex/treedoc/json/TDJSONOption.java index f2bccb5..f57fa19 100644 --- a/treedoc/src/main/java/org/jsonex/treedoc/json/TDJSONOption.java +++ b/treedoc/src/main/java/org/jsonex/treedoc/json/TDJSONOption.java @@ -1,7 +1,12 @@ package org.jsonex.treedoc.json; +import lombok.AccessLevel; import lombok.Data; +import lombok.Getter; +import lombok.Setter; import lombok.experimental.Accessors; +import org.jsonex.core.type.Nullable; +import static org.jsonex.core.util.StringUtil.padEnd; import org.jsonex.treedoc.TDNode; import java.net.URI; @@ -10,12 +15,17 @@ import java.util.List; import java.util.function.BiFunction; -import static org.jsonex.core.util.StringUtil.padEnd; - +// TODO: separate parser and writer options @Accessors(chain = true) @Data public class TDJSONOption { - public enum TextType {OPERATOR, KEY, STRING, NON_STRING} + public enum TextType {OPERATOR, KEY, STRING, NON_STRING, TYPE} + + public static TDJSONOption ofIndentFactor(int factor) { return new TDJSONOption().setIndentFactor(factor); } + public static TDJSONOption ofDefaultRootType(TDNode.Type type) { return new TDJSONOption().setDefaultRootType(type); } + public static TDJSONOption ofMapToString() { return new TDJSONOption().setDeliminatorKey("=").setDeliminatorValue(", "); } + String KEY_ID = "$id"; + String KEY_TYPE = "$type"; // String KEY_REF = "$ref"; // String ObJ_START = "{"; // String ObJ_END = "}"; @@ -23,19 +33,32 @@ public enum TextType {OPERATOR, KEY, STRING, NON_STRING} // String ARRAY_END = "]"; String deliminatorKey = ":"; String deliminatorValue = ","; + String deliminatorObjectStart = "{"; + String deliminatorObjectEnd = "}"; + String deliminatorArrayStart = "["; + String deliminatorArrayEnd = "]"; /** The source */ //final CharSource source; URI uri; // Used for JSONParser - /** In case there's no enclosed '[' of '{' on the root level, the default type. */ - TDNode.Type defaultRootType = TDNode.Type.SIMPLE; + /** + * In case there's no enclosed '[' of '{' on the root level, the default type. + * By default, it will try to interpreter as a single value (either map, if there's ":", or a simple value.) + */ + @Nullable TDNode.Type defaultRootType; // Used for JSONWriter int indentFactor; - boolean alwaysQuoteName = true; - char quoteChar = '"'; + boolean alwaysQuoteKey = true; + boolean alwaysQuoteValue = true; + boolean useTypeWrapper = false; + /** + * QuoteChar can have multiple value. When multiple value is provided, it will dynamically choose the best one + * to minimize the escape. For example "\"\'", if the string contains a lot of single quote, it will use double quote. + */ + String quoteChars = "\""; String indentStr = ""; BiFunction textDecorator; @@ -50,20 +73,19 @@ public enum TextType {OPERATOR, KEY, STRING, NON_STRING} /** Node filters, if it returns null, node will be skipped */ List nodeFilters = new ArrayList<>(); - public static TDJSONOption ofIndentFactor(int factor) { return new TDJSONOption().setIndentFactor(factor); } - public static TDJSONOption ofDefaultRootType(TDNode.Type type) { return new TDJSONOption().setDefaultRootType(type); } - public static TDJSONOption ofMapToString() { return new TDJSONOption().setDeliminatorKey("=").setDeliminatorValue(", "); } - public TDJSONOption setIndentFactor(int _indentFactor) { this.indentFactor = _indentFactor; indentStr = padEnd("", indentFactor); return this; } - public boolean hasIndent() { return !indentStr.isEmpty(); } + /** Keep this for backward compatibility */ + public TDJSONOption setQuoteChar(char quoteChars) { + this.quoteChars = String.valueOf(quoteChars); + return this; + } - public TDJSONOption setDeliminatorKey(String val) { deliminatorKey = val; buildTerms(); return this; } - public TDJSONOption setDeliminatorValue(String val) { deliminatorValue = val; buildTerms(); return this; } + public boolean hasIndent() { return !indentStr.isEmpty(); } public TDNode applyFilters(TDNode n) { for (NodeFilter f : nodeFilters) { @@ -85,32 +107,47 @@ public String deco(String text, TextType type) { return textDecorator == null ? text : textDecorator.apply(text, type); } + public TDJSONOption setDeliminatorObject(String start, String end) { + deliminatorObjectStart = start; + deliminatorObjectEnd = end; + return this; + } + + public TDJSONOption setDeliminatorArray(String start, String end) { + deliminatorArrayStart = start; + deliminatorArrayEnd = end; + return this; + } + // Package scopes used by parser - String termValue; - String termValueInMap; - String termValueInArray; - String termKey; - Collection termValueStrs; - Collection termKeyStrs; - - void buildTerms() { - termValue = "\n\r"; - termKey = "{[}"; - termValueStrs = new ArrayList<>(); - termKeyStrs = new ArrayList<>(); + @Setter(AccessLevel.NONE) @Getter(AccessLevel.NONE) String _termValue; + @Setter(AccessLevel.NONE) @Getter(AccessLevel.NONE) String _termValueInMap; + @Setter(AccessLevel.NONE) @Getter(AccessLevel.NONE) String _termValueInArray; + @Setter(AccessLevel.NONE) @Getter(AccessLevel.NONE) String _termKey; + /** Quote need if a string contains any chars */ + @Setter(AccessLevel.NONE) @Getter(AccessLevel.NONE) String _quoteNeededChars; + @Setter(AccessLevel.NONE) @Getter(AccessLevel.NONE) Collection _termValueStrs; + @Setter(AccessLevel.NONE) @Getter(AccessLevel.NONE) Collection _termKeyStrs; + + public void buildTerms() { + _termValue = "\n\r" + deliminatorKey + deliminatorObjectStart; // support tree with a type in the form of "type{attr1:val1}", key1:key2:type{att1:val1} + _termKey = deliminatorObjectStart + deliminatorObjectEnd + deliminatorArrayStart; + _termValueStrs = new ArrayList<>(); + _termKeyStrs = new ArrayList<>(); if (deliminatorValue.length() == 1) { // If more than 1, will use separate string collection as term - termValue += deliminatorValue; - termKey += deliminatorValue; + _termValue += deliminatorValue; + _termKey += deliminatorValue; } else { - termValueStrs.add(deliminatorValue); - termKeyStrs.add(deliminatorValue); + _termValueStrs.add(deliminatorValue); + _termKeyStrs.add(deliminatorValue); } if (deliminatorKey.length() == 1) - termKey += deliminatorKey; + _termKey += deliminatorKey; else - termKeyStrs.add(deliminatorKey); + _termKeyStrs.add(deliminatorKey); - termValueInMap = termValue + "}"; - termValueInArray = termValue + "]"; + _termValueInMap = _termValue + deliminatorObjectEnd + deliminatorArrayEnd; // It's possible object end is omitted for path compression. e.g [a:b:c] + _termValueInArray = _termValue + deliminatorArrayEnd; + _quoteNeededChars = _termValue + deliminatorObjectEnd + deliminatorArrayEnd + deliminatorKey + deliminatorValue + quoteChars; } } diff --git a/treedoc/src/main/java/org/jsonex/treedoc/json/TDJSONParser.java b/treedoc/src/main/java/org/jsonex/treedoc/json/TDJSONParser.java index ee3a07f..6c2d777 100644 --- a/treedoc/src/main/java/org/jsonex/treedoc/json/TDJSONParser.java +++ b/treedoc/src/main/java/org/jsonex/treedoc/json/TDJSONParser.java @@ -36,31 +36,36 @@ public class TDJSONParser { public TDNode parseAll(Reader reader, TDJSONOption opt) { return parseAll(new ReaderCharSource(reader), opt); } public TDNode parseAll(CharSource src) { return parseAll(src, new TDJSONOption()); } public TDNode parseAll(String str, TDJSONOption opt) { return parseAll(new ArrayCharSource(str), opt); } - /** Parse all the JSON objects in the input stream until EOF and store them inside an root node with array type */ - public TDNode parseAll(CharSource src, TDJSONOption opt) { + /** Parse all the JSON objects in the input stream until EOF and store them inside a root node with array type */ + public TDNode parseAll(CharSource src, TDJSONOption option) { + TDJSONOption opt = option.setDefaultRootType(TDNode.Type.MAP); TreeDoc doc = TreeDoc.ofArray(); int docId = 0; while(src.skipSpacesAndReturnsAndCommas()) - TDJSONParser.get().parse(src, new TDJSONOption().setDocId(docId++), doc.getRoot().createChild()); + TDJSONParser.get().parse(src, TDJSONOption.ofDefaultRootType(TDNode.Type.MAP).setDocId(docId++), doc.getRoot().createChild()); return doc.getRoot(); } public TDNode parse(CharSource src, TDJSONOption opt, TDNode node) { return parse(src, opt, node, true); } + private boolean contains(String str, char c) { + return str.indexOf(c) >= 0; + } public TDNode parse(CharSource src, TDJSONOption opt, TDNode node, boolean isRoot) { + opt.buildTerms(); char c = skipSpaceAndComments(src); if (c == EOF) return node; node.setStart(src.getBookmark()); try { - if (c == '{') + if (contains(opt.deliminatorObjectStart, c)) return parseMap(src, opt, node, true); - if (c == '[') + if (contains(opt.deliminatorArrayStart, c)) return parseArray(src, opt, node, true); - if (isRoot) { + if (isRoot && opt.defaultRootType != null) { switch (opt.defaultRootType) { case MAP: return parseMap(src, opt, node, false); @@ -73,17 +78,35 @@ public TDNode parse(CharSource src, TDJSONOption opt, TDNode node, boolean isRoo if(c == '"' || c == '\'' || c == '`') { src.skip(); StringBuilder sb = new StringBuilder(); - src.readQuotedString(c, sb); + src.readQuotedString(sb, c); readContinuousString(src, sb); return node.setValue(sb.toString()); } - String term = opt.termValue; - if (node.getParent() != null) // parent.type can either by ARRAY or MAP. - term = node.getParent().getType() == TDNode.Type.ARRAY ? opt.termValueInArray : opt.termValueInMap; + if (isRoot && opt.defaultRootType == TDNode.Type.SIMPLE) { + return node.setValue(ClassUtil.toSimpleObject(src.readUntil("\r\n"))); + } + + String term = opt._termValue; + if (node.getParent() != null) // parent.type can either be ARRAY or MAP. + term = node.getParent().getType() == TDNode.Type.ARRAY ? opt._termValueInArray : opt._termValueInMap; - String str = src.readUntil(term, opt.termValueStrs).trim(); - return node.setValue(ClassUtil.toSimpleObject(str)); + String str = src.readUntil(term, opt._termValueStrs).trim(); + if (!src.isEof() && contains(opt.deliminatorKey, src.peek())) { // it's a path compression such as: a:b:c,d:e -> {a: {b: c}} + src.skip(); + node.setType(TDNode.Type.MAP); + parse(src, opt, node.createChild(str), false); + return node; + } + + if (!src.isEof() && contains(opt.deliminatorObjectStart, src.peek())) { + // Type wrapper: A value with type in the form of `type{attr1:val1:attr2:val2} + node.createChild(opt.KEY_TYPE).setValue(str); + return parseMap(src, opt, node, true); + } + // A simple value + node.setValue(ClassUtil.toSimpleObject(str)); + return node; } finally { node.setEnd(src.getBookmark()); } @@ -95,7 +118,7 @@ void readContinuousString(CharSource src, StringBuilder sb) { if ("\"`'".indexOf(c) < 0) break; src.skip(); - src.readQuotedString(c, sb); + src.readQuotedString(sb, c); } } @@ -142,7 +165,7 @@ TDNode parseMap(CharSource src, TDJSONOption opt, TDNode node, boolean withStart break; } - if (c == '}') { + if (contains(opt.deliminatorObjectEnd, c)) { src.skip(); break; } @@ -159,10 +182,14 @@ TDNode parseMap(CharSource src, TDJSONOption opt, TDNode node, boolean withStart c = skipSpaceAndComments(src); // if (c == EOF) // break; - if (!src.startsWith(opt.deliminatorKey) && c != '{' && c != '[' && c != ',' && c != '}') + if (!src.startsWith(opt.deliminatorKey) + && !contains(opt.deliminatorObjectStart, c) + && !contains(opt.deliminatorArrayStart, c) + && !contains(opt.deliminatorValue, c) + && !contains(opt.deliminatorObjectEnd, c)) throw src.createParseRuntimeException("No '" + opt.deliminatorKey + "' after key:" + key); } else { - key = src.readUntil(opt.termKey, opt.termKeyStrs, 1, Integer.MAX_VALUE).trim(); + key = src.readUntil(opt._termKey, opt._termKeyStrs, 1, Integer.MAX_VALUE).trim(); if (src.isEof()) throw src.createParseRuntimeException("No '" + opt.deliminatorKey + "' after key:" + key); c = src.peek(); @@ -170,7 +197,7 @@ TDNode parseMap(CharSource src, TDJSONOption opt, TDNode node, boolean withStart if (src.startsWith(opt.deliminatorKey)) src.skip(opt.deliminatorKey.length()); - if (src.startsWith(opt.deliminatorValue) || c == '}') // If there's no ':', we consider it as indexed value (array) + if (src.startsWith(opt.deliminatorValue) || contains(opt.deliminatorObjectEnd, c)) // If there's no ':', we consider it as indexed value (array) node.createChild(i + "").setValue(key); else { TDNode childNode = parse(src, opt, node.createChild(key), false); @@ -203,7 +230,7 @@ TDNode parseArray(CharSource src, TDJSONOption opt, TDNode node, boolean withSta break; } - if (c == ']') { + if (contains(opt.deliminatorArrayEnd, c)) { src.skip(); break; } diff --git a/treedoc/src/main/java/org/jsonex/treedoc/json/TDJSONWriter.java b/treedoc/src/main/java/org/jsonex/treedoc/json/TDJSONWriter.java index 8b821c4..982cb62 100644 --- a/treedoc/src/main/java/org/jsonex/treedoc/json/TDJSONWriter.java +++ b/treedoc/src/main/java/org/jsonex/treedoc/json/TDJSONWriter.java @@ -9,16 +9,19 @@ package org.jsonex.treedoc.json; +import lombok.SneakyThrows; import org.jsonex.core.factory.InjectableInstance; import org.jsonex.core.util.StringUtil; import org.jsonex.treedoc.TDNode; -import lombok.SneakyThrows; import org.jsonex.treedoc.json.TDJSONOption.TextType; +import static org.jsonex.treedoc.json.TDJSONOption.TextType.KEY; +import static org.jsonex.treedoc.json.TDJSONOption.TextType.NON_STRING; +import static org.jsonex.treedoc.json.TDJSONOption.TextType.OPERATOR; +import static org.jsonex.treedoc.json.TDJSONOption.TextType.STRING; +import static org.jsonex.treedoc.json.TDJSONOption.TextType.TYPE; import java.io.IOException; -import static org.jsonex.treedoc.json.TDJSONOption.TextType.*; - public class TDJSONWriter { public final static InjectableInstance instance = InjectableInstance.of(TDJSONWriter.class); @@ -49,19 +52,22 @@ public T write(T out, TDNode node, TDJSONOption opt, Stri @SneakyThrows T writeMap(T out, TDNode node, TDJSONOption opt, String indentStr, String childIndentStr) { - out.append(opt.deco("{", OPERATOR)); + if (opt.useTypeWrapper) { + String type = (String)node.getChildValue(opt.KEY_TYPE); + if (type != null) out.append(opt.deco(type, TYPE)); + } + out.append(opt.deco(opt.deliminatorObjectStart.substring(0, 1), OPERATOR)); for (int i = 0; i < node.getChildrenSize(); i++) { TDNode cn = opt.applyFilters(node.getChild(i)); - if (cn == null) + if (cn == null || (opt.useTypeWrapper && cn.getKey().equals(opt.KEY_TYPE))) continue; if (opt.hasIndent()) out.append('\n').append(childIndentStr); - if (!StringUtil.isJavaIdentifier(cn.getKey()) || opt.alwaysQuoteName) // Quote the key in case it's not valid java identifier - writeQuotedString(out, cn.getKey(), opt, KEY); - else - out.append(opt.deco(cn.getKey(), KEY)); + // Quote the key in case it's not valid java identifier so that it can be parsed back in Javascript + writeQuotedString(out, cn.getKey(), opt, KEY, !StringUtil.isJavaIdentifier(cn.getKey()) || opt.alwaysQuoteKey); + out.append(opt.deco(opt.deliminatorKey, OPERATOR)); write(out, cn, opt, childIndentStr); if (i < node.getChildrenSize() - 1) // No need "," for last entry @@ -71,12 +77,12 @@ T writeMap(T out, TDNode node, TDJSONOption opt, String i if (opt.hasIndent() && node.hasChildren()) out.append('\n').append(indentStr); - return (T) out.append(opt.deco("}", OPERATOR)); + return (T) out.append(opt.deco(opt.deliminatorObjectEnd.substring(0, 1), OPERATOR)); } @SneakyThrows T writeArray(T out, TDNode node, TDJSONOption opt, String indentStr, String childIndentStr) { - out.append(opt.deco("[", OPERATOR)); + out.append(opt.deco(opt.deliminatorArrayStart.substring(0, 1), OPERATOR)); if (node.hasChildren()) { for (int i = 0; i < node.getChildrenSize(); i++) { TDNode cn = node.getChild(i); @@ -92,24 +98,46 @@ T writeArray(T out, TDNode node, TDJSONOption opt, String out.append('\n').append(indentStr); } - return (T)out.append(opt.deco("]", OPERATOR)); + return (T) out.append(opt.deco(opt.deliminatorArrayEnd.substring(0, 1), OPERATOR)); } @SneakyThrows T writeSimple(T out, TDNode node, TDJSONOption opt) { Object value = node.getValue(); - if (value instanceof String) - return writeQuotedString(out, (String)value, opt, STRING); - if (value instanceof Character) - return writeQuotedString(out, String.valueOf(value), opt, STRING); + value = String.valueOf(value); + return value instanceof String + ? writeQuotedString(out, (String)value, opt, STRING, opt.alwaysQuoteValue) + : (T) out.append(opt.deco(String.valueOf(value), NON_STRING)); + } - return (T)out.append(opt.deco(String.valueOf(value), NON_STRING)); + T writeQuotedString(T out, String str, TDJSONOption opt, TextType type, boolean alwaysQuote) throws IOException { + char quoteChar = determineQuoteChar(str, opt, alwaysQuote); + Appendable result = quoteChar == 0 ? out.append(opt.deco(str, type)) : out.append(quoteChar) + .append(opt.deco(StringUtil.cEscape(str, quoteChar, true), type)) + .append(quoteChar); + return (T) result; } - T writeQuotedString(T out, String str, TDJSONOption opt, TextType type) throws IOException { - return (T) out.append(opt.quoteChar) - .append(opt.deco(StringUtil.cEscape(str, opt.quoteChar, true), type)) - .append(opt.quoteChar); + /** return 0 indicate quote is not necessary */ + static char determineQuoteChar(String str, TDJSONOption opt, boolean alwaysQuote) { + boolean needQuote = alwaysQuote || StringUtil.indexOfAnyChar(str, opt._quoteNeededChars) >= 0; + if (!needQuote) + return 0; + if (opt.quoteChars.length() == 1) + return opt.quoteChars.charAt(0); + + // Determine which quote char to use + int counts[] = new int[opt.quoteChars.length()]; + for(char ch : str.toCharArray()) { + int idx = opt.quoteChars.indexOf(ch); + if (idx >= 0) + counts[idx]++; + } + int minIdx = 0; // default to first quote char + for (int i = 1; i < counts.length; i++) + if (counts[i] < counts[minIdx]) + minIdx = i; + return opt.quoteChars.charAt(minIdx); } } diff --git a/treedoc/src/test/java/org/jsonex/treedoc/TDNodeTest.java b/treedoc/src/test/java/org/jsonex/treedoc/TDNodeTest.java new file mode 100644 index 0000000..5e7ca4c --- /dev/null +++ b/treedoc/src/test/java/org/jsonex/treedoc/TDNodeTest.java @@ -0,0 +1,24 @@ +package org.jsonex.treedoc; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import org.junit.Test; + +import java.util.List; + +public class TDNodeTest { + @Test + public void testCreateLargeNumberOfChildren() { + TDNode node = new TreeDoc().getRoot().setType(TDNode.Type.ARRAY); + long start = System.currentTimeMillis(); + for (int i = 0; i < 1000000; i++) { + node.createChild("name_" + i).setType(TDNode.Type.MAP).createChild("name_" + i + "_1").setValue("value_" + i + "_1"); + } + assertNotNull(node.getChild("name_1")); + assertNotNull(node.getChild("name_1000")); + List keys = node.getChildrenKeys(); + long time = System.currentTimeMillis() - start; + System.out.println(time); + assertTrue(time < 2000); + } +} diff --git a/treedoc/src/test/java/org/jsonex/treedoc/TDPathTest.java b/treedoc/src/test/java/org/jsonex/treedoc/TDPathTest.java new file mode 100644 index 0000000..6024b08 --- /dev/null +++ b/treedoc/src/test/java/org/jsonex/treedoc/TDPathTest.java @@ -0,0 +1,19 @@ +package org.jsonex.treedoc; + +import org.jsonex.treedoc.TDPath; +import org.junit.Test; + +import static org.jsonex.treedoc.TDPath.Part.*; +import static org.jsonex.treedoc.TDPath.parse; +import static org.junit.Assert.assertEquals; + +public class TDPathTest { + @Test public void testParse() { + assertEquals(TDPath.ofParts(ofRoot(), ofChild("p1"), ofChild("p2")), parse("/p1/p2")); + assertEquals(TDPath.ofParts(ofRelative(1), ofChild("p1"), ofChild("p2")), parse("../p1/p2")); + assertEquals(TDPath.ofParts(ofRelative(0), ofChild("p1"), ofChild("p2")), parse("./p1/p2")); + assertEquals(TDPath.ofParts(ofChildOrId("#100", "100"), ofChild("p1"), ofChild("p2")), parse("#100/p1/p2")); + assertEquals(TDPath.ofParts(ofRoot(), ofChild("p1"), ofChild("p2")), parse("#/p1/p2")); + assertEquals(TDPath.ofParts(ofRelative(1)), parse("../")); + } +} diff --git a/treedoc/src/test/java/org/jsonex/treedoc/TreeDocTest.java b/treedoc/src/test/java/org/jsonex/treedoc/TreeDocTest.java new file mode 100644 index 0000000..03db2db --- /dev/null +++ b/treedoc/src/test/java/org/jsonex/treedoc/TreeDocTest.java @@ -0,0 +1,19 @@ +package org.jsonex.treedoc; + +import org.jsonex.core.util.FileUtil; +import org.jsonex.treedoc.json.TDJSONOption; +import org.jsonex.treedoc.json.TDJSONParser; +import org.jsonex.treedoc.json.TDJSONWriter; +import org.junit.Test; + +import static org.jsonex.core.util.FileUtil.readResource; +import static org.junit.Assert.assertEquals; + +public class TreeDocTest { + @Test public void testDedupeNodes() { + TDNode node = TDJSONParser.get().parse(readResource(getClass(), "test.json")); + node.getDoc().dedupeNodes(); + String result = TDJSONWriter.get().writeAsString(node, TDJSONOption.ofIndentFactor(2).setAlwaysQuoteKey(false)); + assertEquals(FileUtil.readResource(getClass(), "test_deduped.json"), result); + } +} diff --git a/treedoc/src/test/java/org/jsonex/treedoc/json/TDJsonParserTest.java b/treedoc/src/test/java/org/jsonex/treedoc/json/TDJsonParserTest.java index 6da4268..e9fc4e8 100644 --- a/treedoc/src/test/java/org/jsonex/treedoc/json/TDJsonParserTest.java +++ b/treedoc/src/test/java/org/jsonex/treedoc/json/TDJsonParserTest.java @@ -1,12 +1,22 @@ package org.jsonex.treedoc.json; +import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.jsonex.core.charsource.ArrayCharSource; import org.jsonex.core.charsource.ReaderCharSource; -import org.jsonex.core.util.ListUtil; -import org.jsonex.core.util.MapBuilder; +import static org.jsonex.core.util.FileUtil.loadResource; +import static org.jsonex.core.util.FileUtil.readResource; +import static org.jsonex.core.util.ListUtil.listOf; +import static org.jsonex.core.util.MapBuilder.mapOf; import org.jsonex.treedoc.TDNode; import org.jsonex.treedoc.TreeDoc; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; import org.junit.Test; import java.io.Reader; @@ -14,10 +24,6 @@ import java.util.List; import java.util.Map; -import static org.jsonex.core.util.FileUtil.loadResource; -import static org.jsonex.core.util.FileUtil.readResource; -import static org.junit.Assert.*; - @Slf4j public class TDJsonParserTest { @Test public void testSkipSpaceAndComments() { @@ -99,7 +105,7 @@ public class TDJsonParserTest { } @Test public void testRootMap() { - TDNode node = TDJSONParser.get().parse("'a':1\nb:2", + TDNode node = TDJSONParser.get().parse("'a':1\nb:2,", TDJSONOption.ofDefaultRootType(TDNode.Type.MAP)); assertEquals(1, node.getValueByPath("a")); assertEquals(2, node.getValueByPath("b")); @@ -180,12 +186,13 @@ public class TDJsonParserTest { } private final static String EXPECTED_STREAM_MERGE_RESULT = - "[{a: 1, obj: {$id: '1_0'}, ref: {$ref: '#1_0'}}, {b: 2, obj: {$id: '1_1'}, ref: {$ref: '#1_1'}}, 'a:1', 'b:2']"; + "[{a: 1, obj: {$id: '1_0'}, ref: {$ref: '#1_0'}}, {b: 2, obj: {$id: '1_1'}, ref: {$ref: '#1_1'}}, {a: 1, b: 2}]"; @Test public void testStream() { ReaderCharSource reader = new ReaderCharSource(loadResource(this.getClass(), "stream.json")); List nodes = new ArrayList<>(); - while(reader.skipSpacesAndReturnsAndCommas()) - nodes.add(TDJSONParser.get().parse(reader)); + while(reader.skipSpacesAndReturnsAndCommas()) { + nodes.add(TDJSONParser.get().parse(reader, TDJSONOption.ofDefaultRootType(TDNode.Type.MAP))); + } TDNode node = TreeDoc.merge(nodes).getRoot(); log.info("testStream=" + node.toString()); assertEquals("1", node.getChild(1).getKey()); @@ -205,6 +212,18 @@ public class TDJsonParserTest { assertEquals("{a: 1, obj: {$id: '1_0'}, ref: {$ref: '#1_0'}}", node.toString()); } + @Test public void testParseWithCustomDeliminator() { + String str = "(a=va; c=(d=23; strs=))"; + TDJSONOption opt = new TDJSONOption() + .setDeliminatorKey("=") + .setDeliminatorValue(";") + .setDeliminatorObject("(", ")") + .setDeliminatorArray("<", ">"); + TDNode node = TDJSONParser.get().parse(str, opt); + assertEquals("{a: 'va', c: {d: 23, strs: ['a', 'b']}}", node.toString()); + assertEquals("(a=`va`;c=(d=23;strs=<`a`;`b`>))", TDJSONWriter.get().writeAsString(node, opt.setAlwaysQuoteKey(false).setQuoteChar('`'))); + } + private static void parseWithException(String str, String expectedError) { String error = null; try { @@ -223,17 +242,49 @@ private static void parseWithException(String str, String expectedError) { } @Test public void testParseMapToString() { - Map map = new MapBuilder() - .put("K1", "v1") + Map map = mapOf("K1", (Object)"v1") .put("k2", 123) - .put("k3", new MapBuilder() - .put("c", "Test with ,in") - .getMap()) - .put("k4", ListUtil.listOf("ab,c", "def")) - .getMap(); + .put("k3", mapOf("c", "Test with ,in").build()) + .put("k4", listOf("ab,c", "def")) + .build(); String str = map.toString(); log.info("testParseMapToString: str=" + str); TDNode node = TDJSONParser.get().parse(str, TDJSONOption.ofMapToString()); assertEquals("{K1: 'v1', k2: 123, k3: {c: 'Test with ,in'}, k4: ['ab,c', 'def']}", node.toString()); } + + @Test public void testParseObjectToString() { + TestCls test = new TestCls("va", new TestCls1(23, new String[]{"a", "b"})); + String str = test.toString(); // TDJsonParserTest.TestCls(a=va, c=TDJsonParserTest.TestCls1(d=23, strs=[a, b])) + TDJSONOption opt = new TDJSONOption().setDeliminatorObject("(", ")").setDeliminatorKey("="); + testParse(str, opt, "{$type:'TDJsonParserTest.TestCls',a:'va',c:{$type:'TDJsonParserTest.TestCls1',d:23,strs:['a','b']}}"); + } + + @Test public void testParsePathCompression() { + TDJSONOption opt = new TDJSONOption(); + testParse("a:b:123", opt, "{a:{b:123}}"); + testParse("[h:i, j:k]", opt, "[{h:'i'},{j:'k'}]"); + testParse("a:b:{e:123, f:g:[h:i, j:k]}", opt, "{a:{b:{e:123,f:{g:[{h:'i'},{j:'k'}]}}}}"); + testParse("a:b:123,c:d:456", opt, "{a:{b:123}}"); // Default, read a single map + testParse("a:b:123,c:d:456", opt.setDefaultRootType(TDNode.Type.MAP), "{a:{b:123},c:{d:456}}"); + testParse("a:b:123,c:d:456", opt.setDefaultRootType(TDNode.Type.ARRAY), "[{a:{b:123}},{c:{d:456}}]"); + testParse("a:b:123,a:d:456", opt.setDefaultRootType(TDNode.Type.MAP), "{a:[{b:123},{d:456}]}"); + } + + private void testParse(String str, TDJSONOption opt, String expectedJson) { + TDNode node = TDJSONParser.get().parse(str, opt); + assertEquals(expectedJson, TDJSONWriter.get().writeAsString(node, new TDJSONOption().setAlwaysQuoteKey(false).setQuoteChar('\''))); + } + + @Data + public static class TestCls { + public final String a; + public final TestCls1 c; + } + + @Data + public static class TestCls1 { + public final int d; + public final String[] strs; + } } diff --git a/treedoc/src/test/java/org/jsonex/treedoc/json/TDJsonWriterTest.java b/treedoc/src/test/java/org/jsonex/treedoc/json/TDJsonWriterTest.java index 081c3e6..32a538b 100644 --- a/treedoc/src/test/java/org/jsonex/treedoc/json/TDJsonWriterTest.java +++ b/treedoc/src/test/java/org/jsonex/treedoc/json/TDJsonWriterTest.java @@ -10,7 +10,7 @@ @Slf4j public class TDJsonWriterTest { @Test public void testWriterWithNodeFilter() { - TDNode node = TDJSONParser.get().parse(FileUtil.readResource(this.getClass(), "testdata.json")); + TDNode node = TDJSONParser.get().parse(readResource("testdata.json")); TDJSONOption opt = TDJSONOption.ofIndentFactor(2) .addNodeFilter(NodeFilter.mask(".*/address", ".*/ip")) .addNodeFilter(NodeFilter.exclude(".*/\\$id")); @@ -19,11 +19,17 @@ public class TDJsonWriterTest { node = TDJSONParser.get().parse(str); assertNull(node.getValueByPath("/data/0/$id")); assertEquals("[Masked:len=10,aac1cfe2]", node.getValueByPath("/data/0/ip")); - assertEquals("{Masked:len=2}", node.getValueByPath("/data/0/address")); + assertEquals("{Masked:len=2}", node.getValueByPath("/data/0/address")); } + + @Test public void testQuote() { + TDNode node = TDJSONParser.get().parse(readResource( "testQuote.json")); + TDJSONOption opt = TDJSONOption.ofIndentFactor(2).setQuoteChars("\"'").setAlwaysQuoteKey(false).setAlwaysQuoteValue(false); + String result = TDJSONWriter.get().writeAsString(node, opt) + "\n"; + assertEquals(readResource("testQuote_result.json"), result); } @Test public void testWriterWithTextDeco() { - TDNode node = TDJSONParser.get().parse(FileUtil.readResource(this.getClass(), "testdata.json")); + TDNode node = TDJSONParser.get().parse(readResource("testdata.json")); TDJSONOption opt = TDJSONOption.ofIndentFactor(2) .setTextDecorator((str, type) -> { switch (type) { @@ -35,11 +41,18 @@ public class TDJsonWriterTest { return str; } }); - String str = TDJSONWriter.get().writeAsString(node, opt); - log.info("testWriterWithNodeFilter: str=\n" + str); - String exp; - assertTrue("Should contains:" + (exp = "total"), str.contains(exp)); - assertTrue("Should contains:" + (exp = ":"), str.contains(exp)); - assertTrue("Should contains:" + (exp = "9007199254740991"), str.contains(exp)); + String str = "
\n" + TDJSONWriter.get().writeAsString(node, opt) + "\n
\n"; + assertEquals(readResource("testData_withTextDeco.html"), str); + } + + @Test public void testWriterWithTypeWrapper() { + TDNode node = TDJSONParser.get().parse(readResource("testQuote.json")); + TDJSONOption opt = TDJSONOption.ofIndentFactor(2).setUseTypeWrapper(true); + String str = TDJSONWriter.get().writeAsString(node, opt) + "\n"; + assertEquals(readResource("testData_withTypeWrapper.json"), str); + } + + private String readResource(String fileName) { + return FileUtil.readResource(this.getClass(), fileName); } } diff --git a/treedoc/src/test/java/org/jsonex/treeedoc/TDPathTest.java b/treedoc/src/test/java/org/jsonex/treeedoc/TDPathTest.java deleted file mode 100644 index d564fc8..0000000 --- a/treedoc/src/test/java/org/jsonex/treeedoc/TDPathTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.jsonex.treeedoc; - -import org.jsonex.treedoc.TDPath; -import org.junit.Test; - -import static org.jsonex.treedoc.TDPath.Part.*; -import static org.jsonex.treedoc.TDPath.parse; -import static org.junit.Assert.assertEquals; - -public class TDPathTest { - @Test public void testParse() { - assertEquals(new TDPath().addParts(ofRoot(), ofChild("p1"), ofChild("p2")), parse("/p1/p2")); - assertEquals(new TDPath().addParts(ofRelative(1), ofChild("p1"), ofChild("p2")), parse("../p1/p2")); - assertEquals(new TDPath().addParts(ofRelative(0), ofChild("p1"), ofChild("p2")), parse("./p1/p2")); - assertEquals( - new TDPath().addParts(ofChildOrId("#100", "100")).addParts(ofChild("p1")).addParts(ofChild("p2")), - parse("#100/p1/p2")); - assertEquals(new TDPath().addParts(ofRoot()).addParts(ofChild("p1")).addParts(ofChild("p2")), parse("#/p1/p2")); - } -} diff --git a/treedoc/src/test/resources/org/jsonex/treedoc/json/testData_withNodeFilter.json b/treedoc/src/test/resources/org/jsonex/treedoc/json/testData_withNodeFilter.json new file mode 100644 index 0000000..731a2ad --- /dev/null +++ b/treedoc/src/test/resources/org/jsonex/treedoc/json/testData_withNodeFilter.json @@ -0,0 +1,24 @@ +{ + "total":"100000000000000000000", + "maxSafeInt":9007199254740991, + "limit":10, + "3":"valueWithoutKey", + "data":[ + { + "name":"Some Name 1", + "address":"{Masked:len=2}", + "createdAt":"2017-07-14T17:17:33.010Z", + "ip":"[Masked:len=10,aac1cfe2]" + }, + { + "name":"Some Name 2", + "address":"{Masked:len=2}", + "createdAt":"2017-07-14T17:17:33.010Z" + }, + "Multiple line literal\n Line2" + ], + "objRef":{ + "$ref":"1" + }, + "6":"lastValueWithoutKey" +} diff --git a/treedoc/src/test/resources/org/jsonex/treedoc/json/testData_withTextDeco.html b/treedoc/src/test/resources/org/jsonex/treedoc/json/testData_withTextDeco.html new file mode 100644 index 0000000..9d24a96 --- /dev/null +++ b/treedoc/src/test/resources/org/jsonex/treedoc/json/testData_withTextDeco.html @@ -0,0 +1,34 @@ +
+{
+  "total":"100000000000000000000",
+  "maxSafeInt":9007199254740991,
+  "limit":10,
+  "3":"valueWithoutKey",
+  "data":[
+    {
+      "$id":"1",
+      "name":"Some Name 1",
+      "address":{
+        "streetLine":"1st st",
+        "city":"san jose"
+      },
+      "createdAt":"2017-07-14T17:17:33.010Z",
+      "ip":"10.1.22.22"
+    },
+    {
+      "$id":"2",
+      "name":"Some Name 2",
+      "address":{
+        "streetLine":"2nd st",
+        "city":"san jose"
+      },
+      "createdAt":"2017-07-14T17:17:33.010Z"
+    },
+    "Multiple line literal\n    Line2"
+  ],
+  "objRef":{
+    "$ref":"1"
+  },
+  "6":"lastValueWithoutKey"
+}
+
diff --git a/treedoc/src/test/resources/org/jsonex/treedoc/json/testData_withTypeWrapper.json b/treedoc/src/test/resources/org/jsonex/treedoc/json/testData_withTypeWrapper.json new file mode 100644 index 0000000..08287f3 --- /dev/null +++ b/treedoc/src/test/resources/org/jsonex/treedoc/json/testData_withTypeWrapper.json @@ -0,0 +1,5 @@ +TestQuote{ + "key with space":"value contains special chars: [", + "normalKey":"normal Value", + "keyWith:":"value has quotes: \" \", '", +} diff --git a/treedoc/src/test/resources/org/jsonex/treedoc/json/testQuote.json b/treedoc/src/test/resources/org/jsonex/treedoc/json/testQuote.json new file mode 100644 index 0000000..1e4b5af --- /dev/null +++ b/treedoc/src/test/resources/org/jsonex/treedoc/json/testQuote.json @@ -0,0 +1,6 @@ +{ + "key with space": "value contains special chars: [", + "normalKey": "normal Value", + "keyWith:": "value has quotes: \" \", '", + "$type": "TestQuote" +} diff --git a/treedoc/src/test/resources/org/jsonex/treedoc/json/testQuote_result.json b/treedoc/src/test/resources/org/jsonex/treedoc/json/testQuote_result.json new file mode 100644 index 0000000..81a9839 --- /dev/null +++ b/treedoc/src/test/resources/org/jsonex/treedoc/json/testQuote_result.json @@ -0,0 +1,6 @@ +{ + "key with space":"value contains special chars: [", + normalKey:normal Value, + "keyWith:":'value has quotes: " ", \'', + $type:TestQuote +} diff --git a/treedoc/src/test/resources/org/jsonex/treedoc/test.json b/treedoc/src/test/resources/org/jsonex/treedoc/test.json new file mode 100644 index 0000000..a151329 --- /dev/null +++ b/treedoc/src/test/resources/org/jsonex/treedoc/test.json @@ -0,0 +1,17 @@ +{ + "a": 1, + "b": { + "c": [1,2,3,4], + "d": { + "e": 1, + "f": 2 + } + }, + "e": { + "c": [1, 2, 3, 4], + "d": { + "e": 1, + "f": 2 + } + } +} \ No newline at end of file diff --git a/treedoc/src/test/resources/org/jsonex/treedoc/test_deduped.json b/treedoc/src/test/resources/org/jsonex/treedoc/test_deduped.json new file mode 100644 index 0000000..39024db --- /dev/null +++ b/treedoc/src/test/resources/org/jsonex/treedoc/test_deduped.json @@ -0,0 +1,19 @@ +{ + a:1, + b:{ + c:[ + 1, + 2, + 3, + 4 + ], + d:{ + e:1, + f:2 + }, + $id:"5a9c3cc0" + }, + e:{ + $ref:"#5a9c3cc0" + } +} \ No newline at end of file