0.21.0 - Performance improvements (~15-20%) and more in-depth performance related logging.

Author: TeamworkGuy2

CHANGELOG.md

@@ -4,7 +4,18 @@ This project does its best to adhere to [Semantic Versioning](http://semver.org/
 
 
 --------
-### [0.20.0](N/A) - 2020-11-22
+### [0.21.0](https://github.com/TeamworkGuy2/JParseCode/commit/7482a2a148a5f42e230c33897d3f17d907c5348f) - 2020-11-22
+__Parsing performance optimizations (~15-20%)__
+#### Changed
+* Update dependencies `[email protected]` and `[email protected]`
+* `ParserActionLogger` and `ParseTimes` optimizations and changes to work with `jtext-tokenizer` new `TokenizationLogger`
+* `JavaFileTokenizer`, `CsFileTokenizer`, `CommentTokenizer`, `IdentifierTokenizer`, and `NumberTokenizer` `reusable/reusableCharParsers` flag implementation to control which `CharParserMatchableFactory` implementation is used
+* `CodeTokenizer.tokenizeDocument()` `fragmentConstructor` parameter optimization to skip creating fragment source string twice, major performance improvement ~5-10%
+* New aggregate performance logging output for text tokenization related counters
+
+
+--------
+### [0.20.0](https://github.com/TeamworkGuy2/JParseCode/commit/7482a2a148a5f42e230c33897d3f17d907c5348f) - 2020-11-22
 __Method generic type parameters parsing support__ (i.e. 'public T Create<T>()' in C#).
 #### Added
 * Method generic type parameters added - `MethodSig` and sub-classes have a new `typeParameters` field. Note: if these 'types' contain lower/upper type bounds these are included in the `typeName` property in Java and are not yet included in C# parsing

bin/jparse_code-with-tests.jar

@@ -1,5 +1,5 @@
 {
-	"version" : "0.20.0",
+	"version" : "0.21.0",
 	"name" : "jparse-code",
 	"description" : "An in-progress suite of parsing/transpilation tools for C#, Java, and TypeScript code.  Generates simple JSON ASTs.",
 	"homepage" : "https://github.com/TeamworkGuy2/JParseCode",

bin/jparse_code.jar

@@ -3,7 +3,6 @@
 import java.io.IOException;
 import java.util.Map;
 
-import twg2.dataUtil.dataUtils.EnumError;
 import twg2.io.files.FileUtil;
 import twg2.io.json.stringify.JsonStringify;
 import twg2.parser.output.JsonWritableSig;
@@ -15,17 +14,6 @@
  * @since 2016-09-07
  */
 public class ParseTimes implements JsonWritableSig {
-
-
-	public static enum TrackerAction {
-		SETUP,
-		READ,
-		TOKENIZE,
-		EXTRACT_AST;
-	}
-
-
-
 	long setupNs;
 	long readNs;
 	long tokenizeNs;
@@ -57,23 +45,23 @@ public long getTotalNs() {
 	}
 
 
-	public void setActionTime(TrackerAction action, long timeNanos) {
-		switch(action) {
-		case SETUP:
-			this.setupNs = timeNanos;
-			break;
-		case READ:
-			this.readNs = timeNanos;
-			break;
-		case TOKENIZE:
-			this.tokenizeNs = timeNanos;
-			break;
-		case EXTRACT_AST:
-			this.extractAstNs = timeNanos;
-			break;
-		default:
-			throw EnumError.unknownValue(action, TrackerAction.class);
-		}
+	public void setTimeSetup(long setupNanos) {
+		this.setupNs = setupNanos;
+	}
+
+
+	public void setTimeRead(long readNanos) {
+		this.readNs = readNanos;
+	}
+
+
+	public void setTimeTokenize(long tokenizeNanos) {
+		this.tokenizeNs = tokenizeNanos;
+	}
+
+
+	public void setTimeExtractAst(long extractAstNanos) {
+		this.extractAstNs = extractAstNanos;
 	}
 
 
@@ -104,7 +92,7 @@ public String toString() {
 
 
 	public String toString(String srcName, boolean includeClassName) {
-		return (includeClassName ? "parserPerformance: { " : "") +
+		return (includeClassName ? "parseTimes: { " : "") +
 				(srcName != null ? ("file: " + srcName + ", ") : "") +
 				"setup: " + roundNsToMs(this.setupNs) + ", " +
 				"read: " + roundNsToMs(this.readNs) + ", " +
@@ -115,7 +103,7 @@ public String toString(String srcName, boolean includeClassName) {
 	}
 
 
-	public static final void toJsons(Map<String, ParseTimes> fileParseTimes, boolean includeSurroundingBrackets, Appendable dst, WriteSettings st) throws IOException {
+	public static void toJsons(Map<String, ParseTimes> fileParseTimes, boolean includeSurroundingBrackets, Appendable dst, WriteSettings st) throws IOException {
 		if(includeSurroundingBrackets) { dst.append("[\n"); }
 		JsonStringify.inst.joinConsume(fileParseTimes.entrySet(), ",\n", dst, (entry) -> {
 			var stat = entry.getValue();
@@ -126,14 +114,13 @@ public static final void toJsons(Map<String, ParseTimes> fileParseTimes, boolean
 	}
 
 
-	public static final String toStrings(Map<String, ParseTimes> fileParseTimes) {
+	public static String toStrings(Map<String, ParseTimes> fileParseTimes) {
 		long total = 0;
 		var sb = new StringBuilder();
 
 		for(var entry : fileParseTimes.entrySet()) {
 			var stat = entry.getValue();
-			long subtotal = stat.setupNs + stat.readNs + stat.tokenizeNs + stat.extractAstNs;
-			total += subtotal;
+			total += stat.getTotalNs();
 
 			String fileName = FileUtil.getFileNameWithoutExtension(entry.getKey());
 			fileName = '"' + fileName.substring(0, Math.min(30, fileName.length())) + '"';
@@ -148,8 +135,13 @@ public static final String toStrings(Map<String, ParseTimes> fileParseTimes) {
 	}
 
 
-	public static String roundNsToMs(long ns) {
-		return String.format("%.2f", ns/1000000D);
+	public static String roundNsToMs(long nanos) {
+		return String.format("%.2f", nanos / 1000000D);
+	}
+
+
+	public static String roundNsToMs(long nanos, int decimalPlaces) {
+		return String.format("%." + decimalPlaces + "f", nanos / 1000000D);
 	}
 
 }

package-lib.json

@@ -1,107 +1,74 @@
 package twg2.parser.codeParser.analytics;
 
 import java.io.IOException;
-import java.util.EnumMap;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 
 import twg2.io.files.FileUtil;
 import twg2.io.json.stringify.JsonStringify;
+import twg2.parser.condition.text.CharParser;
 import twg2.parser.output.JsonWritableSig;
 import twg2.parser.output.WriteSettings;
 import twg2.text.stringEscape.StringEscapeJson;
 import twg2.text.stringUtils.StringPad;
-import twg2.text.tokenizer.analytics.ParserAction;
-import twg2.text.tokenizer.analytics.TypedLogger;
+import twg2.text.tokenizer.CharParserFactory;
+import twg2.text.tokenizer.CharParserMatchableFactory;
+import twg2.text.tokenizer.analytics.TokenizationLogger;
 
 /**
  * @author TeamworkGuy2
  * @since 2016-09-11
  */
-public class ParserActionLogger implements TypedLogger<ParserAction, WriteSettings>, JsonWritableSig {
-
-	static class CountAndDurationLog {
-		String msg;
-		long count;
-		double durationMilliseconds;
-
-
-		@Override
-		public String toString() {
-			return (this.msg != null ? "msg: " + this.msg + ", " : "") +
-					(this.count != 0 ? "count: " + this.count + ", " : "") +
-					(this.durationMilliseconds != 0 ? "durationMillis: " + this.durationMilliseconds : "");
-		}
-
-
-		public void toJson(Appendable dst, WriteSettings st) throws IOException {
-			var json = JsonStringify.inst;
-
-			json.append("{ ", dst);
-
-			if(this.msg != null) {
-				json.comma(dst).toProp("message", this.msg, dst);
-			}
-
-			if(this.count != 0) {
-				json.comma(dst).toProp("count", this.count, dst);
-			}
-
-			if(this.durationMilliseconds != 0) {
-				json.comma(dst).toProp("durationMillis", this.durationMilliseconds, dst);
-			}
-
-			json.append(" }", dst);
-		}
-
-	}
-
-
-	Map<ParserAction, CountAndDurationLog> actions;
+public class ParserActionLogger implements TokenizationLogger, JsonWritableSig {
+	public int countCompoundCharParserMatch;
+	public int countCompoundCharParserAcceptNext;
+	public int countCreateParser;
+	public int countTextFragmentsConsumed;
+	public int totalParserReuseCount;
+	public List<CharParserMatchableFactory.Reusable<CharParser>> reusableParserFactories;
 
 
 	public ParserActionLogger() {
-		this.actions = new EnumMap<>(ParserAction.class);
-	}
-
-
-	public CountAndDurationLog getLog(ParserAction action) {
-		return this.actions.get(action);
 	}
 
 
-	public String getLogMsg(ParserAction action) {
-		var res = this.actions.get(action);
-		return res != null ? res.msg : null;
-	}
-
-
-	public long getLogCount(ParserAction action) {
-		var res = this.actions.get(action);
-		return res != null ? res.count : 0;
+	@Override
+	public void logCountCompoundCharParserMatch(int count) {
+		countCompoundCharParserMatch += count;
 	}
 
 
-	public double getLogDuration(ParserAction action) {
-		var res = this.actions.get(action);
-		return res != null ? res.durationMilliseconds : 0;
+	@Override
+	public void logCountCompoundCharParserAcceptNext(int count) {
+		countCompoundCharParserAcceptNext += count;
 	}
 
 
 	@Override
-	public void logMsg(ParserAction action, String msg) {
-		getTypedAction(action).msg = msg;
+	public void logCountCreateParser(int count) {
+		countCreateParser += count;
 	}
 
 
 	@Override
-	public void logCount(ParserAction action, long count) {
-		getTypedAction(action).count += count;
+	public void logCountTextFragmentsConsumed(int count) {
+		countTextFragmentsConsumed += count;
 	}
 
 
-	@Override
-	public void logDuration(ParserAction action, double durationMilliseconds) {
-		getTypedAction(action).durationMilliseconds += durationMilliseconds;
+	public void logCharParserFactoryReuse(Collection<? extends CharParserFactory> charParserFactories) {
+		this.reusableParserFactories = new ArrayList<>();
+		
+		for(var parserFactory : charParserFactories) {
+			if(parserFactory instanceof CharParserMatchableFactory.Reusable) {
+				@SuppressWarnings("unchecked")
+				var parserFactoryReusable = (CharParserMatchableFactory.Reusable<CharParser>)parserFactory;
+				reusableParserFactories.add(parserFactoryReusable);
+				totalParserReuseCount += parserFactoryReusable.getReuseCount();
+			}
+		}
 	}
 
 
@@ -111,7 +78,6 @@ public void toJson(Appendable dst, WriteSettings st) throws IOException {
 	}
 
 
-	@Override
 	public void toJson(String srcName, boolean includeSurroundingBrackets, Appendable dst, WriteSettings st) throws IOException {
 		if(includeSurroundingBrackets) { dst.append("{\n"); }
 		if(srcName != null) {
@@ -120,16 +86,20 @@ public void toJson(String srcName, boolean includeSurroundingBrackets, Appendabl
 			dst.append("\"");
 		}
 
-		boolean first = true;
-		for(ParserAction action : ParserAction.values()) {
-			var data = this.actions.get(action);
-			if(data != null) {
-				if(!first || srcName != null) { dst.append(",\n"); }
-				String name = action.name();
-				dst.append("\t\"").append(name).append("\": ");
-				data.toJson(dst, st);
-				first = false;
-			}
+		if(countCompoundCharParserMatch > 0) {
+			dst.append("\t\"compoundCharParserMatch\": ").append(Integer.toString(countCompoundCharParserMatch));
+		}
+		if(countCompoundCharParserAcceptNext > 0) {
+			dst.append(",\n");
+			dst.append("\t\"compoundCharParserAcceptNext\": ").append(Integer.toString(countCompoundCharParserAcceptNext));
+		}
+		if(countCreateParser > 0) {
+			dst.append(",\n");
+			dst.append("\t\"createParser\": ").append(Integer.toString(countCreateParser));
+		}
+		if(countTextFragmentsConsumed > 0) {
+			dst.append(",\n");
+			dst.append("\t\"textFragmentsConsumed\": ").append(Integer.toString(countTextFragmentsConsumed));
 		}
 
 		if(includeSurroundingBrackets) { dst.append("\n}"); }
@@ -142,7 +112,6 @@ public String toString() {
 	}
 
 
-	@Override
 	public String toString(String srcName, boolean includeClassName) {
 		var dst = new StringBuilder();
 		if(includeClassName) {
@@ -152,40 +121,38 @@ public String toString(String srcName, boolean includeClassName) {
 			dst.append("file: " + srcName);
 		}
 
-		boolean first = true;
-		for(ParserAction action : ParserAction.values()) {
-			var data = this.actions.get(action);
-			if(data != null) {
-				if(!first || srcName != null) { dst.append(", "); }
-				String name = action.name();
-				dst.append("\"").append(name).append("\": ").append(data.toString());
-				first = false;
-			}
+		if(countCompoundCharParserMatch > 0) {
+			dst.append("compoundCharParserMatch: ").append(Integer.toString(countCompoundCharParserMatch));
+		}
+		if(countCompoundCharParserAcceptNext > 0) {
+			dst.append(", ");
+			dst.append("compoundCharParserAcceptNext: ").append(Integer.toString(countCompoundCharParserAcceptNext));
+		}
+		if(countCreateParser > 0) {
+			dst.append(", ");
+			dst.append("createParser: ").append(Integer.toString(countCreateParser));
 		}
+		if(countTextFragmentsConsumed > 0) {
+			dst.append(", ");
+			dst.append("textFragmentsConsumed: ").append(Integer.toString(countTextFragmentsConsumed));
+		}
+
 		if(includeClassName) {
 			dst.append("\n}");
 		}
 		return dst.toString();
 	}
 
 
-	private CountAndDurationLog getTypedAction(ParserAction action) {
-		var data = actions.get(action);
-		if(data == null) {
-			data = new CountAndDurationLog();
-			actions.put(action, data);
-		}
-		return data;
-	}
-
-
 	public static void toJsons(Map<String, ParserActionLogger> fileParserDetails, boolean includeSurroundingBrackets, Appendable dst, WriteSettings st) throws IOException {
 		if(includeSurroundingBrackets) { dst.append("[\n"); }
+
 		JsonStringify.inst.joinConsume(fileParserDetails.entrySet(), ",\n", dst, (entry) -> {
 			var stat = entry.getValue();
 			String fileName = FileUtil.getFileNameWithoutExtension(entry.getKey());
 			stat.toJson(fileName, includeSurroundingBrackets, dst, st);
 		});
+
 		if(includeSurroundingBrackets) { dst.append("\n]"); }
 	}