feat: further parametrization and parallelization (#317)

* refactor parameter set for pt mapping

Extracted the parameter set class =>  cleaner code
some quality of life name changes

* feat: parametrized pt modes

now for each schedule mode, we can add a specific number of links and maximum search distance parameters

we can also use strict rule: when number of links is reached no additional links are added

backwards compatibility is ensured

* parallelize creating facilities and links sequences

Parallelize the generation of facilities and links from pseudo routes, speeding up the process considerably

* make threads and chunk size configurable

* fix typo

* extract reused variable

* remove duplicate exception

* rename fields to match original names

* update default pt mapper config

* fix copy-paste

* remove static

* use local variables

We now use local variables and do not adjust global ones if, for this mode, we need to use mode-specific rules

* add a modeSpecificRules test

* test to check the strict num of links rule

Author: balacmi

doc/defaultPTMapperConfig.xml

@@ -18,6 +18,8 @@
 		between stops from the schedule. If travelCostType is 
 		linkLength minTravel cost is the beeline distance. -->
 		<param name="maxTravelCostFactor" value="5.0" />
+		<!-- Instead of using general number of links and maximum search distance rule, use the schedule mode specific rules to be defined within the parameter sets. For those that no information is provided the general values will be used. Options: [false, true]. Default: false. -->
+		<param name="modeSpecificRules" value="false" />
 		<!-- All links that do not have a transit route on them are removed, except the ones 
 		listed in this set (typically only car). Separated by comma. -->
 		<param name="modesToKeepOnCleanUp" value="car" />
@@ -47,19 +49,24 @@
 		transit routes passing using it. This is recommended for "artificial", additional 
 		modes (especially "rail", if used) can be added, separated by commas. -->
 		<param name="scheduleFreespeedModes" value="artificial" />
+		<!-- The size of the chunk that is sent to the pt mapper thread at the time to buildfacilities and links. Default: 100. -->
+		<param name="threadChunkSize" value="100" />
 		<!-- Defines which link attribute should be used for routing. Possible values "linkLength" (default) 
 		and "travelTime". -->
 		<param name="travelCostType" value="linkLength" />
 		<parameterset type="transportModeAssignment" >
-			<!-- Transit Routes with the given scheduleMode can only use links with at least one of the network modes
-			defined here. Separate multiple modes by comma. If no network modes are defined, the transit route will
-			use artificial links. -->
+			<param name="maxLinkCandidateDistance" value="90.0" />
+			<param name="nLinkThreshold" value="6" />
 			<param name="networkModes" value="car,bus" />
 			<param name="scheduleMode" value="bus" />
+			<param name="strictLinkRule" value="false" />
 		</parameterset>
 		<parameterset type="transportModeAssignment" >
+			<param name="maxLinkCandidateDistance" value="90.0" />
+			<param name="nLinkThreshold" value="6" />
 			<param name="networkModes" value="rail,light_rail" />
 			<param name="scheduleMode" value="rail" />
+			<param name="strictLinkRule" value="false" />
 		</parameterset>
 	</module>
 

src/main/java/org/matsim/pt2matsim/config/PublicTransitMappingConfigGroup.java

@@ -19,22 +19,23 @@
 
 package org.matsim.pt2matsim.config;
 
-import org.matsim.api.core.v01.network.Link;
-import org.matsim.core.api.internal.MatsimParameters;
-import org.matsim.core.config.*;
-import org.matsim.core.config.groups.ControllerConfigGroup;
-import org.matsim.core.config.groups.ControllerConfigGroup.RoutingAlgorithmType;
-import org.matsim.core.utils.collections.CollectionUtils;
-import org.matsim.pt.transitSchedule.api.TransitRoute;
-import org.matsim.pt2matsim.run.PublicTransitMapper;
+import static org.matsim.pt2matsim.config.PublicTransitMappingConfigGroup.TravelCostType.travelTime;
 
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import static org.matsim.pt2matsim.config.PublicTransitMappingConfigGroup.TravelCostType.travelTime;
+import org.matsim.core.config.Config;
+import org.matsim.core.config.ConfigGroup;
+import org.matsim.core.config.ConfigUtils;
+import org.matsim.core.config.ConfigWriter;
+import org.matsim.core.config.ReflectiveConfigGroup;
+import org.matsim.core.config.groups.ControllerConfigGroup;
+import org.matsim.core.config.groups.ControllerConfigGroup.RoutingAlgorithmType;
+import org.matsim.core.utils.collections.CollectionUtils;
+import org.matsim.pt2matsim.run.PublicTransitMapper;
 
 
 /**
@@ -70,13 +71,19 @@ public enum TravelCostType { linkLength, travelTime }
 	private static final String ROUTING_WITH_CANDIDATE_DISTANCE = "routingWithCandidateDistance";
 
 	private static final String NETWORK_ROUTER = "networkRouter";
+	
+	private static final String USE_MODE_SPECIFIC_RULES = "modeSpecificRules";
+	
+	private static final String THREAD_CHUNK_SIZE = "threadChunkSize";
 
 	// default values
 	private Map<String, Set<String>> transportModeAssignment = new HashMap<>();
+	private Map<String, TransportModeParameterSet> parameterSetsForMode = new HashMap<>();
 	private Set<String> scheduleFreespeedModes = PublicTransitMappingStrings.ARTIFICIAL_LINK_MODE_AS_SET;
 	private Set<String> modesToKeepOnCleanUp = new HashSet<>();
 	private double maxTravelCostFactor = 5.0;
 	private int numOfThreads = 2;
+	private int chunkSize = 100;
 	private boolean removeNotUsedStopFacilities = true;
 
 	private String inputNetworkFile = null;
@@ -91,6 +98,8 @@ public enum TravelCostType { linkLength, travelTime }
 	private double maxLinkCandidateDistance = 90;
 	private double candiateDistanceMulitplier = 1.6;
 
+	private boolean modeSpecificRules = false;
+	
 	private RoutingAlgorithmType networkRouter = ControllerConfigGroup.RoutingAlgorithmType.SpeedyALT;
 
 	public PublicTransitMappingConfigGroup() {
@@ -104,9 +113,9 @@ public static PublicTransitMappingConfigGroup createDefaultConfig() {
 		PublicTransitMappingConfigGroup config = new PublicTransitMappingConfigGroup();
 		config.getModesToKeepOnCleanUp().add("car");
 
-		TransportModeAssignment tmaBus = new TransportModeAssignment("bus");
+		TransportModeParameterSet tmaBus = new TransportModeParameterSet("bus");
 		tmaBus.setNetworkModesStr("car,bus");
-		TransportModeAssignment tmaRail = new TransportModeAssignment("rail");
+		TransportModeParameterSet tmaRail = new TransportModeParameterSet("rail");
 		tmaRail.setNetworkModesStr("rail,light_rail");
 		config.addParameterSet(tmaBus);
 		config.addParameterSet(tmaRail);
@@ -181,14 +190,18 @@ public final Map<String, String> getComments() {
 				"\t\tNo link candidates beyond this distance are added.");
 		map.put(NETWORK_ROUTER,
 				"The router that should be used. Possible options are: [SpeedyALT, AStarLandmarks]");
+		map.put(USE_MODE_SPECIFIC_RULES, "Instead of using general number of links and maximum search distance rule, use the schedule mode specific rules "
+				+ "to be defined within the parameter sets. For those that no information is provided the general values will be used. Options: [false, true]. Default: false.");
+		map.put(THREAD_CHUNK_SIZE, "The size of the chunk that is sent to the pt mapper thread at the time to build"
+				+ "facilities and links. Default: 100.");
 		return map;
 	}
 
 	@Override
 	public ConfigGroup createParameterSet(final String type) {
 		switch(type) {
-			case TransportModeAssignment.SET_NAME:
-				return new TransportModeAssignment();
+			case TransportModeParameterSet.GROUP_NAME:
+				return new TransportModeParameterSet();
 			default:
 				throw new IllegalArgumentException("Unknown parameterset name!");
 		}
@@ -204,9 +217,10 @@ public void addParameterSet(final ConfigGroup set) {
 	 * Loads the parameter set for TransportModeAssignment for easier access
 	 */
 	private void loadParameterSets() {
-		for(ConfigGroup e : this.getParameterSets(TransportModeAssignment.SET_NAME)) {
-			TransportModeAssignment mra = (TransportModeAssignment) e;
-			transportModeAssignment.put(mra.getScheduleMode(), mra.getNetworkModes());
+		for(ConfigGroup e : this.getParameterSets(TransportModeParameterSet.GROUP_NAME)) {
+			TransportModeParameterSet tmps = (TransportModeParameterSet) e;
+			transportModeAssignment.put(tmps.getScheduleMode(), tmps.getNetworkModes());
+			parameterSetsForMode.put(tmps.getScheduleMode(), tmps);
 		}
 	}
 
@@ -233,6 +247,10 @@ public Map<String, Set<String>> getTransportModeAssignment() {
 	public void setTransportModeAssignment(Map<String, Set<String>> transportModeAssignment) {
 		this.transportModeAssignment = transportModeAssignment;
 	}
+	
+	public TransportModeParameterSet getParameterSetForMode(String scheduleMode) {
+		return this.parameterSetsForMode.get(scheduleMode);
+	}
 
 	/**
 	 * All links that do not have a transit route on them are removed, except
@@ -485,71 +503,25 @@ public RoutingAlgorithmType getNetworkRouter() {
 	public void setNetworkRouter(RoutingAlgorithmType networkRouter) {
 		this.networkRouter = networkRouter;
 	}
+	
+	@StringGetter(USE_MODE_SPECIFIC_RULES)
+	public boolean getModeSpecificRules() {
+		return this.modeSpecificRules;
+	}
 
+	@StringSetter(USE_MODE_SPECIFIC_RULES)
+	public void setModeSpecificRules(String modeSpecificRules) {
+		this.modeSpecificRules = Boolean.parseBoolean(modeSpecificRules);
+	}
 
-	/**
-	 * Parameterset that define which network transport modes the router
-	 * can use for each schedule transport mode. If no networkModes are set, the
-	 * transit route is mapped artificially<p/>
-	 * <p>
-	 * Network transport modes are the ones in {@link Link#getAllowedModes()}, schedule
-	 * transport modes are from {@link TransitRoute#getTransportMode()}.
-	 */
-	public static class TransportModeAssignment extends ReflectiveConfigGroup implements MatsimParameters {
-
-		public static final String SET_NAME = "transportModeAssignment";
-
-		private static final String SCHEDULE_MODE = "scheduleMode";
-		private static final String NETWORK_MODES = "networkModes";
-
-		private String scheduleMode;
-		private Set<String> networkModes;
-
-		public TransportModeAssignment() {
-			super(SET_NAME);
-		}
-
-		public TransportModeAssignment(String scheduleMode) {
-			super(SET_NAME);
-			this.scheduleMode = scheduleMode;
-		}
-
-		@Override
-		public Map<String, String> getComments() {
-			Map<String, String> map = super.getComments();
-			map.put(NETWORK_MODES,
-					"Transit Routes with the given scheduleMode can only use links with at least one of the network modes\n" +
-					"\t\t\tdefined here. Separate multiple modes by comma. If no network modes are defined, the transit route will\n" +
-					"\t\t\tuse artificial links.");
-			return map;
-		}
-
-		@StringGetter(SCHEDULE_MODE)
-		public String getScheduleMode() {
-			return scheduleMode;
-		}
-
-		@StringSetter(SCHEDULE_MODE)
-		public void setScheduleMode(String scheduleMode) {
-			this.scheduleMode = scheduleMode;
-		}
-
-		@StringGetter(NETWORK_MODES)
-		public String getNetworkModesStr() {
-			return CollectionUtils.setToString(networkModes);
-		}
-
-		@StringSetter(NETWORK_MODES)
-		public void setNetworkModesStr(String networkModesStr) {
-			this.networkModes = CollectionUtils.stringToSet(networkModesStr);
-		}
-
-		public Set<String> getNetworkModes() {
-			return this.networkModes;
-		}
+	@StringGetter(THREAD_CHUNK_SIZE)
+	public int getChunkSize() {
+		return this.chunkSize;
+	}
 
-		public void setNetworkModes(Set<String> networkModes) {
-			this.networkModes = networkModes;
-		}
+	@StringSetter(THREAD_CHUNK_SIZE)
+	public void setChunkSize(int chunkSize) {
+		this.chunkSize = chunkSize;
 	}
+	
 }

src/main/java/org/matsim/pt2matsim/config/TransportModeParameterSet.java

@@ -0,0 +1,95 @@
+package org.matsim.pt2matsim.config;
+
+import java.util.Set;
+
+import org.matsim.api.core.v01.network.Link;
+import org.matsim.core.config.ReflectiveConfigGroup;
+import org.matsim.core.utils.collections.CollectionUtils;
+import org.matsim.pt.transitSchedule.api.TransitRoute;
+
+/**
+ * Parameterset that define which network transport modes the router
+ * can use for each schedule transport mode. If no networkModes are set, the
+ * transit route is mapped artificially<p/>
+ * <p>
+ * Network transport modes are the ones in {@link Link#getAllowedModes()}, schedule
+ * transport modes are from {@link TransitRoute#getTransportMode()}.
+ **/
+public class TransportModeParameterSet extends ReflectiveConfigGroup{
+	
+	static public final String GROUP_NAME = "transportModeAssignment";
+
+	private static final String SCHEDULE_MODE = "scheduleMode";
+	private static final String NETWORK_MODES = "networkModes";
+	private static final String NUMBER_LINK_CANDIDATES = "nLinkThreshold";
+	private static final String MAXIMUM_SEARCH_DISTANCE = "maxLinkCandidateDistance";
+	private static final String IMPOSE_STRICT_LINK_RULE = "strictLinkRule";
+
+	private String scheduleMode;
+	private Set<String> networkModes;
+	private int numberOfLinkCandidates = 6;
+	private double maximumSearchDistance = 90;
+	private boolean imposeStrictLinksRule = false;
+
+	public TransportModeParameterSet() {
+		super(GROUP_NAME);
+	}
+	
+	public TransportModeParameterSet(String scheduleMode) {
+		super(GROUP_NAME);
+		this.scheduleMode = scheduleMode;
+	}
+
+	@StringGetter(SCHEDULE_MODE)
+	public String getScheduleMode() {
+		return scheduleMode;
+	}
+
+	@StringSetter(SCHEDULE_MODE)
+	public void setScheduleMode(String scheduleMode) {
+		this.scheduleMode = scheduleMode;
+	}
+
+	@StringGetter(NETWORK_MODES)
+	public String getNetworkModesStr() {
+		return CollectionUtils.setToString(networkModes);
+	}
+
+	@StringSetter(NETWORK_MODES)
+	public void setNetworkModesStr(String networkModesStr) {
+		this.networkModes = CollectionUtils.stringToSet(networkModesStr);
+	}
+	
+	public Set<String> getNetworkModes() {
+		return this.networkModes;
+	}
+	
+	@StringGetter(NUMBER_LINK_CANDIDATES)
+	public int getNumberOfLinkCandidates() {
+		return numberOfLinkCandidates;
+	}
+	
+	@StringSetter(NUMBER_LINK_CANDIDATES)
+	public void setNumberOfLinkCandidates(int numberOfLinkCandidates) {
+		this.numberOfLinkCandidates = numberOfLinkCandidates;
+	}
+	
+	@StringGetter(MAXIMUM_SEARCH_DISTANCE)
+	public double getMaximumSearchDistance() {
+		return maximumSearchDistance;
+	}
+	
+	@StringSetter(MAXIMUM_SEARCH_DISTANCE)
+	public void setMaximumSearchDistance(double maximumSearchDistance) {
+		this.maximumSearchDistance = maximumSearchDistance;
+	}
+	@StringGetter(IMPOSE_STRICT_LINK_RULE)
+	public boolean getImposeStrictLinksRule() {
+		return imposeStrictLinksRule;
+	}
+	@StringSetter(IMPOSE_STRICT_LINK_RULE)
+	public void setImposeStrictLinksRule(boolean imposeStrictLinksRule) {
+		this.imposeStrictLinksRule = imposeStrictLinksRule;
+	}
+
+}

src/main/java/org/matsim/pt2matsim/examples/PT2MATSimExample.java

@@ -28,6 +28,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.concurrent.ExecutionException;
 
 /**
  * Usage test of PT2MATSim to document how the package can be used. The example region
@@ -45,7 +46,7 @@ public final class PT2MATSimExample {
 
 	private static final String addisonCountyEPSG = "EPSG:2032";
 
-	public static void main(String[] args) {
+	public static void main(String[] args) throws InterruptedException, ExecutionException {
 		prepare();
 
 		// 1. Convert a gtfs schedule to an unmapped transit schedule