Merge pull request #1100 from GRIDAPPSD/releases/2019.09.1

Release of version 2019.09.1

Dockerfile

@@ -23,7 +23,7 @@ RUN cd ${TEMP_DIR} \
 
 # Get the gridappsd-sensor-simulator from the proper repository
 RUN cd ${TEMP_DIR} \
-  && git clone https://github.com/GRIDAPPSD/gridappsd-voltage-violation -b master \
+  && git clone https://github.com/GRIDAPPSD/gridappsd-voltage-violation -b develop \
   && cd gridappsd-voltage-violation \
   && mkdir -p /gridappsd/services/gridappsd-voltage-violation \
   && rm .git -rf \ 

build-gridappsd-container

@@ -1,3 +1,5 @@
+docker pull gridappsd/gridappsd_base:master
+
 ./gradlew clean
 rm -rf gov.pnnl.goss/gridappsd/generated
 ./gradlew export

gov.pnnl.goss.gridappsd/bnd.bnd

@@ -29,10 +29,10 @@
 	osgi.enroute.base.api,\
 	org.mockito.mockito-all,\
 	httpcore,\
-	blazegraph.cim2glm;version=10.0.3,\
+	blazegraph.cim2glm;version=11.0.0,\
 	httpclient,\
 	com.bigdata.rdf,\
-    proven-client;version=0.2.2,\
+    proven-client;version=0.2.3,\
     proven-message;version=0.4
 
 -plugin org.apache.felix.dm.annotation.plugin.bnd.AnnotationPlugin;log=debug

gov.pnnl.goss.gridappsd/conf/pnnl.goss.gridappsd.cfg

@@ -5,7 +5,7 @@ fncs.bridge.path = ./scripts/goss_fncs_bridge.py
 applications.path = /gridappsd/applications
 services.path = /gridappsd/services
 # Requires no / 
-blazegraph.ns.path = http://localhost:8889/bigdata/namespace/kb/sparql
+blazegraph.ns.path = http://localhost:8889/bigdata/sparql
 
 # Uses docker composed blazegraph host here.  Note this is not the
 # external address, but from inside one of the containers on the

gov.pnnl.goss.gridappsd/run.bnd.bndrun

@@ -74,12 +74,12 @@
 	httpcore,\
 	httpclient,\
 	xml-apis,\
-	blazegraph.cim2glm;version=10.0.3,\
+	blazegraph.cim2glm;version=11.0.0,\
 	org.eclipse.jetty.aggregate.jetty-all-server;version=7.6.9,\
 	javax.servlet-api,\
 	com.bigdata.rdf,\
 	proven-message;version=0.4,\
-	proven-client;version=0.2.2,\
+	proven-client;version=0.2.3,\
 	javax.ws.rs-api,\
 	org.apache.commons.codec,\
 	shacl,\

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/api/ServiceManager.java

@@ -42,9 +42,11 @@
 import gov.pnnl.goss.gridappsd.dto.ServiceInfo;
 import gov.pnnl.goss.gridappsd.dto.ServiceInstance;
 import gov.pnnl.goss.gridappsd.dto.SimulationContext;
+import gov.pnnl.goss.gridappsd.dto.UserOptions;
 
 import java.io.File;
 import java.io.Serializable;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -90,9 +92,9 @@ public interface ServiceManager {
 	 * @param runtimeOptions Runtime options for the service instance, in most cases these will be passed in on the command-line 
 	 * @return String containing service instance ID
 	 */
-	String startService(String service_id, String runtimeOptions);  //may also need input/output topics or simulation id
+	String startService(String service_id, HashMap<String, Object> runtimeOptions);  //may also need input/output topics or simulation id
 
-	String startServiceForSimultion(String service_id, String runtimeOptions, Map<String, Object> simulationContext);  //may also need input/output topics??
+	String startServiceForSimultion(String service_id, HashMap<String, Object> hashMap, Map<String, Object> simulationContext);  //may also need input/output topics??
 
 	/**
 	 * Stops all instances of the service with requested service ID

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/configuration/CIMDictionaryConfigurationHandler.java

@@ -83,6 +83,7 @@ public class CIMDictionaryConfigurationHandler extends BaseConfigurationHandler
 	public static final String TYPENAME = "CIM Dictionary";
 	public static final String MODELID = "model_id";
 	public static final String SIMULATIONID = "simulation_id";
+	public static final String USEHOUSES = "use_houses";
 
 	public CIMDictionaryConfigurationHandler() {
 	}
@@ -112,6 +113,9 @@ public void generateConfig(Properties parameters, PrintWriter out, String proces
 		logRunning("Generating Dictionary GridLAB-D configuration file using parameters: "+parameters, processId, "", logManager);
 
 		String simulationId = GridAppsDConstants.getStringProperty(parameters, SIMULATIONID, null);
+		boolean useHouses = false;
+                if(parameters.containsKey(USEHOUSES))
+                        useHouses = GridAppsDConstants.getBooleanProperty(parameters, USEHOUSES, false);
 		File configFile = null;
 		if(simulationId!=null){
 			SimulationContext simulationContext = simulationManager.getSimulationContextForId(simulationId);
@@ -149,9 +153,9 @@ public void generateConfig(Properties parameters, PrintWriter out, String proces
 		CIMImporter cimImporter = new CIMImporter(); 
 		//If the simulation info is available also write to file
 		if(configFile!=null){
-			cimImporter.generateDictionaryFile(queryHandler, new PrintWriter(new FileWriter(configFile)));
+			cimImporter.generateDictionaryFile(queryHandler, new PrintWriter(new FileWriter(configFile)),useHouses);
 		} else {
-			cimImporter.generateDictionaryFile(queryHandler, out);
+			cimImporter.generateDictionaryFile(queryHandler, out, useHouses);
 		}
 		if(configFile!=null){
 			//config was written to file, so return that

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/configuration/GLDAllConfigurationHandler.java

@@ -252,9 +252,9 @@ public void generateConfig(Properties parameters, PrintWriter out, String proces
 				//Convert to UTC time until the input time is correct
 				////TODO this will be changed in the future
 				//c.add(Calendar.HOUR, 6);
-				queryFilter.put(STARTTIME_FILTER, ""+c.getTimeInMillis()+"000");
+				queryFilter.put(STARTTIME_FILTER, ""+c.getTimeInMillis()+"000000");
 				c.add(Calendar.SECOND, new Long(simulationDuration).intValue());
-				queryFilter.put(ENDTIME_FILTER, ""+c.getTimeInMillis()+"000");
+				queryFilter.put(ENDTIME_FILTER, ""+c.getTimeInMillis()+"000000");
 				weatherRequest.setQueryFilter(queryFilter);
 				DataResponse resp = (DataResponse)dataManager.processDataRequest(weatherRequest, ProvenTimeSeriesDataManagerImpl.DATA_MANAGER_TYPE, simId, tempDataPath, username);
 				if(resp.getData()==null){
@@ -289,6 +289,7 @@ public void generateConfig(Properties parameters, PrintWriter out, String proces
 		String dictFile = tempDataPath+File.separator+DICTIONARY_FILENAME;
 		simOutputParams.setProperty(GLDSimulationOutputConfigurationHandler.DICTIONARY_FILE, dictFile);
 		simOutputParams.setProperty(GLDSimulationOutputConfigurationHandler.MODELID, modelId);
+		simOutputParams.setProperty(GLDSimulationOutputConfigurationHandler.USEHOUSES, Boolean.toString(useHouses));
 		GLDSimulationOutputConfigurationHandler simulationOutputConfig = new GLDSimulationOutputConfigurationHandler(configManager, powergridModelManager, logManager);
 		simulationOutputConfig.generateConfig(simOutputParams, simulationOutputs, processId, username);
 

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/configuration/GLDSimulationOutputConfigurationHandler.java

@@ -100,6 +100,7 @@ public class GLDSimulationOutputConfigurationHandler extends BaseConfigurationHa
 	public static final String MODELID = "model_id";
 	public static final String DICTIONARY_FILE = "dictionary_file";
 	public static final String SIMULATIONID = "simulation_id";
+	public static final String USEHOUSES = "use_houses";
 
 	public GLDSimulationOutputConfigurationHandler() {
 	}
@@ -132,6 +133,7 @@ public void generateConfig(Properties parameters, PrintWriter out, String proces
 		logRunning("Generating simulation output configuration file using parameters: "+parameters, processId, username, logManager);
 		File dictFile = null;
 		String simulationId = GridAppsDConstants.getStringProperty(parameters, SIMULATIONID, null);
+		boolean useHouses = GridAppsDConstants.getBooleanProperty(parameters, USEHOUSES, false);
 		File configFile = null;
 		if(simulationId!=null){
 			SimulationContext simulationContext = simulationManager.getSimulationContextForId(simulationId);
@@ -180,7 +182,7 @@ public void generateConfig(Properties parameters, PrintWriter out, String proces
 			StringWriter dictionaryStringOutput = new StringWriter();
 			PrintWriter dictionaryOutput = new PrintWriter(dictionaryStringOutput);
 
-			cimImporter.generateDictionaryFile(queryHandler, dictionaryOutput);
+			cimImporter.generateDictionaryFile(queryHandler, dictionaryOutput, useHouses);
 			String dictOut = dictionaryStringOutput.toString();
 			measurementFileReader = null;
 			if(dictFile!=null && dictFile.getName().length()>0 && !dictFile.exists()){
@@ -358,7 +360,7 @@ void parseMeasurement(Map<String, JsonArray> measurements, JsonObject measuremen
 			if(measurementType.equals("VA")) {
 				objectName = conductingEquipmentName;
 				if(phases.equals("1") || phases.equals("2")) {
-					propertyName = "measured_power_" + phases;
+					propertyName = "indiv_measured_power_" + phases;
 				} else {
 					propertyName = "measured_power_" + phases;
 				}

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/data/LogDataManagerMySQL.java

@@ -84,7 +84,6 @@ public class LogDataManagerMySQL implements LogDataManager, DataManagerHandler {
 	ClientFactory clientFactory;
 
 	private Connection connection;
-	private PreparedStatement preparedStatement;
 	Client client;
 
 	public static final String DATA_MANAGER_TYPE = "log";
@@ -121,7 +120,7 @@ public void store(String source, String processId, long timestamp,
 		if(connection!=null){
 			try {
 
-				preparedStatement = connection.prepareStatement("INSERT INTO gridappsd.log ("
+				PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO gridappsd.log ("
 						+ "id, "
 						+ "source, "
 						+ "process_id, "
@@ -141,7 +140,6 @@ public void store(String source, String processId, long timestamp,
 				preparedStatement.setString(7, username);
 				preparedStatement.setString(8, process_type);
 
-
 				preparedStatement.executeUpdate();
 
 			} catch (Exception e) {
@@ -167,7 +165,7 @@ public void storeExpectedResults(String test_id, String processId, long simulati
 		if(connection!=null){
 			try {
 
-				preparedStatement = connection.prepareStatement("INSERT INTO gridappsd.expected_results VALUES (default, ?, ?, ?, ?, ?, ?,?)");
+				PreparedStatement preparedStatement = connection.prepareStatement("INSERT INTO gridappsd.expected_results VALUES (default, ?, ?, ?, ?, ?, ?,?)");
 				preparedStatement.setString(1, test_id);
 				preparedStatement.setString(2, processId);
 				preparedStatement.setString(3, mrid);
@@ -255,7 +253,7 @@ public Serializable query(String source, String processId, long timestamp, LogLe
 					queryString+=" timestamp="+timestamp;
 				}
 
-			preparedStatement = connection.prepareStatement(queryString);
+			PreparedStatement preparedStatement = connection.prepareStatement(queryString);
 
 			ResultSet rs = preparedStatement.executeQuery();
 
@@ -299,7 +297,7 @@ public Serializable query(String queryString){
 
 		if(connection!=null){
 			try{
-				preparedStatement = connection.prepareStatement(queryString);
+				PreparedStatement preparedStatement = connection.prepareStatement(queryString);
 				ResultSet rs = preparedStatement.executeQuery();
 				return this.getJSONFromResultSet(rs);
 		}catch (SQLException e) {

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/dto/RequestSimulation.java

@@ -45,6 +45,7 @@
 import gov.pnnl.goss.gridappsd.dto.events.ScheduledCommandEvent;
 
 import java.io.Serializable;
+import java.util.List;
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
@@ -64,6 +65,8 @@ public enum SimulationRequestType {
 
 	public ApplicationConfig application_config;
 
+	public List<ServiceConfig> service_configs;
+
 	public TestConfig test_config;
 
 	public SimulationRequestType simulation_request_type = SimulationRequestType.NEW;

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/dto/RequestTimeseriesData.java

@@ -54,6 +54,14 @@ public static RequestTimeseriesData parse(String jsonString){
 		if(obj.queryMeasurement.equals("simulation"))
 			if(obj.queryFilter==null || !obj.queryFilter.containsKey("simulation_id"))
 				throw new JsonSyntaxException("Expected filter simulation_id not found.");
+		if(obj.queryFilter.containsKey("startTime")){
+            		String startTime = obj.queryFilter.get("startTime");
+            		obj.queryFilter.put("startTime", startTime+"000");
+        	}	
+        	if(obj.queryFilter.containsKey("endTime")){
+            		String endTime = obj.queryFilter.get("endTime");
+            		obj.queryFilter.put("endTime", endTime+"000");
+        	}
 		return obj;
 	}
 

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/dto/ServiceConfig.java

@@ -0,0 +1,30 @@
+package gov.pnnl.goss.gridappsd.dto;
+
+import java.io.Serializable;
+import java.util.HashMap;
+
+public class ServiceConfig implements Serializable {
+
+	private static final long serialVersionUID = -2413334775260242364L;
+
+	String id;
+	HashMap<String,Object> user_options;
+	Object user_value;
+
+	public String getId() {
+		return id;
+	}
+	public void setId(String id) {
+		this.id = id;
+	}
+	public HashMap<String, Object> getUser_options() {
+		return user_options;
+	}
+	public void setUser_input(HashMap<String, Object> user_options) {
+		this.user_options = user_options;
+	}
+
+
+
+
+}

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/dto/ServiceInfo.java

@@ -40,7 +40,12 @@
 package gov.pnnl.goss.gridappsd.dto;
 
 
+import java.io.File;
+import java.io.IOException;
 import java.io.Serializable;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.HashMap;
 import java.util.List;
 
 import com.google.gson.Gson;
@@ -60,6 +65,7 @@ public enum ServiceType {
 	List<String> output_topics;
 	List<String> static_args;
 	String execution_path;
+	HashMap<String,UserOptions> user_input;
 	ServiceType type;
 	boolean launch_on_startup;
 	List<String> service_dependencies;
@@ -123,6 +129,8 @@ public void setExecution_path(String execution_path) {
 		this.execution_path = execution_path;
 	}
 
+
+
 	public ServiceType getType() {
 		return type;
 	}
@@ -177,4 +185,10 @@ public static ServiceInfo parse(String jsonString){
 		return obj;
 	}
 
+	public static void main(String[] args) throws IOException{
+
+		File test = new File("test.config");
+		System.out.println(ServiceInfo.parse(new String(Files.readAllBytes(Paths.get(test.getAbsolutePath())))));
+	}
+
 }

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/dto/ServiceInstance.java

@@ -50,12 +50,12 @@ public class ServiceInstance implements Serializable{
 
 	String instance_id;
 	ServiceInfo servcie_info;
-	String runtime_options;
+	HashMap<String, Object> runtime_options;
 	String simulation_id;
 	Process process;
 
 
-	public ServiceInstance(String instance_id, ServiceInfo servcie_info, String runtime_options, String simulation_id, Process process){
+	public ServiceInstance(String instance_id, ServiceInfo servcie_info, HashMap<String, Object> runtime_options, String simulation_id, Process process){
 		this.instance_id = instance_id;
 		this.servcie_info = servcie_info;
 		this.runtime_options = runtime_options;
@@ -80,11 +80,11 @@ public void setService_info(ServiceInfo servcie_info) {
 		this.servcie_info = servcie_info;
 	}
 
-	public String getRuntime_options() {
+	public HashMap<String, Object> getRuntime_options() {
 		return runtime_options;
 	}
 
-	public void setRuntime_options(String runtime_options) {
+	public void setRuntime_options(HashMap<String, Object> runtime_options) {
 		this.runtime_options = runtime_options;
 	}
 

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/dto/UserOptions.java

@@ -0,0 +1,25 @@
+package gov.pnnl.goss.gridappsd.dto;
+
+import java.io.Serializable;
+
+import com.google.gson.Gson;
+
+public class UserOptions implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+	String name;
+	String help;
+	String type;
+	Object help_example;
+	Object default_value;
+	Object min_value;
+	Object max_value;
+
+	@Override
+	public String toString() {
+		Gson  gson = new Gson();
+		return gson.toJson(this);
+	}
+
+}