Skip to content

Commit

Permalink
Adaguc-server logfile is embedded in java log. Num instances, timeout…
Browse files Browse the repository at this point in the history 
… and queue size are configurable
  • Loading branch information
@maartenplieger
maartenplieger committed Jun 4, 2019
1 parent 79bb3f6 commit 56a97a7
Show file tree
Hide file tree
Showing 8 changed files with 190 additions and 39 deletions.
5 changes: 3 additions & 2 deletions src/main/java/nl/knmi/adaguc/config/ConfigurationReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,9 +124,10 @@ static public synchronized void readConfig() throws ElementNotFoundException, I
String envKey = key.substring(5,key.length()-1);
String envValue = System.getenv(envKey);
if(envValue == null){
throw new ElementNotFoundException("Environment variable ["+envKey+"] not set");
Debug.errprintln("[WARNING]: Environment variable ["+envKey+"] not set");
} else {
configFile = configFile.replace(key,envValue);
}
configFile = configFile.replace(key,envValue);
}
// Debug.println("configfile=" + configFile);

Expand Down
32 changes: 28 additions & 4 deletions src/main/java/nl/knmi/adaguc/services/adagucserver/ADAGUCConfigurator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<adaguc-services>
<adaguc-server>
<timeout>1000</timeout>
<maxinstances>4</maxinstances>
<queuesize>4</queuesize>
<adagucexecutable>/home/c3smagic/code/maartenplieger/adaguc-server/bin/adagucserver</adagucexecutable>
<export>ADAGUC_PATH=/home/c3smagic/code/maartenplieger/adaguc-server/</export>
<export>ADAGUC_TMP=/tmp</export>
<export>ADAGUC_CONFIG=/home/c3smagic/code/maartenplieger/adaguc-server/data/config/adaguc.autoresource.xml</export>
<export>ADAGUC_DATARESTRICTION="FALSE"</export>
<export>ADAGUC_LOGFILE=/tmp/adaguc.autoresource.log</export>
<export>ADAGUC_ERRORFILE=/tmp/adaguc.autoresource.errlog</export>
<export>ADAGUC_FONT=/home/c3smagic/code/maartenplieger/adaguc-server/data/fonts/FreeSans.ttf</export>
<export>ADAGUC_ONLINERESOURCE=http://adaguc-services/adagucserver?</export>
</adaguc-server>
</adaguc-services>
*/
Expand All @@ -38,10 +37,20 @@ public class ADAGUCConfigurator implements nl.knmi.adaguc.config.ConfiguratorInt

private static String[] environmentVariables = {
};

/* How long adaguc-server is allowed to run in milliseconds, -1 is unlimted */
private static long timeOut = -1;
/* Number of maximum simultaneaous instances before instances are queued, -1 is unlimited */
private static int maxInstances = -1;
/* Maximum queuesize, -1 is unlimited */
private static int maxInstancesInQueue = -1;

public static void doConfig(XMLElement configReader){
ADAGUCExecutable=configReader.getNodeValue("adaguc-services.adaguc-server.adagucexecutable");
environmentVariables = configReader.getNodeValues("adaguc-services.adaguc-server.export");
try { timeOut = Long.parseLong(configReader.getNodeValue("adaguc-services.adaguc-server.timeout"));} catch (Exception e) {}
try { maxInstances = Integer.parseInt(configReader.getNodeValue("adaguc-services.adaguc-server.maxinstances"));} catch (Exception e) {}
try { maxInstancesInQueue = Integer.parseInt(configReader.getNodeValue("adaguc-services.adaguc-server.queuesize"));} catch (Exception e) {}
}

public static String getADAGUCExecutable() throws ElementNotFoundException, IOException {
Expand All @@ -53,5 +62,20 @@ public static String[] getADAGUCEnvironment() throws ElementNotFoundException, I

ConfigurationReader.readConfig();
return environmentVariables;
}

public static long getTimeOut() throws ElementNotFoundException, IOException {
ConfigurationReader.readConfig();
return timeOut;
}

public static int getMaxInstances () throws ElementNotFoundException, IOException {
ConfigurationReader.readConfig();
return maxInstances;
}

public static int getMaxInstancesInQueue () throws ElementNotFoundException, IOException {
ConfigurationReader.readConfig();
return maxInstancesInQueue;
}
}
39 changes: 36 additions & 3 deletions src/main/java/nl/knmi/adaguc/services/adagucserver/ADAGUCRequestMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.json.JSONObject;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
Expand All @@ -11,9 +12,14 @@
import nl.knmi.adaguc.tools.Debug;
import nl.knmi.adaguc.tools.JSONResponse;

import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean;



@RestController
public class ADAGUCRequestMapper {

@ResponseBody
@CrossOrigin
@RequestMapping("wms")
Expand All @@ -22,8 +28,9 @@ public void ADAGUCSERVERWMS(HttpServletResponse response, HttpServletRequest req
try {
ADAGUCServer.runADAGUCWMS(request,response,null,null);
} catch (Exception e) {
Debug.printStackTrace(e);
JSONResponse jsonResponse = new JSONResponse(request);
jsonResponse.setException("ADAGUCServer WMS request failed",e);
jsonResponse.setException("[ADAGUC-Server] WMS request failed",e);
try {
jsonResponse.print(response);
} catch (Exception e1) {
Expand All @@ -34,14 +41,40 @@ public void ADAGUCSERVERWMS(HttpServletResponse response, HttpServletRequest req
}
@ResponseBody
@CrossOrigin
@RequestMapping("adagucload")
public void adagucLoad(HttpServletResponse response, HttpServletRequest request){
JSONResponse jsonResponse = new JSONResponse(request);
try {
OperatingSystemMXBean operatingSystemMXBean = ManagementFactory.getOperatingSystemMXBean();

jsonResponse.setMessage(new JSONObject().put("adagucServer",
new JSONObject().put("instancesInQueue", ADAGUCServer.getNumInstancesInQueue())
.put("instancesRunning", ADAGUCServer.getNumInstancesRunning())
.put("maxQueueSize" ,ADAGUCConfigurator.getMaxInstancesInQueue())
.put("maxInstances", ADAGUCConfigurator.getMaxInstances())
.put("instanceTimeout", ADAGUCConfigurator.getTimeOut())
.put("getSystemLoadAverage", operatingSystemMXBean.getSystemLoadAverage())
.put("getAvailableProcessors", operatingSystemMXBean.getAvailableProcessors())
));
} catch (Exception e) {
jsonResponse.setException("ADAGUCServer ADAGUC Load request failed",e);
}
try {
jsonResponse.print(response);
} catch (Exception e1) {

}
}
@ResponseBody
@CrossOrigin
@RequestMapping("adagucserver")
public void ADAGUCSERVER(HttpServletResponse response, HttpServletRequest request){
Debug.println("/adagucserver");
try {
ADAGUCServer.runADAGUCWMS(request,response,null,null);
} catch (Exception e) {
JSONResponse jsonResponse = new JSONResponse(request);
jsonResponse.setException("ADAGUCServer WMS request failed",e);
jsonResponse.setException("[ADAGUC-Server] WMS request failed",e);
try {
jsonResponse.print(response);
} catch (Exception e1) {
Expand Down
109 changes: 95 additions & 14 deletions src/main/java/nl/knmi/adaguc/services/adagucserver/ADAGUCServer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,19 @@
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.UUID;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.json.JSONObject;

import nl.knmi.adaguc.config.MainServicesConfigurator;
import nl.knmi.adaguc.security.AuthenticatorFactory;
import nl.knmi.adaguc.security.AuthenticatorInterface;
Expand Down Expand Up @@ -65,8 +71,58 @@ public static void runADAGUCWCS(HttpServletRequest request,HttpServletResponse r
enum ADAGUCServiceType{
WMS, WCS, OPENDAP
}

private static int numInstancesRunning = 0;
private static int numInstancesInQue = 0;
public static void runADAGUC(HttpServletRequest request,HttpServletResponse response,String queryString,OutputStream outputStream, ADAGUCServiceType serviceType) throws Exception{
Debug.println("runADAGUCWMS");
Debug.println("runADAGUC");
int maxInstances = ADAGUCConfigurator.getMaxInstances();
int maxInstancesInQueue = ADAGUCConfigurator.getMaxInstancesInQueue();
Exception exception = null;
String instanceId = UUID.randomUUID().toString();

if (maxInstancesInQueue > 0 && numInstancesInQue > maxInstancesInQueue) {

String msg = "[ADAGUC-Server] Queue limit [" + maxInstancesInQueue + "] exceeded";
Debug.errprintln(msg);
response.setStatus(500);
OutputStream o = response.getOutputStream();
o.write(msg.getBytes());
return;
}

try {
if (maxInstances > 0 && maxInstancesInQueue > 0) {
if (numInstancesRunning >= maxInstances) {
Debug.println("[ADAGUC-Server] Too many instances running, Queued: [" + numInstancesInQue + "], Running: [" + numInstancesRunning + "]");
}
if (maxInstancesInQueue > 0)numInstancesInQue++;
try {
while (numInstancesRunning >= maxInstances) {
Thread.sleep(100);
}
}catch (Exception e){
exception = e;
}
if (maxInstancesInQueue > 0)numInstancesInQue--;
}
}catch (Exception e) {
exception = e;
}
if (exception == null) {
numInstancesRunning++;
try {
_runADAGUC(request, response, queryString, outputStream, serviceType, instanceId);
}catch (Exception e) {
exception = e;
}
numInstancesRunning--;
}

if (exception != null) throw exception;
}

private static void _runADAGUC(HttpServletRequest request,HttpServletResponse response,String queryString,OutputStream outputStream, ADAGUCServiceType serviceType, String instanceId) throws Exception{

// Debug.println("Headers:");
// Enumeration<String> headerNames = request.getHeaderNames();
Expand All @@ -76,7 +132,7 @@ public static void runADAGUC(HttpServletRequest request,HttpServletResponse resp
// Debug.println(headerName + ":" + headerValue);
// }
//
List<String> environmentVariables = new ArrayList<String>();

String userHomeDir="/tmp/";

AuthenticatorInterface authenticator = AuthenticatorFactory.getAuthenticator(request);
Expand All @@ -88,10 +144,10 @@ public static void runADAGUC(HttpServletRequest request,HttpServletResponse resp
}

}
Debug.println("Using home " + userHomeDir);
//Debug.println("Using home " + userHomeDir);
String homeURL=MainServicesConfigurator.getServerExternalURL();
String adagucExecutableLocation = ADAGUCConfigurator.getADAGUCExecutable();
Debug.println("adagucExecutableLocation: "+adagucExecutableLocation);
//Debug.println("adagucExecutableLocation: "+adagucExecutableLocation);

if(adagucExecutableLocation == null){
Debug.errprintln("Adagucserver executable not configured");
Expand Down Expand Up @@ -121,10 +177,16 @@ public static void runADAGUC(HttpServletRequest request,HttpServletResponse resp
if(queryString == null){
queryString = request.getQueryString();
}
Debug.println("[ADAGUC-Server]" + queryString);
Debug.println("[ADAGUC-Server] queryString [" + queryString + "]");



List<String> environmentVariables = new ArrayList<String>();
String tmpDir = userHomeDir+"/adaguctmp/";
Tools.mksubdirs(tmpDir);
environmentVariables.add("ADAGUC_TMP="+tmpDir);
String tmpLogFile = tmpDir + "adaguc-server-log" + instanceId;
Debug.println("Logging to " + tmpLogFile);
environmentVariables.add("ADAGUC_LOGFILE=" + tmpLogFile);
environmentVariables.add("HOME="+userHomeDir);
environmentVariables.add("QUERY_STRING="+queryString);
environmentVariables.add("CONTENT_TYPE="+request.getHeader("Content-Type"));
Expand All @@ -137,30 +199,43 @@ public static void runADAGUC(HttpServletRequest request,HttpServletResponse resp

if(serviceType == ADAGUCServiceType.OPENDAP){

/* localenv['SCRIPT_NAME']="/myscriptname";
SCRIPT_NAME [/cgi-bin/autoresource.cgi],
REQUEST_URI [/cgi-bin/autoresource.cgi/opendap/clipc/combinetest/wcs_nc2.nc.das]
localenv['REQUEST_URI']="/myscriptname/" + path*/
environmentVariables.add("ADAGUC_ONLINERESOURCE="+homeURL+"/adagucopendap?");
environmentVariables.add("REQUEST_URI="+request.getRequestURI());
environmentVariables.add("SCRIPT_NAME=");
Debug.println(request.getRequestURI());
}

Tools.mksubdirs(userHomeDir+"/adaguctmp/");
environmentVariables.add("ADAGUC_TMP="+userHomeDir+"/adaguctmp/");


String[] configEnv = ADAGUCConfigurator.getADAGUCEnvironment();
if(configEnv == null){
Debug.println("ADAGUC environment is not configured");
}else{
for(int j=0;j<configEnv.length;j++)environmentVariables.add(configEnv[j]);
for(int j=0;j<configEnv.length;j++){
if (!configEnv[j].startsWith("ADAGUC_LOGFILE") && !configEnv[j].startsWith("ADAGUC_TMP") && !configEnv[j].startsWith("ADAGUC_ONLINERESOURCE")) {
environmentVariables.add(configEnv[j]);
} else {
Debug.errprintln("[WARNING]: Environment " + configEnv[j] + " is controlled by adaguc-services.");
}
}
}
String commands[] = {adagucExecutableLocation};

String[] environmentVariablesAsArray = new String[ environmentVariables.size() ];
environmentVariables.toArray( environmentVariablesAsArray );
CGIRunner.runCGIProgram(commands,environmentVariablesAsArray,userHomeDir,response,outputStream,null);
long timeOutMs = ADAGUCConfigurator.getTimeOut();

int statusCode = CGIRunner.runCGIProgram(commands,environmentVariablesAsArray,userHomeDir,response,outputStream,null, timeOutMs);
if (statusCode != 143) {
try {
Debug.println("\n" + Tools.readFile(tmpLogFile));
Tools.rmfile(tmpLogFile);
} catch (Exception e) {
Debug.println("[ADAGUC-Server]: No logfile");
}
} else {
Debug.println("[ADAGUC-Server]: Timeout");
}
}

/**
Expand Down Expand Up @@ -197,6 +272,12 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) {
public static void runADAGUCOpenDAP(HttpServletRequest request,HttpServletResponse response,String queryString,OutputStream outputStream) throws Exception {
runADAGUC(request,response, queryString, outputStream, ADAGUCServiceType.OPENDAP);
}
public static int getNumInstancesInQueue() {
return numInstancesInQue;
}
public static int getNumInstancesRunning() {
return numInstancesRunning;
}


}
2 changes: 1 addition & 1 deletion src/main/java/nl/knmi/adaguc/services/esgfsearch/NetCDFC.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public boolean hasData() {
}catch(Exception e){
}
String[] environmentVariables = {"HOME="+userHome};
ProcessRunner processRunner = new ProcessRunner (stdoutPrinter,stderrPrinter,environmentVariables,userHome);
ProcessRunner processRunner = new ProcessRunner (stdoutPrinter,stderrPrinter,environmentVariables,userHome, -1);
try{
String commands[]={"ncdump","-h","-x",url};
Debug.println("starting ncdump for "+url);
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/nl/knmi/adaguc/services/pywpsserver/PyWPSServer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,15 +152,15 @@ public static void runPyWPS(HttpServletRequest request,HttpServletResponse respo
String wpsRequest=HTTPTools.getHTTPParam(request, "request");
if (wpsRequest.equalsIgnoreCase("execute")) {
ByteArrayOutputStream baos=new ByteArrayOutputStream(0);
CGIRunner.runCGIProgram(commands,environmentVariablesAsArray,userHomeDir,response,baos,null);
CGIRunner.runCGIProgram(commands,environmentVariablesAsArray,userHomeDir,response,baos,null, -1);

Writer w = new OutputStreamWriter(outputStream, "UTF-8");
w.write(baos.toString());
w.close();
getUserJobInfo(queryString, userDataDir, baos.toString());

} else {
CGIRunner.runCGIProgram(commands,environmentVariablesAsArray,userHomeDir,response,outputStream,null);
CGIRunner.runCGIProgram(commands,environmentVariablesAsArray,userHomeDir,response,outputStream,null, -1);

}
} catch (Exception e) {
Expand Down
Loading

0 comments on commit 56a97a7

Please sign in to comment.