taskrunner fix (#3882)

containers/scripts/docker-compose-taskrunner.yml

@@ -1,14 +1,5 @@
 ---
 services:
-    taskrunner-dev-builder:
-        build:
-            context: ../..
-            dockerfile: ./packages/taskrunner/Dockerfile
-            target: taskrunner_dev_builder
-        container_name: taskrunner-dev-builder
-        volumes:
-            - ../../packages/taskrunner:/taskrunner
-
     gollm-taskrunner:
         build:
             context: ../..
@@ -24,8 +15,6 @@ services:
             TERARIUM_TASKRUNNER_REQUEST_TYPE: "gollm"
             OPENAI_API_KEY: "${secret_gollm_openai_key}"
         depends_on:
-            taskrunner-dev-builder:
-                condition: service_started
             rabbitmq:
                 condition: service_healthy
         extra_hosts:
@@ -49,8 +38,6 @@ services:
             TERARIUM_MQ_USERNAME: "terarium"
             TERARIUM_TASKRUNNER_REQUEST_TYPE: "mira"
         depends_on:
-            taskrunner-dev-builder:
-                condition: service_started
             rabbitmq:
                 condition: service_healthy
         extra_hosts:
@@ -74,8 +61,6 @@ services:
             TERARIUM_MQ_USERNAME: "terarium"
             TERARIUM_TASKRUNNER_REQUEST_TYPE: "funman"
         depends_on:
-            taskrunner-dev-builder:
-                condition: service_started
             rabbitmq:
                 condition: service_healthy
         extra_hosts:

packages/funman/Dockerfile

@@ -63,7 +63,6 @@ RUN pip install --no-cache-dir . && \
 	pip install --no-cache-dir auxiliary_packages/funman_demo
 
 # Install taskrunner
-COPY ./packages/taskrunner/src/test/resources/echo.py /echo.py
 COPY ./packages/taskrunner/setup.py /taskrunner/setup.py
 COPY ./packages/taskrunner/taskrunner.py /taskrunner/taskrunner.py
 WORKDIR /taskrunner

packages/funman/dev.sh

@@ -8,4 +8,8 @@ pip install -e .
 # run it
 echo "Running taskrunner"
 cd /taskrunner
-./gradlew --project-cache-dir /tmp/.gradle/$$ bootRun
+pip install -e .
+
+BUILD_DIR=/taskrunner-build-funman
+mkdir -p $BUILD_DIR
+./gradlew --project-cache-dir /tmp/.gradle/$$ -PcustomBuildDir=$BUILD_DIR bootRun

packages/gollm/Dockerfile

@@ -60,7 +60,6 @@ WORKDIR /GoLLM
 RUN pip install --no-cache-dir .
 
 # Install taskrunner
-COPY ./packages/taskrunner/src/test/resources/echo.py /echo.py
 COPY ./packages/taskrunner/setup.py /taskrunner/setup.py
 COPY ./packages/taskrunner/taskrunner.py /taskrunner/taskrunner.py
 WORKDIR /taskrunner

packages/gollm/dev.sh

@@ -8,4 +8,8 @@ pip install -e .
 # run it
 echo "Running taskrunner"
 cd /taskrunner
-./gradlew --project-cache-dir /tmp/.gradle/$$ bootRun
+pip install -e .
+
+BUILD_DIR=/taskrunner-build-mira
+mkdir -p $BUILD_DIR
+./gradlew --project-cache-dir /tmp/.gradle/$$ -PcustomBuildDir=$BUILD_DIR bootRun

packages/mira/Dockerfile

@@ -58,7 +58,6 @@ WORKDIR /mira
 RUN pip install --no-cache-dir .[sbml,biomodels]
 
 # Install taskrunner
-COPY ./packages/taskrunner/src/test/resources/echo.py /echo.py
 COPY ./packages/taskrunner/setup.py /taskrunner/setup.py
 COPY ./packages/taskrunner/taskrunner.py /taskrunner/taskrunner.py
 WORKDIR /taskrunner

packages/mira/dev.sh

@@ -8,4 +8,8 @@ pip install -e .
 # run it
 echo "Running taskrunner"
 cd /taskrunner
-./gradlew --project-cache-dir /tmp/.gradle/$$ bootRun
+pip install -e .
+
+BUILD_DIR=/taskrunner-build-gollm
+mkdir -p $BUILD_DIR
+./gradlew --project-cache-dir /tmp/.gradle/$$ -PcustomBuildDir=$BUILD_DIR bootRun

packages/taskrunner/README.md

@@ -27,11 +27,9 @@ Use the following as a reference for creating a taskrunner image:
 
 # Local Development:
 
-Everything is setup in the `docker-compose-taskrunner.yml` for local development. The docker-compose entries use the *builder* image and volume mounts the taskrunner directories into the running container. Both the java `taskrunner` package and the python repos will automatically pickup and local changes and hotswap accordingly.
+Everything is setup in the `docker-compose-taskrunner.yml` for local development. The docker-compose entries use the *builder* image and volume mounts the taskrunner directories into the running container. The python repos will automatically pickup and local changes and hotswap accordingly.
 
-This also runs an auxiliary container called `taskrunner-dev-builer` which re-builds the java code on changes.
-
-**NOTE: When starting these local images, it takes about a minute or so to build and start the java taskrunner.**
+**NOTE: When starting these local images, it takes about a minute or so to setup the python code and start the java taskrunner.**
 
 # Building docker images:
 

packages/taskrunner/build.gradle

@@ -8,6 +8,8 @@ group = 'software.uncharted'
 version = '1.0.0-SNAPSHOT'
 sourceCompatibility = '17'
 
+buildDir = project.hasProperty('customBuildDir') ? project.getProperty('customBuildDir') : buildDir
+
 configurations {
 	compileOnly {
 		extendsFrom annotationProcessor

packages/taskrunner/src/main/java/software/uncharted/terarium/taskrunner/TaskRunnerApplication.java

@@ -8,7 +8,7 @@
 @PropertySource("classpath:application.properties")
 public class TaskRunnerApplication {
 
-	public static void main(String[] args) {
+	public static void main(final String[] args) {
 		SpringApplication.run(TaskRunnerApplication.class, args);
 	}
 

packages/taskrunner/src/main/java/software/uncharted/terarium/taskrunner/service/Task.java

@@ -45,6 +45,8 @@ public class Task {
 	private StringBuilder stdout = new StringBuilder();
 	private StringBuilder stderr = new StringBuilder();
 
+	private final String ECHO_SCRIPT_PATH = getClass().getResource("/echo.py").getPath();
+
 	private int PROCESS_KILL_TIMEOUT_SECONDS = 10;
 
 	private int BYTES_PER_READ = 1024 * 1024;
@@ -82,13 +84,27 @@ private String getExtension(final String input) {
 	}
 
 	private void setup() throws IOException, InterruptedException {
-		if (getExtension(req.getScript()).equals("py")) {
+		if (req.getScript().equals("echo.py")) {
+
+			// raw python file, execute it through the runtime
+			final boolean fileExists = Files.exists(Paths.get(ECHO_SCRIPT_PATH));
+			if (!fileExists) {
+				throw new FileNotFoundException("Script file: " + ECHO_SCRIPT_PATH + " not found");
+			}
+			processBuilder = new ProcessBuilder("python3", ECHO_SCRIPT_PATH, "--id", req.getId().toString(),
+					"--input_pipe",
+					inputPipeName,
+					"--output_pipe",
+					outputPipeName,
+					"--progress_pipe",
+					progressPipeName);
+		} else if (getExtension(req.getScript()).equals("py")) {
 			// raw python file, execute it through the runtime
 			final boolean fileExists = Files.exists(Paths.get(req.getScript()));
 			if (!fileExists) {
 				throw new FileNotFoundException("Script file: " + req.getScript() + " not found");
 			}
-			processBuilder = new ProcessBuilder("python", req.getScript(), "--id", req.getId().toString(),
+			processBuilder = new ProcessBuilder("python3", req.getScript(), "--id", req.getId().toString(),
 					"--input_pipe",
 					inputPipeName,
 					"--output_pipe",