Run xxx-taskrunner develop containerrs with hotswapping (#3845)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

containers/scripts/docker-compose-taskrunner.yml

@@ -1,50 +1,85 @@
 ---
 services:
-  gollm-taskrunner:
-    container_name: gollm-taskrunner
-    image: ghcr.io/darpa-askem/gollm-taskrunner:latest
-    networks:
-      - terarium
-    environment:
-      TERARIUM_MQ-ADDRESSES: "amqp://rabbitmq:5672"
-      TERARIUM_MQ-PASSWORD: "terarium123"
-      TERARIUM_MQ-USERNAME: "terarium"
-      TERARIUM_TASKRUNNER_REQUEST-TYPE: "gollm"
-      OPENAI_API_KEY: "${secret_gollm_openai_key}"
-    depends_on:
-      rabbitmq:
-        condition: service_healthy
-    extra_hosts:
-      - "${local_host_name}:host-gateway"
+    taskrunner-dev-builder:
+        build:
+            dockerfile: ../../packages/taskrunner/Dockerfile
+            target: taskrunner_dev_builder
+        container_name: taskrunner-dev-builder
+        volumes:
+            - ../../packages/taskrunner:/taskrunner
 
-  mira-taskrunner:
-    container_name: mira-taskrunner
-    image: ghcr.io/darpa-askem/mira-taskrunner:latest
-    networks:
-      - terarium
-    environment:
-      TERARIUM_MQ-ADDRESSES: "amqp://rabbitmq:5672"
-      TERARIUM_MQ-PASSWORD: "terarium123"
-      TERARIUM_MQ-USERNAME: "terarium"
-      TERARIUM_TASKRUNNER_REQUEST-TYPE: "mira"
-    depends_on:
-      rabbitmq:
-        condition: service_healthy
-    extra_hosts:
-      - "${local_host_name}:host-gateway"
+    gollm-taskrunner:
+        build:
+            dockerfile: ./packages/gollm/Dockerfile
+            target: gollm_taskrunner_builder
+        container_name: gollm-taskrunner
+        image: ghcr.io/darpa-askem/gollm-taskrunner:latest
+        networks:
+            - terarium
+        environment:
+            TERARIUM_MQ_ADDRESSES: "amqp://rabbitmq:5672"
+            TERARIUM_MQ_PASSWORD: "terarium123"
+            TERARIUM_MQ_USERNAME: "terarium"
+            TERARIUM_TASKRUNNER_REQUEST_TYPE: "gollm"
+            OPENAI_API_KEY: "${secret_gollm_openai_key}"
+        depends_on:
+            taskrunner-dev-builder:
+                condition: service_started
+            rabbitmq:
+                condition: service_healthy
+        extra_hosts:
+            - "${local_host_name}:host-gateway"
+        volumes:
+            - ../../packages/gollm:/gollm_task
+            - ../../packages/taskrunner:/taskrunner
+        command: /gollm_task/dev.sh
 
-  funman-taskrunner:
-    container_name: funman-taskrunner
-    image: ghcr.io/darpa-askem/funman-taskrunner:latest
-    networks:
-      - terarium
-    environment:
-      TERARIUM_MQ-ADDRESSES: "amqp://rabbitmq:5672"
-      TERARIUM_MQ-PASSWORD: "terarium123"
-      TERARIUM_MQ-USERNAME: "terarium"
-      TERARIUM_TASKRUNNER_REQUEST-TYPE: "funman"
-    depends_on:
-      rabbitmq:
-        condition: service_healthy
-    extra_hosts:
-      - "${local_host_name}:host-gateway"
+    mira-taskrunner:
+        build:
+            dockerfile: ../../packages/mira/Dockerfile
+            target: mira_taskrunner_builder
+        container_name: mira-taskrunner
+        image: ghcr.io/darpa-askem/mira-taskrunner:latest
+        networks:
+            - terarium
+        environment:
+            TERARIUM_MQ_ADDRESSES: "amqp://rabbitmq:5672"
+            TERARIUM_MQ_PASSWORD: "terarium123"
+            TERARIUM_MQ_USERNAME: "terarium"
+            TERARIUM_TASKRUNNER_REQUEST_TYPE: "mira"
+        depends_on:
+            taskrunner-dev-builder:
+                condition: service_started
+            rabbitmq:
+                condition: service_healthy
+        extra_hosts:
+            - "${local_host_name}:host-gateway"
+        volumes:
+            - ../../packages/mira:/mira_task
+            - ../../packages/taskrunner:/taskrunner
+        command: /mira_task/dev.sh
+
+    funman-taskrunner:
+        build:
+            dockerfile: ../../packages/funman/Dockerfile
+            target: funman_taskrunner_builder
+        container_name: funman-taskrunner
+        image: ghcr.io/darpa-askem/funman-taskrunner:latest
+        networks:
+            - terarium
+        environment:
+            TERARIUM_MQ_ADDRESSES: "amqp://rabbitmq:5672"
+            TERARIUM_MQ_PASSWORD: "terarium123"
+            TERARIUM_MQ_USERNAME: "terarium"
+            TERARIUM_TASKRUNNER_REQUEST_TYPE: "funman"
+        depends_on:
+            taskrunner-dev-builder:
+                condition: service_started
+            rabbitmq:
+                condition: service_healthy
+        extra_hosts:
+            - "${local_host_name}:host-gateway"
+        volumes:
+            - ../../packages/funman:/funman_task
+            - ../../packages/taskrunner:/taskrunner
+        command: /funman_task/dev.sh

packages/funman/.gitignore

@@ -0,0 +1,40 @@
+HELP.md
+.gradle
+build/
+!gradle/wrapper/gradle-wrapper.jar
+!**/src/main/**/build/
+!**/src/test/**/build/
+funman_task.egg-info
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+bin/
+!**/src/main/**/bin/
+!**/src/test/**/bin/
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+out/
+!**/src/main/**/out/
+!**/src/test/**/out/
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+
+### VS Code ###
+.vscode/
+mira.egg-info
+__pycache__

packages/funman/Dockerfile

@@ -1,5 +1,37 @@
 # Build the Spring Boot application
-FROM eclipse-temurin:17.0.11_9-jdk-focal AS taskrunner_builder
+FROM eclipse-temurin:17.0.11_9-jdk-focal AS funman_taskrunner_builder
+
+###### DEV ONLY ######
+#VVVVVVVVVVVVVVVVVVVVV
+
+# These deps are installed only for use during local development
+
+# Install Python
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	python3.10 \
+	wget \
+	git \
+	python3-pip && \
+	rm -rf /var/lib/apt/lists/*
+
+# Install funman-api
+COPY ./packages/funman/funman-version.txt /funmanVersion.txt
+RUN COMMIT_SHA="$(cat /funmanVersion.txt)" && \
+	echo "Using FUNMAN commit $COMMIT_SHA" && \
+	wget --progress=dot:giga -O funman.tar.gz "https://github.com/DARPA-ASKEM/funman-api/archive/${COMMIT_SHA}.tar.gz" && \
+	tar -zxvf funman.tar.gz && \
+	rm funman.tar.gz && \
+	mv -v funman-* funman-api
+WORKDIR /funman-api
+RUN pip install --no-cache-dir . && \
+	pip install --no-cache-dir auxiliary_packages/funman_dreal && \
+	pip install --no-cache-dir auxiliary_packages/funman_demo
+
+#^^^^^^^^^^^^^^^^^^^^
+######################
 
 WORKDIR /taskrunner
 
@@ -14,37 +46,26 @@ FROM ghcr.io/darpa-askem/funman-base:latest
 
 WORKDIR /
 
-# Install OpenJDK JRE and git
+# Install OpenJDK JRE and wget
 RUN apt-get update && \
-	apt-get install -y --no-install-recommends openjdk-17-jre-headless && \
-  apt-get install -y --no-install-recommends git && \
+	apt-get install -y --no-install-recommends \
+	openjdk-17-jre-headless && \
 	rm -rf /var/lib/apt/lists/*
 
 # Copy the Spring Boot fat JAR from the builder image
-COPY --from=taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar
-
-# Copy the echo script for testing
-COPY ./packages/taskrunner/src/test/resources/echo.py /echo.py
+COPY --from=funman_taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar
 
 # Install funman-api
-RUN echo "Cache bust so we can always pull latest Funman" $(date -u)
-COPY ./packages/funman/funman-version.txt /funmanVersion.txt
-
-RUN git clone https://github.com/DARPA-ASKEM/funman-api.git
-
+COPY --from=funman_taskrunner_builder /funman-api /funman-api
 WORKDIR /funman-api
-RUN COMMIT_SHA=$(cat /funmanVersion.txt) && \
-    echo "Using FUNMAN commit $COMMIT_SHA" && \
-    git reset --hard $COMMIT_SHA
 RUN pip install --no-cache-dir . && \
-    pip install --no-cache-dir auxiliary_packages/funman_dreal && \
-    pip install --no-cache-dir auxiliary_packages/funman_demo
+	pip install --no-cache-dir auxiliary_packages/funman_dreal && \
+	pip install --no-cache-dir auxiliary_packages/funman_demo
 
 # Install taskrunner
-WORKDIR /
+COPY ./packages/taskrunner/src/test/resources/echo.py /echo.py
 COPY ./packages/taskrunner/setup.py /taskrunner/setup.py
 COPY ./packages/taskrunner/taskrunner.py /taskrunner/taskrunner.py
-
 WORKDIR /taskrunner
 RUN pip install --no-cache-dir -e .
 

packages/funman/dev.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# ensure the volume mounted python code is using editable mode
+echo "Installing python tasks"
+cd /funman_task
+pip install -e .
+
+# run it
+echo "Running taskrunner"
+cd /taskrunner
+./gradlew --project-cache-dir /tmp/.gradle/$$ bootRun

packages/gollm/.gitignore

@@ -0,0 +1,40 @@
+HELP.md
+.gradle
+build/
+!gradle/wrapper/gradle-wrapper.jar
+!**/src/main/**/build/
+!**/src/test/**/build/
+gollm_task.egg-info
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+bin/
+!**/src/main/**/bin/
+!**/src/test/**/bin/
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+out/
+!**/src/main/**/out/
+!**/src/test/**/out/
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+
+### VS Code ###
+.vscode/
+mira.egg-info
+__pycache__

packages/gollm/Dockerfile

@@ -1,12 +1,43 @@
 # Build the Spring Boot application
-FROM eclipse-temurin:17.0.11_9-jdk-focal AS taskrunner_builder
+FROM eclipse-temurin:17.0.11_9-jdk-focal AS gollm_taskrunner_builder
+
+###### DEV ONLY ######
+#VVVVVVVVVVVVVVVVVVVVV
+
+# These deps are installed only for use during local development
+
+# Install Python
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	python3.10 \
+	wget \
+	python3-pip && \
+	rm -rf /var/lib/apt/lists/*
+
+# Install gollm
+COPY ./packages/gollm/gollm-version.txt /gollmVersion.txt
+RUN COMMIT_SHA="$(cat /gollmVersion.txt)" && \
+	echo "Using GoLLM commit $COMMIT_SHA" && \
+	wget --progress=dot:giga -O gollm.tar.gz "https://github.com/DARPA-ASKEM/GoLLM/archive/${COMMIT_SHA}.tar.gz" && \
+	tar -zxvf gollm.tar.gz && \
+	rm gollm.tar.gz && \
+	mv GoLLM-* GoLLM
+WORKDIR /GoLLM
+RUN pip install --no-cache-dir .
+
+#^^^^^^^^^^^^^^^^^^^^
+######################
 
 WORKDIR /taskrunner
 
 COPY ./packages/taskrunner .
 
 RUN ./gradlew bootJar
 
+WORKDIR /
+
 # ------------------------------------------------------------------------------
 
 # Set up the Python image with JRE
@@ -16,35 +47,22 @@ WORKDIR /
 
 # Install OpenJDK JRE and wget
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends openjdk-17-jre-headless && \
-    apt-get install -y --no-install-recommends wget && \
-    rm -rf /var/lib/apt/lists/*
+	apt-get install -y --no-install-recommends \
+	openjdk-17-jre-headless && \
+	rm -rf /var/lib/apt/lists/*
 
 # Copy the Spring Boot fat JAR from the builder image
-COPY --from=taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar
-
-# Copy the echo script for testing
-COPY ./packages/taskrunner/src/test/resources/echo.py /echo.py
+COPY --from=gollm_taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar
 
 # Install GoLLM
-RUN echo "Cache bust so we can always pull latest GoLLM" $(date -u)
-COPY ./packages/gollm/gollm-version.txt /gollmVersion.txt
-RUN COMMIT_SHA=$(cat /gollmVersion.txt) && \
-    echo "Using GoLLM commit $COMMIT_SHA" && \
-    wget -O gollm.tar.gz https://github.com/DARPA-ASKEM/GoLLM/archive/${COMMIT_SHA}.tar.gz
-
-RUN tar -zxvf gollm.tar.gz && \
-    rm gollm.tar.gz
-RUN mv GoLLM-* GoLLM
-
+COPY --from=gollm_taskrunner_builder /GoLLM /GoLLM
 WORKDIR /GoLLM
-RUN pip install --no-cache-dir -e .
+RUN pip install --no-cache-dir .
 
 # Install taskrunner
-WORKDIR /
+COPY ./packages/taskrunner/src/test/resources/echo.py /echo.py
 COPY ./packages/taskrunner/setup.py /taskrunner/setup.py
 COPY ./packages/taskrunner/taskrunner.py /taskrunner/taskrunner.py
-
 WORKDIR /taskrunner
 RUN pip install --no-cache-dir -e .