4790 bug app fails to extract configurations from timeseries datasets (…

…#4830)
DARPA-ASKEM · Sep 17, 2024 · 958ed27 · 958ed27
1 parent e67d9f3
commit 958ed27
Show file tree

Hide file tree

Showing 12 changed files with 119 additions and 54 deletions.
diff --git a/packages/funman/Dockerfile b/packages/funman/Dockerfile
@@ -11,13 +11,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 	software-properties-common
 RUN add-apt-repository ppa:deadsnakes/ppa
 RUN apt-get update && apt-get install -y --no-install-recommends \
-	python3.10 \
-	gcc \
-	python3-dev \
-	wget \
-	git \
-	python3-pip && \
-	rm -rf /var/lib/apt/lists/*
+ python3.8 \
+ python3.8-venv \
+ gcc \
+ git \
+ wget && \
+ rm -rf /var/lib/apt/lists/*
+
+# Set the default Python version to 3.8
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 && \
+ update-alternatives --set python /usr/bin/python3.8
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 && \
+ update-alternatives --set python3 /usr/bin/python3.8
+
+# install pip for python 3.8
+RUN python -m ensurepip --upgrade
+RUN /bin/sh -c set -eux [ -s "/usr/local/bin/pip3" ]; [ ! -e "/usr/local/bin/pip" ]; ln -svT "pip3" "/usr/local/bin/pip";
+
+# Verify the installation
+RUN python --version
+RUN pip --version
 
 # Install funman-api
 COPY ./packages/funman/funman-version.txt /funmanVersion.txt
@@ -51,7 +64,7 @@ WORKDIR /
 # Install OpenJDK JRE and wget
 RUN apt-get update && \
 	apt-get install -y --no-install-recommends \
-	openjdk-17-jre-headless && \
+	openjdk-21-jre-headless && \
 	rm -rf /var/lib/apt/lists/*
 
 # Copy the Spring Boot fat JAR from the builder image

diff --git a/packages/funman/dev.sh b/packages/funman/dev.sh
@@ -3,12 +3,12 @@
 # ensure the volume mounted python code is using editable mode
 echo "Installing python tasks"
 cd /funman_task
-pip install -e .
+pip install --no-cache-dir -e .
 
 # run it
 echo "Running taskrunner"
 cd /taskrunner
-pip install -e .
+pip install --no-cache-dir -e .
 
 BUILD_DIR=/taskrunner-build-funman
 mkdir -p $BUILD_DIR

diff --git a/packages/gollm/Dockerfile b/packages/gollm/Dockerfile
@@ -11,34 +11,36 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 	software-properties-common
 RUN add-apt-repository ppa:deadsnakes/ppa
 RUN apt-get update && apt-get install -y --no-install-recommends \
-	software-properties-common \
-	python3.11 \
-	python3.11-venv \
-	python3.11-dev \
-	wget \
-	python3-pip && \
-	rm -rf /var/lib/apt/lists/*
+ python3.11 \
+ python3.11-venv \
+ wget && \
+ rm -rf /var/lib/apt/lists/*
 
 # Set the default Python version to 3.11
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 && \
-	update-alternatives --set python /usr/bin/python3.11
+ update-alternatives --set python /usr/bin/python3.11
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
+ update-alternatives --set python3 /usr/bin/python3.11
 
+# install pip for python 3.11
 RUN python -m ensurepip --upgrade
+RUN /bin/sh -c set -eux [ -s "/usr/local/bin/pip3" ]; [ ! -e "/usr/local/bin/pip" ]; ln -svT "pip3" "/usr/local/bin/pip";
 
 # Verify the installation
 RUN python --version
-RUN pip3 --version
+RUN pip --version
 
 # Install gollm
 COPY ./packages/gollm/gollm-version.txt /gollmVersion.txt
 RUN COMMIT_SHA="$(cat /gollmVersion.txt)" && \
-	echo "Using GoLLM commit $COMMIT_SHA" && \
-	wget --progress=dot:giga -O gollm.tar.gz "https://github.com/DARPA-ASKEM/GoLLM/archive/${COMMIT_SHA}.tar.gz" && \
-	tar -zxvf gollm.tar.gz && \
-	rm gollm.tar.gz && \
-	mv GoLLM-* GoLLM
+ echo "Using GoLLM commit $COMMIT_SHA" && \
+ wget --progress=dot:giga -O gollm.tar.gz "https://github.com/DARPA-ASKEM/GoLLM/archive/${COMMIT_SHA}.tar.gz" && \
+ tar -zxvf gollm.tar.gz && \
+ rm gollm.tar.gz && \
+ mv GoLLM-* GoLLM
+
 WORKDIR /GoLLM
-RUN pip3 install --no-cache-dir .
+RUN pip install --no-cache-dir .
 
 #^^^^^^^^^^^^^^^^^^^^
 ######################
@@ -52,17 +54,32 @@ RUN ./gradlew bootJar
 WORKDIR /
 
 # ------------------------------------------------------------------------------
-
-# Set up the Python image with JRE
-FROM python:3.11-slim
+FROM eclipse-temurin:21-jre-noble
 
 WORKDIR /
 
-# Install OpenJDK JRE and wget
-RUN apt-get update && \
-	apt-get install -y --no-install-recommends \
-	openjdk-21-jre-headless && \
-	rm -rf /var/lib/apt/lists/*
+# Install Python
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ python3.11 \
+ python3.11-venv && \
+ rm -rf /var/lib/apt/lists/*
+
+# Set the default Python version to 3.11
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 && \
+ update-alternatives --set python /usr/bin/python3.11
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
+ update-alternatives --set python3 /usr/bin/python3.11
+
+# install pip for python 3.11
+RUN python -m ensurepip --upgrade
+RUN /bin/sh -c set -eux [ -s "/usr/local/bin/pip3" ]; [ ! -e "/usr/local/bin/pip" ]; ln -svT "pip3" "/usr/local/bin/pip";
+
+# Verify the installation
+RUN python --version
+RUN pip --version
 
 # Copy the Spring Boot fat JAR from the builder image
 COPY --from=gollm_taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar
@@ -85,4 +102,5 @@ WORKDIR /gollm_task
 RUN pip install --no-cache-dir -e .
 
 WORKDIR /
+
 CMD ["java", "-jar", "taskrunner.jar"]
diff --git a/packages/gollm/dev.sh b/packages/gollm/dev.sh
@@ -3,12 +3,12 @@
 # ensure the volume mounted python code is using editable mode
 echo "Installing python tasks"
 cd /gollm_task
-pip3 install -e .
+pip install --no-cache-dir -e .
 
 # run it
 echo "Running taskrunner"
 cd /taskrunner
-pip3 install -e .
+pip install --no-cache-dir -e .
 
 BUILD_DIR=/taskrunner-build-gollm
 mkdir -p $BUILD_DIR

diff --git a/packages/gollm/gollm-version.txt b/packages/gollm/gollm-version.txt
@@ -1 +1 @@
-b5e9a5e5b2ffa1d4b10fb53cedf4f27231d0f706
+eb7617dac99d0ec6e1dbf75bfca9f70a7892ec96
diff --git a/packages/gollm/tasks/__init__.py b/packages/gollm/tasks/__init__.py
diff --git a/packages/gollm/tasks/configure_model_from_dataset.py b/packages/gollm/tasks/configure_model_from_dataset.py
@@ -23,7 +23,7 @@ def main():
         amr = json.dumps(input_model.amr, separators=(",", ":"))
 
         taskrunner.log("Sending request to OpenAI API")
-        response = model_config_from_dataset(dataset=input_model.dataset, amr=amr)
+        response = model_config_from_dataset(dataset=input_model.dataset, amr=amr, matrix=input_model.matrix)
         taskrunner.log("Received response from OpenAI API")
 
         taskrunner.write_output_dict_with_timeout({"response": response})

diff --git a/packages/mira/Dockerfile b/packages/mira/Dockerfile
@@ -11,10 +11,24 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 	software-properties-common
 RUN add-apt-repository ppa:deadsnakes/ppa
 RUN apt-get update && apt-get install -y --no-install-recommends \
-	python3.10 \
-	wget \
-	python3-pip && \
-	rm -rf /var/lib/apt/lists/*
+ python3.10 \
+ python3.10-venv \
+ wget && \
+ rm -rf /var/lib/apt/lists/*
+
+# Set the default Python version to 3.10
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 && \
+ update-alternatives --set python /usr/bin/python3.10
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 && \
+ update-alternatives --set python3 /usr/bin/python3.10
+
+# install pip for python 3.10
+RUN python -m ensurepip --upgrade
+RUN /bin/sh -c set -eux [ -s "/usr/local/bin/pip3" ]; [ ! -e "/usr/local/bin/pip" ]; ln -svT "pip3" "/usr/local/bin/pip";
+
+# Verify the installation
+RUN python --version
+RUN pip --version
 
 # Install Mira
 COPY ./packages/mira/mira-version.txt /miraVersion.txt
@@ -37,17 +51,32 @@ COPY ./packages/taskrunner .
 RUN ./gradlew bootJar
 
 # ------------------------------------------------------------------------------
-
-# Set up the Python image with JRE
-FROM python:3.10-slim
+FROM eclipse-temurin:21-jre-noble
 
 WORKDIR /
 
-# Install OpenJDK JRE and wget
-RUN apt-get update && \
-	apt-get install -y --no-install-recommends \
-	openjdk-21-jre-headless && \
-	rm -rf /var/lib/apt/lists/*
+# Install Python
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ python3.10 \
+ python3.10-venv && \
+ rm -rf /var/lib/apt/lists/*
+
+# Set the default Python version to 3.10
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 && \
+ update-alternatives --set python /usr/bin/python3.10
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 && \
+ update-alternatives --set python3 /usr/bin/python3.10
+
+# install pip for python 3.10
+RUN python -m ensurepip --upgrade
+RUN /bin/sh -c set -eux [ -s "/usr/local/bin/pip3" ]; [ ! -e "/usr/local/bin/pip" ]; ln -svT "pip3" "/usr/local/bin/pip";
+
+# Verify the installation
+RUN python --version
+RUN pip --version
 
 # Copy the Spring Boot fat JAR from the builder image
 COPY --from=mira_taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar

diff --git a/packages/mira/dev.sh b/packages/mira/dev.sh
@@ -3,12 +3,12 @@
 # ensure the volume mounted python code is using editable mode
 echo "Installing python tasks"
 cd /mira_task
-pip install -e .
+pip install --no-cache-dir -e .
 
 # run it
 echo "Running taskrunner"
 cd /taskrunner
-pip install -e .
+pip install --no-cache-dir -e .
 
 BUILD_DIR=/taskrunner-build-mira
 mkdir -p $BUILD_DIR

diff --git a/packages/mira/setup.py b/packages/mira/setup.py
@@ -13,5 +13,5 @@
             "mira_task:amr_to_mmt=tasks.amr_to_mmt:main",
         ],
     },
-    python_requires=">=3.8",
+    python_requires=">=3.10",
 )
diff --git a/...src/main/java/software/uncharted/terarium/hmiserver/controller/gollm/GoLLMController.java b/...src/main/java/software/uncharted/terarium/hmiserver/controller/gollm/GoLLMController.java
@@ -392,6 +392,11 @@ public ResponseEntity<TaskResponse> createConfigureModelFromDatasetTask(
 		model.get().setMetadata(null);
 		input.setAmr(model.get().serializeWithoutTerariumFields());
 
+		// set matrix string if provided
+		if (body != null && !body.getMatrixStr().isEmpty()) {
+			input.setMatrix(body.getMatrixStr());
+		}
+
 		// Create the task
 		final TaskRequest req = new TaskRequest();
 		req.setType(TaskType.GOLLM);

diff --git a/.../uncharted/terarium/hmiserver/service/tasks/ConfigureModelFromDatasetResponseHandler.java b/.../uncharted/terarium/hmiserver/service/tasks/ConfigureModelFromDatasetResponseHandler.java
@@ -47,8 +47,8 @@ public static class Input {
 		@JsonProperty("amr")
 		String amr;
 
-		@JsonProperty("matrix_str")
-		String matrixStr;
+		@JsonProperty("matrix")
+		String matrix;
 	}
 
 	@Data