From 958ed27c528e87518343779008810d1e03a4dc2e Mon Sep 17 00:00:00 2001 From: David Gauldie Date: Tue, 17 Sep 2024 15:47:45 -0400 Subject: [PATCH] 4790 bug app fails to extract configurations from timeseries datasets (#4830) --- packages/funman/Dockerfile | 29 ++++++--- packages/funman/dev.sh | 4 +- packages/gollm/Dockerfile | 64 ++++++++++++------- packages/gollm/dev.sh | 4 +- packages/gollm/gollm-version.txt | 2 +- packages/gollm/tasks/__init__.py | 0 .../tasks/configure_model_from_dataset.py | 2 +- packages/mira/Dockerfile | 53 +++++++++++---- packages/mira/dev.sh | 4 +- packages/mira/setup.py | 2 +- .../controller/gollm/GoLLMController.java | 5 ++ ...figureModelFromDatasetResponseHandler.java | 4 +- 12 files changed, 119 insertions(+), 54 deletions(-) create mode 100644 packages/gollm/tasks/__init__.py diff --git a/packages/funman/Dockerfile b/packages/funman/Dockerfile index bfe2dbf6f7..cee0b26ed3 100644 --- a/packages/funman/Dockerfile +++ b/packages/funman/Dockerfile @@ -11,13 +11,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ software-properties-common RUN add-apt-repository ppa:deadsnakes/ppa RUN apt-get update && apt-get install -y --no-install-recommends \ - python3.10 \ - gcc \ - python3-dev \ - wget \ - git \ - python3-pip && \ - rm -rf /var/lib/apt/lists/* + python3.8 \ + python3.8-venv \ + gcc \ + git \ + wget && \ + rm -rf /var/lib/apt/lists/* + +# Set the default Python version to 3.8 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 && \ + update-alternatives --set python /usr/bin/python3.8 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 && \ + update-alternatives --set python3 /usr/bin/python3.8 + +# install pip for python 3.8 +RUN python -m ensurepip --upgrade +RUN /bin/sh -c set -eux [ -s "/usr/local/bin/pip3" ]; [ ! -e "/usr/local/bin/pip" ]; ln -svT "pip3" "/usr/local/bin/pip"; + +# Verify the installation +RUN python --version +RUN pip --version # Install funman-api COPY ./packages/funman/funman-version.txt /funmanVersion.txt @@ -51,7 +64,7 @@ WORKDIR / # Install OpenJDK JRE and wget RUN apt-get update && \ apt-get install -y --no-install-recommends \ - openjdk-17-jre-headless && \ + openjdk-21-jre-headless && \ rm -rf /var/lib/apt/lists/* # Copy the Spring Boot fat JAR from the builder image diff --git a/packages/funman/dev.sh b/packages/funman/dev.sh index 3b858c2b95..43ea70a0e1 100755 --- a/packages/funman/dev.sh +++ b/packages/funman/dev.sh @@ -3,12 +3,12 @@ # ensure the volume mounted python code is using editable mode echo "Installing python tasks" cd /funman_task -pip install -e . +pip install --no-cache-dir -e . # run it echo "Running taskrunner" cd /taskrunner -pip install -e . +pip install --no-cache-dir -e . BUILD_DIR=/taskrunner-build-funman mkdir -p $BUILD_DIR diff --git a/packages/gollm/Dockerfile b/packages/gollm/Dockerfile index d97d69d231..e3e9f431e6 100644 --- a/packages/gollm/Dockerfile +++ b/packages/gollm/Dockerfile @@ -11,34 +11,36 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ software-properties-common RUN add-apt-repository ppa:deadsnakes/ppa RUN apt-get update && apt-get install -y --no-install-recommends \ - software-properties-common \ - python3.11 \ - python3.11-venv \ - python3.11-dev \ - wget \ - python3-pip && \ - rm -rf /var/lib/apt/lists/* + python3.11 \ + python3.11-venv \ + wget && \ + rm -rf /var/lib/apt/lists/* # Set the default Python version to 3.11 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 && \ - update-alternatives --set python /usr/bin/python3.11 + update-alternatives --set python /usr/bin/python3.11 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ + update-alternatives --set python3 /usr/bin/python3.11 +# install pip for python 3.11 RUN python -m ensurepip --upgrade +RUN /bin/sh -c set -eux [ -s "/usr/local/bin/pip3" ]; [ ! -e "/usr/local/bin/pip" ]; ln -svT "pip3" "/usr/local/bin/pip"; # Verify the installation RUN python --version -RUN pip3 --version +RUN pip --version # Install gollm COPY ./packages/gollm/gollm-version.txt /gollmVersion.txt RUN COMMIT_SHA="$(cat /gollmVersion.txt)" && \ - echo "Using GoLLM commit $COMMIT_SHA" && \ - wget --progress=dot:giga -O gollm.tar.gz "https://github.com/DARPA-ASKEM/GoLLM/archive/${COMMIT_SHA}.tar.gz" && \ - tar -zxvf gollm.tar.gz && \ - rm gollm.tar.gz && \ - mv GoLLM-* GoLLM + echo "Using GoLLM commit $COMMIT_SHA" && \ + wget --progress=dot:giga -O gollm.tar.gz "https://github.com/DARPA-ASKEM/GoLLM/archive/${COMMIT_SHA}.tar.gz" && \ + tar -zxvf gollm.tar.gz && \ + rm gollm.tar.gz && \ + mv GoLLM-* GoLLM + WORKDIR /GoLLM -RUN pip3 install --no-cache-dir . +RUN pip install --no-cache-dir . #^^^^^^^^^^^^^^^^^^^^ ###################### @@ -52,17 +54,32 @@ RUN ./gradlew bootJar WORKDIR / # ------------------------------------------------------------------------------ - -# Set up the Python image with JRE -FROM python:3.11-slim +FROM eclipse-temurin:21-jre-noble WORKDIR / -# Install OpenJDK JRE and wget -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - openjdk-21-jre-headless && \ - rm -rf /var/lib/apt/lists/* +# Install Python +RUN apt-get update && apt-get install -y --no-install-recommends \ + software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3.11 \ + python3.11-venv && \ + rm -rf /var/lib/apt/lists/* + +# Set the default Python version to 3.11 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 && \ + update-alternatives --set python /usr/bin/python3.11 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ + update-alternatives --set python3 /usr/bin/python3.11 + +# install pip for python 3.11 +RUN python -m ensurepip --upgrade +RUN /bin/sh -c set -eux [ -s "/usr/local/bin/pip3" ]; [ ! -e "/usr/local/bin/pip" ]; ln -svT "pip3" "/usr/local/bin/pip"; + +# Verify the installation +RUN python --version +RUN pip --version # Copy the Spring Boot fat JAR from the builder image COPY --from=gollm_taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar @@ -85,4 +102,5 @@ WORKDIR /gollm_task RUN pip install --no-cache-dir -e . WORKDIR / + CMD ["java", "-jar", "taskrunner.jar"] diff --git a/packages/gollm/dev.sh b/packages/gollm/dev.sh index 3173e674ee..253f01cdef 100755 --- a/packages/gollm/dev.sh +++ b/packages/gollm/dev.sh @@ -3,12 +3,12 @@ # ensure the volume mounted python code is using editable mode echo "Installing python tasks" cd /gollm_task -pip3 install -e . +pip install --no-cache-dir -e . # run it echo "Running taskrunner" cd /taskrunner -pip3 install -e . +pip install --no-cache-dir -e . BUILD_DIR=/taskrunner-build-gollm mkdir -p $BUILD_DIR diff --git a/packages/gollm/gollm-version.txt b/packages/gollm/gollm-version.txt index a00e414e74..de7773f669 100644 --- a/packages/gollm/gollm-version.txt +++ b/packages/gollm/gollm-version.txt @@ -1 +1 @@ -b5e9a5e5b2ffa1d4b10fb53cedf4f27231d0f706 +eb7617dac99d0ec6e1dbf75bfca9f70a7892ec96 diff --git a/packages/gollm/tasks/__init__.py b/packages/gollm/tasks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/gollm/tasks/configure_model_from_dataset.py b/packages/gollm/tasks/configure_model_from_dataset.py index c9a843f0a1..acdd7cb75e 100644 --- a/packages/gollm/tasks/configure_model_from_dataset.py +++ b/packages/gollm/tasks/configure_model_from_dataset.py @@ -23,7 +23,7 @@ def main(): amr = json.dumps(input_model.amr, separators=(",", ":")) taskrunner.log("Sending request to OpenAI API") - response = model_config_from_dataset(dataset=input_model.dataset, amr=amr) + response = model_config_from_dataset(dataset=input_model.dataset, amr=amr, matrix=input_model.matrix) taskrunner.log("Received response from OpenAI API") taskrunner.write_output_dict_with_timeout({"response": response}) diff --git a/packages/mira/Dockerfile b/packages/mira/Dockerfile index 2908a83c74..ef909c1362 100644 --- a/packages/mira/Dockerfile +++ b/packages/mira/Dockerfile @@ -11,10 +11,24 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ software-properties-common RUN add-apt-repository ppa:deadsnakes/ppa RUN apt-get update && apt-get install -y --no-install-recommends \ - python3.10 \ - wget \ - python3-pip && \ - rm -rf /var/lib/apt/lists/* + python3.10 \ + python3.10-venv \ + wget && \ + rm -rf /var/lib/apt/lists/* + +# Set the default Python version to 3.10 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 && \ + update-alternatives --set python /usr/bin/python3.10 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 && \ + update-alternatives --set python3 /usr/bin/python3.10 + +# install pip for python 3.10 +RUN python -m ensurepip --upgrade +RUN /bin/sh -c set -eux [ -s "/usr/local/bin/pip3" ]; [ ! -e "/usr/local/bin/pip" ]; ln -svT "pip3" "/usr/local/bin/pip"; + +# Verify the installation +RUN python --version +RUN pip --version # Install Mira COPY ./packages/mira/mira-version.txt /miraVersion.txt @@ -37,17 +51,32 @@ COPY ./packages/taskrunner . RUN ./gradlew bootJar # ------------------------------------------------------------------------------ - -# Set up the Python image with JRE -FROM python:3.10-slim +FROM eclipse-temurin:21-jre-noble WORKDIR / -# Install OpenJDK JRE and wget -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - openjdk-21-jre-headless && \ - rm -rf /var/lib/apt/lists/* +# Install Python +RUN apt-get update && apt-get install -y --no-install-recommends \ + software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3.10 \ + python3.10-venv && \ + rm -rf /var/lib/apt/lists/* + +# Set the default Python version to 3.10 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 && \ + update-alternatives --set python /usr/bin/python3.10 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 && \ + update-alternatives --set python3 /usr/bin/python3.10 + +# install pip for python 3.10 +RUN python -m ensurepip --upgrade +RUN /bin/sh -c set -eux [ -s "/usr/local/bin/pip3" ]; [ ! -e "/usr/local/bin/pip" ]; ln -svT "pip3" "/usr/local/bin/pip"; + +# Verify the installation +RUN python --version +RUN pip --version # Copy the Spring Boot fat JAR from the builder image COPY --from=mira_taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar diff --git a/packages/mira/dev.sh b/packages/mira/dev.sh index a7ecae7b88..e736187e40 100755 --- a/packages/mira/dev.sh +++ b/packages/mira/dev.sh @@ -3,12 +3,12 @@ # ensure the volume mounted python code is using editable mode echo "Installing python tasks" cd /mira_task -pip install -e . +pip install --no-cache-dir -e . # run it echo "Running taskrunner" cd /taskrunner -pip install -e . +pip install --no-cache-dir -e . BUILD_DIR=/taskrunner-build-mira mkdir -p $BUILD_DIR diff --git a/packages/mira/setup.py b/packages/mira/setup.py index fef5152800..44e9952efd 100644 --- a/packages/mira/setup.py +++ b/packages/mira/setup.py @@ -13,5 +13,5 @@ "mira_task:amr_to_mmt=tasks.amr_to_mmt:main", ], }, - python_requires=">=3.8", + python_requires=">=3.10", ) diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/gollm/GoLLMController.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/gollm/GoLLMController.java index 6bbaa06b72..c864bc48a7 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/gollm/GoLLMController.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/gollm/GoLLMController.java @@ -392,6 +392,11 @@ public ResponseEntity createConfigureModelFromDatasetTask( model.get().setMetadata(null); input.setAmr(model.get().serializeWithoutTerariumFields()); + // set matrix string if provided + if (body != null && !body.getMatrixStr().isEmpty()) { + input.setMatrix(body.getMatrixStr()); + } + // Create the task final TaskRequest req = new TaskRequest(); req.setType(TaskType.GOLLM); diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/tasks/ConfigureModelFromDatasetResponseHandler.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/tasks/ConfigureModelFromDatasetResponseHandler.java index 2d52253297..f36c5acde5 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/tasks/ConfigureModelFromDatasetResponseHandler.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/tasks/ConfigureModelFromDatasetResponseHandler.java @@ -47,8 +47,8 @@ public static class Input { @JsonProperty("amr") String amr; - @JsonProperty("matrix_str") - String matrixStr; + @JsonProperty("matrix") + String matrix; } @Data