Update taskrunner images for easier python version management (#4845)

containers/scripts/docker-compose-taskrunner.yml

@@ -70,12 +70,12 @@ services:
       - ../../packages/taskrunner:/taskrunner
     command: /funman_task/dev.sh
 
-  nougat_cpu-taskrunner:
+  equation_extraction_cpu-taskrunner:
     build:
       context: ../..
-      dockerfile: ./packages/nougat_cpu/Dockerfile
-      target: nougat_cpu_taskrunner_builder
-    container_name: nougat_cpu-taskrunner
+      dockerfile: ./packages/equation_extraction/Dockerfile
+      target: equation_extraction_taskrunner_builder
+    container_name: equation_extraction_cpu-taskrunner
     networks:
       - terarium
     ports:
@@ -84,13 +84,13 @@ services:
       TERARIUM_MQ_ADDRESSES: "amqp://rabbitmq:5672"
       TERARIUM_MQ_PASSWORD: "terarium123"
       TERARIUM_MQ_USERNAME: "terarium"
-      TERARIUM_TASKRUNNER_REQUEST_TYPE: "nougat_cpu"
+      TERARIUM_TASKRUNNER_REQUEST_TYPE: "equation_extraction"
     depends_on:
       rabbitmq:
         condition: service_healthy
     extra_hosts:
       - "${local_host_name}:host-gateway"
     volumes:
-      - ../../packages/nougat_cpu:/nougat_task
+      - ../../packages/equation_extraction:/equation_extraction_task
       - ../../packages/taskrunner:/taskrunner
-    command: /nougat_task/dev.sh
+    command: /equation_extraction_task/dev.sh

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

gradlew

@@ -15,8 +15,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# SPDX-License-Identifier: Apache-2.0
-#
 
 ##############################################################################
 #
@@ -57,7 +55,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -86,8 +84,7 @@ done
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
 # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
-APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
-' "$PWD" ) || exit
+APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum

gradlew.bat

@@ -13,8 +13,6 @@
 @rem See the License for the specific language governing permissions and
 @rem limitations under the License.
 @rem
-@rem SPDX-License-Identifier: Apache-2.0
-@rem
 
 @if "%DEBUG%"=="" @echo off
 @rem ##########################################################################
@@ -45,11 +43,11 @@ set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
 if %ERRORLEVEL% equ 0 goto execute
 
-echo. 1>&2
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
-echo. 1>&2
-echo Please set the JAVA_HOME variable in your environment to match the 1>&2
-echo location of your Java installation. 1>&2
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
 
 goto fail
 
@@ -59,11 +57,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe
 
 if exist "%JAVA_EXE%" goto execute
 
-echo. 1>&2
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
-echo. 1>&2
-echo Please set the JAVA_HOME variable in your environment to match the 1>&2
-echo location of your Java installation. 1>&2
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
 
 goto fail
 

packages/nougat_cpu/.gitignore → packages/equation_extraction/.gitignore

@@ -4,7 +4,7 @@ build/
 !gradle/wrapper/gradle-wrapper.jar
 !**/src/main/**/build/
 !**/src/test/**/build/
-nougat_task.egg-info
+*.egg-info
 
 ### STS ###
 .apt_generated

packages/nougat_cpu/Dockerfile → packages/equation_extraction/Dockerfile

@@ -1,45 +1,36 @@
 # Build the Spring Boot application
-FROM eclipse-temurin:17.0.12_7-jdk-focal AS nougat_cpu_taskrunner_builder
+FROM python:3.11-slim AS equation_extraction_taskrunner_builder
 
 ###### DEV ONLY ######
 #VVVVVVVVVVVVVVVVVVVVV
 
 # These deps are installed only for use during local development
 
-# Install Python
-RUN apt-get update && apt-get install -y --no-install-recommends \
-	software-properties-common
-RUN add-apt-repository ppa:deadsnakes/ppa
+# Install OpenJDK21
 RUN apt-get update && apt-get install -y --no-install-recommends \
 	build-essential \
-	python3.10 \
-	python3.10-venv \
-	python3.10-dev \
-	python3-setuptools \
-	wget \
 	git \
 	libgl1 \
+	libglib2.0-0 \
 	poppler-utils \
-	python3-pip && \
+	wget && \
 	rm -rf /var/lib/apt/lists/*
 
-# Set the default Python version to 3.10
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 && \
-	update-alternatives --set python /usr/bin/python3.10
-
-RUN python -m ensurepip --upgrade
+RUN wget https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.4%2B7/OpenJDK21U-jdk_x64_linux_hotspot_21.0.4_7.tar.gz && \
+	tar xvf OpenJDK21U-jdk_x64_linux_hotspot_21.0.4_7.tar.gz && \
+	mv jdk-21.0.4+7 /usr/local/jdk-21 && \
+	rm OpenJDK21U-jdk_x64_linux_hotspot_21.0.4_7.tar.gz
 
-# Verify the installation
-RUN python --version
-RUN pip3 --version
+ENV JAVA_HOME=/usr/local/jdk-21
+ENV PATH="$JAVA_HOME/bin:$PATH"
 
-# Install nougat
-COPY ./packages/nougat_cpu/nougat-version.txt /nougatVersion.txt
-RUN COMMIT_SHA="$(cat /nougatVersion.txt)" && \
+# Install equation-extraction
+COPY ./packages/equation_extraction/equation-extraction-version.txt /extractionVersion.txt
+RUN COMMIT_SHA="$(cat /extractionVersion.txt)" && \
 	echo "Using document_intelligence commit $COMMIT_SHA" && \
-	wget --progress=dot:giga -O nougat.tar.gz "https://github.com/DARPA-ASKEM/document_intelligence/archive/${COMMIT_SHA}.tar.gz" && \
-	tar -zxvf nougat.tar.gz && \
-	rm nougat.tar.gz && \
+	wget --progress=dot:giga -O equation-extraction.tar.gz "https://github.com/DARPA-ASKEM/document_intelligence/archive/${COMMIT_SHA}.tar.gz" && \
+	tar -zxvf equation-extraction.tar.gz && \
+	rm equation-extraction.tar.gz && \
 	mv document_intelligence-* document_intelligence
 WORKDIR /document_intelligence/document_intelligence/fast_latex
 RUN pip3 install -r requirements.txt
@@ -63,30 +54,36 @@ RUN mv /taskrunner/build/libs/*.jar .
 RUN mv /terarium-1.0.0-SNAPSHOT.jar /taskrunner.jar
 
 # ------------------------------------------------------------------------------
-
-# Set up the Python image with JRE
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 WORKDIR /
 
-# Copy the supervisord configuration file
-COPY ./packages/nougat_cpu/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
-
-# Install OpenJDK JRE and wget
-RUN apt-get update && \
-	apt-get install -y --no-install-recommends \
+# Install OpenJRE21
+RUN apt-get update && apt-get install -y --no-install-recommends \
 	build-essential \
 	git \
 	libgl1 \
+	libglib2.0-0 \
 	poppler-utils \
-	openjdk-17-jre-headless && \
+	wget && \
 	rm -rf /var/lib/apt/lists/*
 
+RUN wget https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.4%2B7/OpenJDK21U-jre_x64_linux_hotspot_21.0.4_7.tar.gz && \
+	tar xvf OpenJDK21U-jre_x64_linux_hotspot_21.0.4_7.tar.gz && \
+	mv jdk-21.0.4+7-jre /usr/local/jre-21 && \
+	rm OpenJDK21U-jre_x64_linux_hotspot_21.0.4_7.tar.gz
+
+ENV JAVA_HOME=/usr/local/jre-21
+ENV PATH="$JAVA_HOME/bin:$PATH"
+
+# Copy the supervisord configuration file
+COPY ./packages/equation_extraction/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
 # Copy the Spring Boot fat JAR from the builder image
-COPY --from=nougat_cpu_taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar
+COPY --from=equation_extraction_taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar
 
-# Install nougat
-COPY --from=nougat_cpu_taskrunner_builder /document_intelligence /document_intelligence
+# Install equation-extraction
+COPY --from=equation_extraction_taskrunner_builder /document_intelligence /document_intelligence
 WORKDIR /document_intelligence/document_intelligence/fast_latex
 RUN pip install -r requirements.txt
 
@@ -97,8 +94,8 @@ WORKDIR /taskrunner
 RUN pip install --no-cache-dir -e .
 
 # Install document_intelligence tasks
-COPY ./packages/nougat_cpu /nougat_task
-WORKDIR /nougat_task
+COPY ./packages/equation_extraction /equation_extraction_task
+WORKDIR /equation_extraction_task
 RUN pip install --no-cache-dir -e .
 
 #Install uvicorn and supervisord

packages/nougat_cpu/dev.sh → packages/equation_extraction/dev.sh

@@ -2,7 +2,7 @@
 
 # ensure the volume mounted python code is using editable mode
 echo "Installing python tasks"
-cd /nougat_task
+cd /equation_extraction_task
 pip3 install -e .
 
 # run it
@@ -14,4 +14,4 @@ pip3 install -e .
 mkdir /var/log/supervisor
 
 echo "Starting supervisord"
-supervisord -c /nougat_task/supervisord.conf
+supervisord -c /equation_extraction_task/supervisord.conf

packages/nougat_cpu/setup.py → packages/equation_extraction/setup.py

@@ -1,13 +1,13 @@
 from setuptools import setup, find_packages
 
 setup(
-    name="nougat_task",
+    name="equation_extraction_task",
     version="0.1.0",
     packages=find_packages(),
     install_requires=[],
     entry_points={
         "console_scripts": [
-            "nougat_task:extract_equations=tasks.extraction:main",
+            "equation_extraction_task:extract_equations=tasks.extract_equations:main",
         ],
     },
     python_requires=">=3.10",

packages/funman/Dockerfile

@@ -1,36 +1,25 @@
 # Build the Spring Boot application
-FROM eclipse-temurin:21.0.4_7-jdk-noble AS funman_taskrunner_builder
+FROM ghcr.io/darpa-askem/funman-base:latest AS funman_taskrunner_builder
 
 ###### DEV ONLY ######
 #VVVVVVVVVVVVVVVVVVVVV
 
 # These deps are installed only for use during local development
 
-# Install Python
+# Install OpenJDK21
 RUN apt-get update && apt-get install -y --no-install-recommends \
-	software-properties-common
-RUN add-apt-repository ppa:deadsnakes/ppa
-RUN apt-get update && apt-get install -y --no-install-recommends \
- python3.8 \
- python3.8-venv \
- gcc \
- git \
- wget && \
- rm -rf /var/lib/apt/lists/*
-
-# Set the default Python version to 3.8
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 && \
- update-alternatives --set python /usr/bin/python3.8
-RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 && \
- update-alternatives --set python3 /usr/bin/python3.8
-
-# install pip for python 3.8
-RUN python -m ensurepip --upgrade
-RUN /bin/sh -c set -eux [ -s "/usr/local/bin/pip3" ]; [ ! -e "/usr/local/bin/pip" ]; ln -svT "pip3" "/usr/local/bin/pip";
-
-# Verify the installation
-RUN python --version
-RUN pip --version
+	gcc \
+	git \
+	wget && \
+	rm -rf /var/lib/apt/lists/*
+
+RUN wget https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.4%2B7/OpenJDK21U-jdk_x64_linux_hotspot_21.0.4_7.tar.gz && \
+	tar xvf OpenJDK21U-jdk_x64_linux_hotspot_21.0.4_7.tar.gz && \
+	mv jdk-21.0.4+7 /usr/local/jdk-21 && \
+	rm OpenJDK21U-jdk_x64_linux_hotspot_21.0.4_7.tar.gz
+
+ENV JAVA_HOME=/usr/local/jdk-21
+ENV PATH="$JAVA_HOME/bin:$PATH"
 
 # Install funman-api
 COPY ./packages/funman/funman-version.txt /funmanVersion.txt