Add unittests for custom Python environments (triton-inference-server#2897)

* Add unittests for Python environments

* Fix libarchive dependency

* Add checks for installed NumPy version

Author: Tabrizian

Dockerfile.QA

@@ -252,9 +252,11 @@ RUN if [ $(cat /etc/os-release | grep 'VERSION_ID="20.04"' | wc -l) -ne 0 ]; the
     fi
 
 # CI/QA for memcheck requires valgrind
+# libarchive-dev is required by Python backend
 RUN apt-get update && apt-get install -y --no-install-recommends \
                               curl \
                               libopencv-dev \
+                              libarchive-dev \
                               libopencv-core-dev \
                               libzmq3-dev \
                               python3-dev \

build.py

@@ -489,7 +489,7 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap, backends):
 
 # libcurl4-openSSL-dev is needed for GCS
 # python3-dev is needed by Torchvision
-# python3-pip is needed by python backend
+# python3-pip and libarchive-dev is needed by python backend
 # uuid-dev and pkg-config is needed for Azure Storage
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -513,14 +513,13 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap, backends):
             unzip \
             wget \
             zlib1g-dev \
+            libarchive-dev \
             pkg-config \
             uuid-dev && \
     rm -rf /var/lib/apt/lists/*
 
-# grpcio-tools grpcio-channelz are needed by python backend
 RUN pip3 install --upgrade pip && \
-    pip3 install --upgrade wheel setuptools docker && \
-    pip3 install grpcio-tools grpcio-channelz
+    pip3 install --upgrade wheel setuptools docker
 
 # Server build requires recent version of CMake (FetchContent required)
 RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \
@@ -648,11 +647,11 @@ def create_dockerfile_linux(ddir, dockerfile_name, argmap, backends, repoagents,
 # python3, python3-pip and some pip installs required for the python backend
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-         python3 \
+         python3 libarchive-dev \
          python3-pip && \
     pip3 install --upgrade pip && \
     pip3 install --upgrade wheel setuptools && \
-    pip3 install --upgrade grpcio-tools grpcio-channelz numpy && \
+    pip3 install --upgrade numpy && \
     rm -rf /var/lib/apt/lists/*
 '''
     df += '''

qa/L0_backend_python/test.sh

@@ -31,6 +31,7 @@ EXPECTED_NUM_TESTS="10"
 
 SERVER=/opt/tritonserver/bin/tritonserver
 BASE_SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1"
+PYTHON_BACKEND_BRANCH=$PYTHON_BACKEND_REPO_TAG
 SERVER_ARGS=$BASE_SERVER_ARGS
 SERVER_LOG="./inference_server.log"
 REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION}
@@ -42,6 +43,34 @@ get_shm_pages() {
   echo ${#shm_pages[@]}
 }
 
+install_conda() {
+  rm -rf ./miniconda
+  file_name="Miniconda3-py38_4.9.2-Linux-x86_64.sh"
+  wget https://repo.anaconda.com/miniconda/$file_name
+
+  # install miniconda in silent mode
+  bash $file_name -p ./miniconda -b
+
+  # activate conda
+  eval "$(./miniconda/bin/conda shell.bash hook)"
+}
+
+create_conda_env() {
+  python_version=$1
+  env_name=$2
+  conda create -n $env_name python=$python_version -y
+  conda activate $env_name
+  conda install conda-pack -y
+}
+
+create_python_backend_stub() {
+  rm -rf python_backend
+  git clone https://github.com/triton-inference-server/python_backend -b $PYTHON_BACKEND_BRANCH
+  (cd python_backend/ && mkdir builddir && cd builddir && \
+  cmake -DTRITON_BACKEND_REPO_TAG=$TRITON_BACKEND_REPO_TAG -DTRITON_COMMON_REPO_TAG=$TRITON_COMMON_REPO_TAG -DTRITON_CORE_REPO_TAG=$TRITON_CORE_REPO_TAG ../ && \
+  make -j18 triton-python-backend-stub)
+}
+
 rm -fr *.log ./models
 
 mkdir -p models/identity_fp32/1/
@@ -163,6 +192,7 @@ set -e
 
 kill $SERVER_PID
 wait $SERVER_PID
+sleep 4
 
 current_num_pages=`get_shm_pages`
 if [ $current_num_pages -ne $prev_num_pages ]; then
@@ -261,9 +291,11 @@ if [ "$SERVER_PID" == "0" ]; then
     cat $SERVER_LOG
     exit 1
 fi
+set -e
 
 kill $SERVER_PID
 wait $SERVER_PID
+sleep 5
 
 current_num_pages=`get_shm_pages`
 if [ $current_num_pages -ne $prev_num_pages ]; then
@@ -272,9 +304,10 @@ if [ $current_num_pages -ne $prev_num_pages ]; then
     echo -e "\n***\n*** Test Failed. Shared memory pages where not cleaned properly.
 Shared memory pages before starting triton equals to $prev_num_pages
 and shared memory pages after starting triton equals to $current_num_pages \n***"
-    RET=1
+    exit 1
 fi
 
+set +e
 grep "name 'undefined_variable' is not defined" $SERVER_LOG
 
 if [ $? -ne 0 ]; then
@@ -284,6 +317,7 @@ if [ $? -ne 0 ]; then
 fi
 set -e
 
+set +e
 # Test KIND_GPU
 rm -rf models/
 mkdir -p models/add_sub_gpu/1/
@@ -299,13 +333,15 @@ if [ "$SERVER_PID" == "0" ]; then
 fi
 
 if [ $? -ne 0 ]; then
-    cat $CLIENT_LOG
+    cat $SERVER_LOG
     echo -e "\n***\n*** KIND_GPU model test failed \n***"
     RET=1
 fi
+set -e
 
 kill $SERVER_PID
 wait $SERVER_PID
+sleep 5
 
 current_num_pages=`get_shm_pages`
 if [ $current_num_pages -ne $prev_num_pages ]; then
@@ -314,7 +350,7 @@ if [ $current_num_pages -ne $prev_num_pages ]; then
     echo -e "\n***\n*** Test Failed. Shared memory pages where not cleaned properly.
 Shared memory pages before starting triton equals to $prev_num_pages
 and shared memory pages after starting triton equals to $current_num_pages \n***"
-    RET=1
+    exit 1
 fi
 
 # Test Multi file models
@@ -325,6 +361,7 @@ cp ../python_models/identity_fp32/config.pbtxt ./models/multi_file/
 (cd models/multi_file && \
           sed -i "s/^name:.*/name: \"multi_file\"/" config.pbtxt)
 
+set +e
 prev_num_pages=`get_shm_pages`
 run_server
 if [ "$SERVER_PID" == "0" ]; then
@@ -334,22 +371,25 @@ if [ "$SERVER_PID" == "0" ]; then
 fi
 
 if [ $? -ne 0 ]; then
-    cat $CLIENT_LOG
+    cat $SERVER_LOG
     echo -e "\n***\n*** multi-file model test failed \n***"
     RET=1
 fi
 
+set +e
 kill $SERVER_PID
 wait $SERVER_PID
+sleep 5
+set -e
 
 current_num_pages=`get_shm_pages`
 if [ $current_num_pages -ne $prev_num_pages ]; then
-    cat $CLIENT_LOG
+    cat $SERVER_LOG
     ls /dev/shm
     echo -e "\n***\n*** Test Failed. Shared memory pages where not cleaned properly.
 Shared memory pages before starting triton equals to $prev_num_pages
 and shared memory pages after starting triton equals to $current_num_pages \n***"
-    RET=1
+    exit 1
 fi
 
 # Test environment variable propagation
@@ -367,9 +407,11 @@ if [ "$SERVER_PID" == "0" ]; then
     cat $SERVER_LOG
     exit 1
 fi
+set +e
 
 kill $SERVER_PID
 wait $SERVER_PID
+sleep 5
 
 current_num_pages=`get_shm_pages`
 if [ $current_num_pages -ne $prev_num_pages ]; then
@@ -378,7 +420,7 @@ if [ $current_num_pages -ne $prev_num_pages ]; then
     echo -e "\n***\n*** Test Failed. Shared memory pages where not cleaned properly.
 Shared memory pages before starting triton equals to $prev_num_pages
 and shared memory pages after starting triton equals to $current_num_pages \n***"
-    RET=1
+    exit 1
 fi
 
 rm -fr ./models
@@ -405,14 +447,84 @@ $page_size."
         RET=1
     fi
 done
+set +e
 
 kill $SERVER_PID
 wait $SERVER_PID
+sleep 5
+
+rm -fr ./models
+rm -rf *.tar.gz
+apt update && apt install software-properties-common rapidjson-dev -y
+wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \
+	gpg --dearmor - |  \
+	tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \
+	apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' && \
+	apt-get update && \
+	apt-get install -y --no-install-recommends \
+	cmake-data=3.18.4-0kitware1ubuntu20.04.1 cmake=3.18.4-0kitware1ubuntu20.04.1
+install_conda
+
+# Create a model with python 3.9 version
+create_conda_env "3.9" "python-3-9"
+conda install numpy=1.20.1 -y
+create_python_backend_stub
+conda-pack -o python3.9.tar.gz
+path_to_conda_pack=`pwd`/python3.9.tar.gz
+mkdir -p models/python_3_9/1/
+cp ../python_models/python_version/config.pbtxt ./models/python_3_9
+(cd models/python_3_9 && \
+          sed -i "s/^name:.*/name: \"python_3_9\"/" config.pbtxt && \
+          echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt)
+cp ../python_models/python_version/model.py ./models/python_3_9/1/
+cp python_backend/builddir/triton_python_backend_stub ./models/python_3_9
+conda deactivate
+
+# Create a model with python 3.6 version
+create_conda_env "3.6" "python-3-6"
+conda install numpy=1.18.1 -y
+conda-pack -o python3.6.tar.gz
+path_to_conda_pack=`pwd`/python3.6.tar.gz
+create_python_backend_stub
+mkdir -p models/python_3_6/1/
+cp ../python_models/python_version/config.pbtxt ./models/python_3_6
+(cd models/python_3_6 && \
+          sed -i "s/^name:.*/name: \"python_3_6\"/" config.pbtxt && \
+          echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}" >> config.pbtxt)
+cp ../python_models/python_version/model.py ./models/python_3_6/1/
+cp python_backend/builddir/triton_python_backend_stub ./models/python_3_6
+
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    cat $SERVER_LOG
+    exit 1
+fi
+set +e
+
+kill $SERVER_PID
+wait $SERVER_PID
+sleep 5
+
+grep "Python version is 3.6 and NumPy version is 1.18.1" $SERVER_LOG
+if [ $? -ne 0 ]; then
+    cat $SERVER_LOG
+    echo -e "\n***\n*** Python 3.6 and NumPy 1.18.1 was not found in Triton logs. \n***"
+    RET=1
+fi
+
+grep "Python version is 3.9 and NumPy version is 1.20.1" $SERVER_LOG
+if [ $? -ne 0 ]; then
+    cat $SERVER_LOG
+    echo -e "\n***\n*** Python 3.9 and NumPy 1.20.1 was not found in Triton logs. \n***"
+    RET=1
+fi
+set -e
 
 if [ $RET -eq 0 ]; then
   echo -e "\n***\n*** Test Passed\n***"
 else
-    cat $CLIENT_LOG
+    cat $SERVER_LOG
     echo -e "\n***\n*** Test FAILED\n***"
 fi
 

qa/python_models/python_version/config.pbtxt

@@ -0,0 +1,45 @@
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+name: "python_version"
+backend: "python"
+
+input [
+  {
+    name: "INPUT"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+  }
+]
+output [
+  {
+    name: "OUTPUT"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+  }
+]
+
+instance_group [{ kind: KIND_CPU }]

qa/python_models/python_version/model.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import numpy as np
+import sys
+
+sys.path.append('../../')
+import triton_python_backend_utils as pb_utils
+
+
+class TritonPythonModel:
+
+    def initialize(self, args):
+        self.model_config = args['model_config']
+        print(f'Python version is {sys.version_info.major}.{sys.version_info.minor} and NumPy version is {np.version.version}')
+
+    def execute(self, requests):
+        """ This function is called on inference request.
+        """
+        responses = []
+        for request in requests:
+            input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0")
+            out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy())
+            responses.append(pb_utils.InferenceResponse([out_tensor]))
+        return responses