From acdaac104db5d3b647b1220be8ff60d1477a4965 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 5 Mar 2024 13:43:34 -0800 Subject: [PATCH 01/62] updating with torch compile option for baseline --- .../rayserve/tritonserver_deployment.py | 23 +++++++++++++++++-- Triton_Inference_Server_Python_API/run.sh | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/Triton_Inference_Server_Python_API/examples/rayserve/tritonserver_deployment.py b/Triton_Inference_Server_Python_API/examples/rayserve/tritonserver_deployment.py index 1610b583..ec5f8a56 100644 --- a/Triton_Inference_Server_Python_API/examples/rayserve/tritonserver_deployment.py +++ b/Triton_Inference_Server_Python_API/examples/rayserve/tritonserver_deployment.py @@ -31,6 +31,7 @@ import numpy import requests import torch +import torch_tensorrt import tritonserver from fastapi import FastAPI from PIL import Image @@ -54,7 +55,9 @@ def _print_heading(message): @serve.deployment(ray_actor_options={"num_gpus": 1}) @serve.ingress(app) class BaseDeployment: - def __init__(self): + def __init__(self, use_torch_compile=True): + import torch_tensorrt + self._image_size = 512 self._model_id = "runwayml/stable-diffusion-v1-5" from diffusers import StableDiffusionPipeline @@ -63,6 +66,19 @@ def __init__(self): self._model_id, revision="fp16", torch_dtype=torch.float16 ) self._pipeline = self._pipeline.to("cuda") + if use_torch_compile: + backend = "torch_tensorrt" + print("compiling") + print(torch._dynamo.list_backends()) + self._pipeline.unet = torch.compile( + self._pipeline.unet, + backend=backend, + options={ + "truncate_long_and_double": True, + "precision": torch.float16, + }, + dynamic=False, + ) @app.get("/generate") def generate(self, prompt: str, filename: Optional[str] = None) -> None: @@ -153,7 +169,10 @@ def tritonserver_deployment(_args): def base_deployment(_args): - return BaseDeployment.bind() + if "use_torch_compile" in _args: + return BaseDeployment.bind(use_torch_compile=True) + else: + return BaseDeployment.bind(use_torch_compile=False) if __name__ == "__main__": diff --git a/Triton_Inference_Server_Python_API/run.sh b/Triton_Inference_Server_Python_API/run.sh index c465e7f5..040e1610 100755 --- a/Triton_Inference_Server_Python_API/run.sh +++ b/Triton_Inference_Server_Python_API/run.sh @@ -137,7 +137,7 @@ fi $RUN_PREFIX mkdir -p backend/diffusion -$RUN_PREFIX docker run --gpus all -it --rm --network host --shm-size=10G --ulimit memlock=-1 --ulimit stack=67108864 -eHF_TOKEN -eGITHUB_TOKEN -eAWS_DEFAULT_REGION -eAWS_ACCESS_KEY_ID -eAWS_SECRET_ACCESS_KEY -eS3_BUCKET_URL -v ${SOURCE_DIR}:/workspace -v${SOURCE_DIR}/.cache/huggingface:/root/.cache/huggingface -w /workspace -v${SOURCE_DIR}/../Popular_Models_Guide/StableDiffusion/backend/diffusion:/opt/tritonserver/backends/diffusion $IMAGE +$RUN_PREFIX docker run --gpus all -it --rm --network host --shm-size=10G --ulimit memlock=-1 --ulimit stack=67108864 -eHF_TOKEN -eGITHUB_TOKEN -eAWS_DEFAULT_REGION -eAWS_ACCESS_KEY_ID -eAWS_SECRET_ACCESS_KEY -eS3_BUCKET_URL -v ${SOURCE_DIR}:/workspace -v${SOURCE_DIR}/.cache/huggingface:/root/.cache/huggingface -w /workspace -v${SOURCE_DIR}/../Popular_Models_Guide/StableDiffusion/backend/diffusion:/opt/tritonserver/backends/diffusion -v/tmp:/tmp $IMAGE { set +x; } 2>/dev/null From 452c70f76727a2a8bff204bda941b904df95deda Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 5 Mar 2024 14:07:43 -0800 Subject: [PATCH 02/62] updated to build model on initialization --- .../examples/rayserve/tritonserver_deployment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Triton_Inference_Server_Python_API/examples/rayserve/tritonserver_deployment.py b/Triton_Inference_Server_Python_API/examples/rayserve/tritonserver_deployment.py index ec5f8a56..86d8b77a 100644 --- a/Triton_Inference_Server_Python_API/examples/rayserve/tritonserver_deployment.py +++ b/Triton_Inference_Server_Python_API/examples/rayserve/tritonserver_deployment.py @@ -79,6 +79,7 @@ def __init__(self, use_torch_compile=True): }, dynamic=False, ) + self.generate("temp") @app.get("/generate") def generate(self, prompt: str, filename: Optional[str] = None) -> None: From e5189236e743c95e3266c5846f65cf6edeff64e4 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 5 Mar 2024 16:38:48 -0800 Subject: [PATCH 03/62] moving torch tensorrt import --- .../examples/rayserve/tritonserver_deployment.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Triton_Inference_Server_Python_API/examples/rayserve/tritonserver_deployment.py b/Triton_Inference_Server_Python_API/examples/rayserve/tritonserver_deployment.py index 86d8b77a..77bcdb32 100644 --- a/Triton_Inference_Server_Python_API/examples/rayserve/tritonserver_deployment.py +++ b/Triton_Inference_Server_Python_API/examples/rayserve/tritonserver_deployment.py @@ -31,7 +31,6 @@ import numpy import requests import torch -import torch_tensorrt import tritonserver from fastapi import FastAPI from PIL import Image @@ -56,8 +55,6 @@ def _print_heading(message): @serve.ingress(app) class BaseDeployment: def __init__(self, use_torch_compile=True): - import torch_tensorrt - self._image_size = 512 self._model_id = "runwayml/stable-diffusion-v1-5" from diffusers import StableDiffusionPipeline @@ -67,6 +64,8 @@ def __init__(self, use_torch_compile=True): ) self._pipeline = self._pipeline.to("cuda") if use_torch_compile: + import torch_tensorrt + backend = "torch_tensorrt" print("compiling") print(torch._dynamo.list_backends()) From 90a2266e0a2f2747aa02c7684236406a4e1f1765 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sat, 27 Apr 2024 05:53:36 -0700 Subject: [PATCH 04/62] updating dependencies --- Triton_Inference_Server_Python_API/build.sh | 15 +++++++++++---- .../docker/Dockerfile | 12 ++++++++++++ Triton_Inference_Server_Python_API/run.sh | 7 ++++++- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/Triton_Inference_Server_Python_API/build.sh b/Triton_Inference_Server_Python_API/build.sh index cdfa725d..6e660de0 100755 --- a/Triton_Inference_Server_Python_API/build.sh +++ b/Triton_Inference_Server_Python_API/build.sh @@ -30,7 +30,7 @@ RUN_PREFIX= BUILD_MODELS= # Frameworks -declare -A FRAMEWORKS=(["DIFFUSION"]=1 ["TRT_LLM"]=2 ["IDENTITY"]=3) +declare -A FRAMEWORKS=(["DIFFUSION"]=1 ["TRT_LLM"]=2 ["IDENTITY"]=3 ["VLLM"]=4) DEFAULT_FRAMEWORK=IDENTITY SOURCE_DIR=$(dirname "$(readlink -f "$0")") @@ -39,9 +39,11 @@ DOCKERFILE=${SOURCE_DIR}/docker/Dockerfile # Base Images BASE_IMAGE=nvcr.io/nvidia/tritonserver -BASE_IMAGE_TAG_IDENTITY=24.01-py3 -BASE_IMAGE_TAG_DIFFUSION=24.01-py3 -BASE_IMAGE_TAG_TRT_LLM=24.01-trtllm-python-py3 +BASE_IMAGE_TAG_IDENTITY=24.03-py3 +BASE_IMAGE_TAG_DIFFUSION=24.03-py3 +BASE_IMAGE_TAG_TRT_LLM=24.03-trtllm-python-py3 +BASE_IMAGE_TAG_VLLM=24.03-vllm-python-py3 + get_options() { while :; do @@ -148,6 +150,11 @@ get_options() { TAG+="-diffusion" fi + if [[ $FRAMEWORK == "VLLM" ]]; then + TAG+="-vllm" + fi + + fi } diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index da0611d6..8ce507f6 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -49,6 +49,18 @@ RUN pip3 show tritonserver 1>/dev/null || \ pip3 install /tmp/tritonserver-2.41.0.dev0-py3-none-any.whl[all] ;\ fi +ARG GIT_REF="0.0.6" +RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${GIT_REF} + +RUN pip install "git+https://github.com/triton-inference-server/client.git@r24.04#subdirectory=src/c++/perf_analyzer/genai-perf" + +# Install TRT LLM building dependencies +RUN pip install \ + "psutil" \ + "pynvml>=11.5.0" \ + "torch==2.1.2" \ + "tensorrt_llm==0.8.0" --extra-index-url https://pypi.nvidia.com/ + RUN ln -sf /bin/bash /bin/sh COPY . /workspace diff --git a/Triton_Inference_Server_Python_API/run.sh b/Triton_Inference_Server_Python_API/run.sh index 040e1610..8f4c59b8 100755 --- a/Triton_Inference_Server_Python_API/run.sh +++ b/Triton_Inference_Server_Python_API/run.sh @@ -29,7 +29,7 @@ TAG= RUN_PREFIX= # Frameworks -declare -A FRAMEWORKS=(["DIFFUSION"]=1 ["TRT_LLM"]=2 ["IDENTITY"]=3) +declare -A FRAMEWORKS=(["DIFFUSION"]=1 ["TRT_LLM"]=2 ["IDENTITY"]=3 ["VLLM"]=4) DEFAULT_FRAMEWORK=IDENTITY SOURCE_DIR=$(dirname "$(readlink -f "$0")") @@ -38,6 +38,7 @@ SOURCE_DIR=$(dirname "$(readlink -f "$0")") IMAGE= IMAGE_TAG_DIFFUSERS=diffusion IMAGE_TAG_TRT_LLM=trt-llm +IMAGE_TAG_VLLM=vllm get_options() { while :; do @@ -110,6 +111,10 @@ get_options() { IMAGE+="-diffusion" fi + if [[ $FRAMEWORK == "VLLM" ]]; then + IMAGE+="-vllm" + fi + fi } From c25db8f7249557b7496b2b96d65154dd12cbe4a4 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Mon, 29 Apr 2024 13:13:37 -0700 Subject: [PATCH 05/62] updates for meetup tutorials --- .../StableDiffusion/docker/Dockerfile | 2 +- Triton_Inference_Server_Python_API/build.sh | 21 +++---------- .../deps/requirements_trt_llm.txt | 31 +++++++++++++++++++ .../docker/Dockerfile | 19 +++--------- Triton_Inference_Server_Python_API/run.sh | 4 +-- 5 files changed, 44 insertions(+), 33 deletions(-) create mode 100644 Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt diff --git a/Popular_Models_Guide/StableDiffusion/docker/Dockerfile b/Popular_Models_Guide/StableDiffusion/docker/Dockerfile index 6f7c050e..bb5c3ad4 100644 --- a/Popular_Models_Guide/StableDiffusion/docker/Dockerfile +++ b/Popular_Models_Guide/StableDiffusion/docker/Dockerfile @@ -29,7 +29,7 @@ ARG BASE_IMAGE_TAG=24.01-py3 FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} as tritonserver-stable-diffusion -RUN pip install --pre --upgrade --extra-index-url https://pypi.nvidia.com tensorrt +RUN pip install --pre --upgrade --extra-index-url https://pypi.nvidia.com tensorrt==9.3.0.post12.dev1 RUN git clone https://github.com/NVIDIA/TensorRT.git -b release/9.2 --single-branch /tmp/TensorRT diff --git a/Triton_Inference_Server_Python_API/build.sh b/Triton_Inference_Server_Python_API/build.sh index 6e660de0..36ff36f9 100755 --- a/Triton_Inference_Server_Python_API/build.sh +++ b/Triton_Inference_Server_Python_API/build.sh @@ -40,7 +40,7 @@ DOCKERFILE=${SOURCE_DIR}/docker/Dockerfile # Base Images BASE_IMAGE=nvcr.io/nvidia/tritonserver BASE_IMAGE_TAG_IDENTITY=24.03-py3 -BASE_IMAGE_TAG_DIFFUSION=24.03-py3 +BASE_IMAGE_TAG_DIFFUSION=24.01-py3 BASE_IMAGE_TAG_TRT_LLM=24.03-trtllm-python-py3 BASE_IMAGE_TAG_VLLM=24.03-vllm-python-py3 @@ -140,14 +140,14 @@ get_options() { fi if [ -z "$TAG" ]; then - TAG="triton-python-api:r24.01" + TAG="triton-python-api:r24.03" if [[ $FRAMEWORK == "TRT_LLM" ]]; then TAG+="-trt-llm" fi if [[ $FRAMEWORK == "DIFFUSION" ]]; then - TAG+="-diffusion" + TAG="triton-python-api:r24.01-diffusion" fi if [[ $FRAMEWORK == "VLLM" ]]; then @@ -215,7 +215,7 @@ if [[ $FRAMEWORK == DIFFUSION ]]; then set -x fi $RUN_PREFIX mkdir -p backend/diffusion - $RUN_PREFIX $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/build.sh --framework diffusion --tag tritonserver:r24.01-diffusion + $RUN_PREFIX $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/build.sh --framework diffusion --tag $BASE_IMAGE:$BASE_IMAGE_TAG $NO_CACHE $RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/backend/diffusion/model.py backend/diffusion/model.py $RUN_PREFIX mkdir -p diffusion-models/stable_diffusion_1_5/1 $RUN_PREFIX cp $SOURCE_DIR/../Popular_Models_Guide/StableDiffusion/diffusion-models/stable_diffusion_1_5/config.pbtxt diffusion-models/stable_diffusion_1_5/config.pbtxt @@ -238,17 +238,6 @@ $RUN_PREFIX docker build -f $DOCKERFILE $BUILD_OPTIONS $BUILD_ARGS -t $TAG $SOUR { set +x; } 2>/dev/null -if [[ $FRAMEWORK == TRT_LLM ]]; then - if [ -z "$RUN_PREFIX" ]; then - set -x - fi - - $RUN_PREFIX docker build -f $SOURCE_DIR/docker/Dockerfile.trt-llm-engine-builder $BUILD_OPTIONS $BUILD_ARGS -t trt-llm-engine-builder $SOURCE_DIR $NO_CACHE - - { set +x; } 2>/dev/null - -fi; - if [[ $FRAMEWORK == IDENTITY ]] || [[ $BUILD_MODELS == TRUE ]]; then if [[ $FRAMEWORK == DIFFUSION ]]; then @@ -256,7 +245,7 @@ if [[ $FRAMEWORK == IDENTITY ]] || [[ $BUILD_MODELS == TRUE ]]; then set -x fi - $RUN_PREFIX docker run --rm -it -v $PWD:/workspace $TAG /bin/bash -c "/workspace/scripts/stable_diffusion/build_models.sh --model stable_diffusion_1_5" + $RUN_PREFIX docker run --gpus all --rm -it -v $PWD:/workspace $TAG /bin/bash -c "/workspace/scripts/stable_diffusion/build_models.sh --model stable_diffusion_1_5" { set +x; } 2>/dev/null fi diff --git a/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt b/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt new file mode 100644 index 00000000..c86a4946 --- /dev/null +++ b/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt @@ -0,0 +1,31 @@ +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--extra-index-url https://pypi.nvidia.com/ +psutil +pynvml>=11.5.0 +tensorrt_llm==0.8.0 +torch==2.1.2 diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index 8ce507f6..969166b8 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -25,7 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver -ARG BASE_IMAGE_TAG=24.01-py3 +ARG BASE_IMAGE_TAG=24.03-py3 +ARG FRAMEWORK=DIFFUSERS FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} as triton-python-api @@ -33,33 +34,23 @@ RUN apt-get update; apt-get install -y gdb COPY ./deps/requirements.txt /tmp/requirements.txt +COPY ./deps/requirements_trt_llm.txt /tmp/requirements_trt_llm.txt + RUN pip install --timeout=2000 -r /tmp/requirements.txt # Finish pyright install RUN pyright --help -COPY ./deps/tritonserver-2.41.0.dev0-py3-none-any.whl /tmp/tritonserver-2.41.0.dev0-py3-none-any.whl - RUN find /opt/tritonserver/python -maxdepth 1 -type f -name \ "tritonserver-*.whl" | xargs -I {} pip3 install --force-reinstall --upgrade {}[all] -RUN pip3 show tritonserver 1>/dev/null || \ - if [ $? != 0 ]; then \ - pip3 install /tmp/tritonserver-2.41.0.dev0-py3-none-any.whl[all] ;\ - fi - ARG GIT_REF="0.0.6" RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${GIT_REF} RUN pip install "git+https://github.com/triton-inference-server/client.git@r24.04#subdirectory=src/c++/perf_analyzer/genai-perf" -# Install TRT LLM building dependencies -RUN pip install \ - "psutil" \ - "pynvml>=11.5.0" \ - "torch==2.1.2" \ - "tensorrt_llm==0.8.0" --extra-index-url https://pypi.nvidia.com/ +RUN if [[ "$FRAMEWORK" == "TRT_LLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_trt_llm.txt ; fi RUN ln -sf /bin/bash /bin/sh diff --git a/Triton_Inference_Server_Python_API/run.sh b/Triton_Inference_Server_Python_API/run.sh index 8f4c59b8..c25eee02 100755 --- a/Triton_Inference_Server_Python_API/run.sh +++ b/Triton_Inference_Server_Python_API/run.sh @@ -101,14 +101,14 @@ get_options() { fi if [ -z "$IMAGE" ]; then - IMAGE="triton-python-api:r24.01" + IMAGE="triton-python-api:r24.03" if [[ $FRAMEWORK == "TRT_LLM" ]]; then IMAGE+="-trt-llm" fi if [[ $FRAMEWORK == "DIFFUSION" ]]; then - IMAGE+="-diffusion" + IMAGE="triton-python-api:r24.01-diffusion" fi if [[ $FRAMEWORK == "VLLM" ]]; then From 3885dbf6c05a37589f687706ed18fb6047b0006b Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Mon, 29 Apr 2024 15:58:34 -0700 Subject: [PATCH 06/62] updated with tutorial repl --- Triton_Inference_Server_Python_API/docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index 969166b8..4bdedb82 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -50,6 +50,8 @@ RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${ RUN pip install "git+https://github.com/triton-inference-server/client.git@r24.04#subdirectory=src/c++/perf_analyzer/genai-perf" +RUN pip install bpython + RUN if [[ "$FRAMEWORK" == "TRT_LLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_trt_llm.txt ; fi RUN ln -sf /bin/bash /bin/sh From 4bec62ba71f7b1963b8af715a9d9d8383f4d51ef Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Mon, 29 Apr 2024 16:16:14 -0700 Subject: [PATCH 07/62] updating to volume mount local directory --- Triton_Inference_Server_Python_API/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Triton_Inference_Server_Python_API/run.sh b/Triton_Inference_Server_Python_API/run.sh index c25eee02..0aeb8e83 100755 --- a/Triton_Inference_Server_Python_API/run.sh +++ b/Triton_Inference_Server_Python_API/run.sh @@ -142,7 +142,7 @@ fi $RUN_PREFIX mkdir -p backend/diffusion -$RUN_PREFIX docker run --gpus all -it --rm --network host --shm-size=10G --ulimit memlock=-1 --ulimit stack=67108864 -eHF_TOKEN -eGITHUB_TOKEN -eAWS_DEFAULT_REGION -eAWS_ACCESS_KEY_ID -eAWS_SECRET_ACCESS_KEY -eS3_BUCKET_URL -v ${SOURCE_DIR}:/workspace -v${SOURCE_DIR}/.cache/huggingface:/root/.cache/huggingface -w /workspace -v${SOURCE_DIR}/../Popular_Models_Guide/StableDiffusion/backend/diffusion:/opt/tritonserver/backends/diffusion -v/tmp:/tmp $IMAGE +$RUN_PREFIX docker run --gpus all -it --rm --network host --shm-size=10G --ulimit memlock=-1 --ulimit stack=67108864 -eHF_TOKEN -eGITHUB_TOKEN -eAWS_DEFAULT_REGION -eAWS_ACCESS_KEY_ID -eAWS_SECRET_ACCESS_KEY -eS3_BUCKET_URL -v ${SOURCE_DIR}:/workspace -v${SOURCE_DIR}/.cache/huggingface:/root/.cache/huggingface -w /workspace -v${SOURCE_DIR}/backend/diffusion:/opt/tritonserver/backends/diffusion -v/tmp:/tmp $IMAGE { set +x; } 2>/dev/null From 9b5ce1f8566b4f498776dd36d86caeef0ce38c89 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Mon, 29 Apr 2024 17:06:06 -0700 Subject: [PATCH 08/62] update to install 24.03 version of genai-perf --- Triton_Inference_Server_Python_API/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index 4bdedb82..801abf41 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -48,7 +48,7 @@ RUN find /opt/tritonserver/python -maxdepth 1 -type f -name \ ARG GIT_REF="0.0.6" RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${GIT_REF} -RUN pip install "git+https://github.com/triton-inference-server/client.git@r24.04#subdirectory=src/c++/perf_analyzer/genai-perf" +RUN pip install "git+https://github.com/triton-inference-server/client.git@r24.03#subdirectory=src/c++/perf_analyzer/genai-perf" RUN pip install bpython From 25692bdf1fe3ca5545c3adc3f3ec87c72a63ee30 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Mon, 29 Apr 2024 19:22:02 -0700 Subject: [PATCH 09/62] updates to make versions configurable --- Triton_Inference_Server_Python_API/build.sh | 2 +- .../docker/Dockerfile | 15 +++++++++++---- Triton_Inference_Server_Python_API/run.sh | 4 ++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Triton_Inference_Server_Python_API/build.sh b/Triton_Inference_Server_Python_API/build.sh index 36ff36f9..6cbef3be 100755 --- a/Triton_Inference_Server_Python_API/build.sh +++ b/Triton_Inference_Server_Python_API/build.sh @@ -63,7 +63,7 @@ get_options() { --build-models) BUILD_MODELS=TRUE ;; - --base) + --base-image) if [ "$2" ]; then BASE_IMAGE=$2 shift diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index 801abf41..9e608c6f 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -26,7 +26,9 @@ ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver ARG BASE_IMAGE_TAG=24.03-py3 -ARG FRAMEWORK=DIFFUSERS +ARG FRAMEWORK=DIFFUSION +ARG GENAI_PERF_TAG=r24.03 +ARG TRITON_CLI_TAG="0.0.6" FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} as triton-python-api @@ -45,13 +47,18 @@ RUN pyright --help RUN find /opt/tritonserver/python -maxdepth 1 -type f -name \ "tritonserver-*.whl" | xargs -I {} pip3 install --force-reinstall --upgrade {}[all] -ARG GIT_REF="0.0.6" -RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${GIT_REF} +ARG TRITON_CLI_TAG="0.0.6" -RUN pip install "git+https://github.com/triton-inference-server/client.git@r24.03#subdirectory=src/c++/perf_analyzer/genai-perf" +RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${TRITON_CLI_TAG} + +ARG GENAI_PERF_TAG=r24.03 + +RUN pip install "git+https://github.com/triton-inference-server/client.git@${GENAI_PERF_TAG}#subdirectory=src/c++/perf_analyzer/genai-perf" RUN pip install bpython +ARG FRAMEWORK=DIFFUSION + RUN if [[ "$FRAMEWORK" == "TRT_LLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_trt_llm.txt ; fi RUN ln -sf /bin/bash /bin/sh diff --git a/Triton_Inference_Server_Python_API/run.sh b/Triton_Inference_Server_Python_API/run.sh index 0aeb8e83..a705b21a 100755 --- a/Triton_Inference_Server_Python_API/run.sh +++ b/Triton_Inference_Server_Python_API/run.sh @@ -34,7 +34,7 @@ DEFAULT_FRAMEWORK=IDENTITY SOURCE_DIR=$(dirname "$(readlink -f "$0")") -# Base Images +# Images IMAGE= IMAGE_TAG_DIFFUSERS=diffusion IMAGE_TAG_TRT_LLM=trt-llm @@ -57,7 +57,7 @@ get_options() { ;; --image) if [ "$2" ]; then - BASE_IMAGE=$2 + IMAGE=$2 shift else error 'ERROR: "--base" requires an argument.' From 49b526e60eb782207d35747d741c3f762f2012b2 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Mon, 29 Apr 2024 21:27:25 -0700 Subject: [PATCH 10/62] update with repl --- Triton_Inference_Server_Python_API/docker/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index 9e608c6f..1ef5fb58 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -57,6 +57,10 @@ RUN pip install "git+https://github.com/triton-inference-server/client.git@${GEN RUN pip install bpython +RUN pip install ipython + +RUN pip install ptpython + ARG FRAMEWORK=DIFFUSION RUN if [[ "$FRAMEWORK" == "TRT_LLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_trt_llm.txt ; fi From 53e12b974bcfe28a7e459a277d05dc9b456c3462 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sat, 4 May 2024 12:47:18 -0700 Subject: [PATCH 11/62] updates with partial openai api support --- Triton_Inference_Server_Python_API/build.sh | 6 +- .../deps/requirements_trt_llm.txt | 1 + .../deps/requirements_vllm.txt | 27 + .../tritonserver-2.46.0.dev0-py3-none-any.whl | Bin 0 -> 258264 bytes .../docker/Dockerfile | 12 +- .../examples/fastapi/openapi_modified.yaml | 2326 +++++++++++++++++ .../examples/fastapi/simple/main.py | 146 ++ .../examples/fastapi/simple/models.py | 830 ++++++ Triton_Inference_Server_Python_API/run.sh | 2 +- 9 files changed, 3344 insertions(+), 6 deletions(-) create mode 100644 Triton_Inference_Server_Python_API/deps/requirements_vllm.txt create mode 100644 Triton_Inference_Server_Python_API/deps/tritonserver-2.46.0.dev0-py3-none-any.whl create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openapi_modified.yaml create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/simple/main.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/simple/models.py diff --git a/Triton_Inference_Server_Python_API/build.sh b/Triton_Inference_Server_Python_API/build.sh index 6cbef3be..05ea4a1e 100755 --- a/Triton_Inference_Server_Python_API/build.sh +++ b/Triton_Inference_Server_Python_API/build.sh @@ -39,10 +39,10 @@ DOCKERFILE=${SOURCE_DIR}/docker/Dockerfile # Base Images BASE_IMAGE=nvcr.io/nvidia/tritonserver -BASE_IMAGE_TAG_IDENTITY=24.03-py3 +BASE_IMAGE_TAG_IDENTITY=24.04-py3 BASE_IMAGE_TAG_DIFFUSION=24.01-py3 -BASE_IMAGE_TAG_TRT_LLM=24.03-trtllm-python-py3 -BASE_IMAGE_TAG_VLLM=24.03-vllm-python-py3 +BASE_IMAGE_TAG_TRT_LLM=24.04-trtllm-python-py3 +BASE_IMAGE_TAG_VLLM=24.04-vllm-python-py3 get_options() { diff --git a/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt b/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt index c86a4946..5c07da65 100644 --- a/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt +++ b/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt @@ -25,6 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --extra-index-url https://pypi.nvidia.com/ +litellm[all] psutil pynvml>=11.5.0 tensorrt_llm==0.8.0 diff --git a/Triton_Inference_Server_Python_API/deps/requirements_vllm.txt b/Triton_Inference_Server_Python_API/deps/requirements_vllm.txt new file mode 100644 index 00000000..ca4149ae --- /dev/null +++ b/Triton_Inference_Server_Python_API/deps/requirements_vllm.txt @@ -0,0 +1,27 @@ +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +litellm[all] diff --git a/Triton_Inference_Server_Python_API/deps/tritonserver-2.46.0.dev0-py3-none-any.whl b/Triton_Inference_Server_Python_API/deps/tritonserver-2.46.0.dev0-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..953ac2ce9ef81cd0683822d1648b9c7d350cd605 GIT binary patch literal 258264 zcmbrlQ;=q1vo-jZZQIpl+qP}nwr$(CjV{}+E_T_rtN#8bV&a?=GZ7OpXUE=ou`lwu z%FMM^u17%{1QZnj06+pLl1ntM*Sq%MKmdSMFaUt=-(MFeOBZ`PXHzFPQzu#`IuhSjJQq0Bn;5V>~H>3|QHWz+I5SvnAH z(Kvle1Mxl4vd-w);5dBdj~HJy7Ucdu$xgStle5kNf%Ae!!x#9U&cD0L5|I{8gSppm z|JaqG^b?Q9(rUhILSrJ7&%(KBeM$jW{1v?=2m1TYZ1M$;=E8_!=;hCW;0C+B>BC3l z6+aajMz=QMTKdjOWeM>iCP$NIK`uHPphdDZE4Y)Y-AQtb_N! zH2#=Vb9`FNV$sReO~!GsQxW;yQ*w$o6D~Z~tbW+b*n*@wiCT82moat{fgoaqjPE1n zXNjVYRZ`SwQw4ZCK)Hr3=psi%SH+g{+*NLBRe=^x)rcB)GG>}Y`gw;fiF(6H)#f&x z_O3V6cV8PyS-cKy9FtWBOR)t_S#ycleuna$B9xAbyzRFR;;EqIXWpN%$1v5EKA#EQ z8MGGj(3)7}1d2VU38ExgYCprO7l7(-hK1QH6+)`hYNtgwdQf=C;2~J$51wNnnVG&e zib1keo2k{?=aV1tBtuBi>Q^=?G59WBD-s!Nj@nWh<@u~?Z$*c%P^wC0t=eNwOxWmQ zBw)oo?A9r@Xw<9tP0e1;7Y%Hy*6goFnEl~lbE9QVvSfy9aD>mFY)oEt3 z5hGCSNVH>LtXIT>2|N7C;|r-8Pu*J&YLZDgi*KZ6WG?gwn5(!e$+ih6TqG6KvsXLO z2NnrrYNMl`h^G3Hj2`qqypoA%+16Xpmkk zjp+Y-`O)j^TiRK==F6xix$^=>NmiaM}wo{BtVO5nRuJlZY&x)Urj4-K})h#=zQ|l&C zIB#G{R*@FFtCG{2R!S*eS(aAEV$BCv-AJfE*Z*Aste@)Cs*qzE2^1x@q^^jqv`$-U zpLwSmEAB*M^CL5e780m*mdT`d8WDjYqHm7BGp9ZN`Q?VSSRE{wIXlq7%*s;<25TzQ zxcuU?JQ~IrA*EX_1K(c&ixyyxQGbx}Q+1p9mRy5XDxMmWmw88Mb8><@h1jk{7C^$;d}HBGx2IpQ2b9Ps0XN z#THA+&^NH08hFo3OO_6gyFfaHt1_VqY|Pe`wmb~^ls+Vtq7TwEXo-SWto{V@M?hQ6 zq>bsIDyU@Ns=6a@vaBh_5*J9q3ym~IB>8bP)_2egUEE=aVCfV+obPw$_UOQos+9VI z%jFo9@6R>)?&sMrry<7gZQ#+LIhyC>kfMyBi*=R&9)K~s37s5 zD4UO9#=9d=zGJB#VGnx*{G|&KPn)9{Q1cL^3aWD&&~#%%BhSb?JN!Cxho_BiJ*15M*m$#r-e zEy|1p?r-FVJ6dikBB+h7#=?iXjeZ2Ylc8_~N;?p^77a6SK+cx;X@@;1^jEe3G+|OXneI zDoENki*iwgu?ZBJ6_pA_?X(N27A8g1*x_K8MA0+2!F5MCsVWcixh<`5cj>X2%*0EX zY)o_ECc=Td)%wUC=~V{o`jez?R^8#*S?1ou>ExV9_QdTp_2CdcZ%@_l{k}VIf(p3} z=U+SMpR(yXMCoHB)FSX-GdX~IPK&NgyP>_#jpne${Lb+~8j#1q?Q>;pMwHMn(PaeU zS-Ow>$@YVFOx%lfq-9=_nI$KED4X~(Uj@-K|BlPC=37&ndf+Ddn9!qIk1DD2R5K!! zjELK_JCrp&Xm1-g!Fn@9Jm$_KS-!>`HmZWyJr!wf?hT_z)GZ$v0{iOwn5&$Ssni*b zH5v=puEhw^hAR*{DMqJ>Fu#(nBBRSZ)=hs^Ev^J4pyPwoy4a%pQTgjY)CSFuSvHO` zSB`$FW#`iBj^HA6pO-KTj&r3(kxHV}%#TC9ZGVIm;%+YB<8-+xP&Zsxfq?_FV2*P9 zG!*E3S5@|VZNti{C8oTi-eVM3=ITa2v?0CSdBtu`S}%u)8QIm1opQJf<(k60pd|jN zrgH5IY8BL60)4+|F*zT;~Tgn@)+BOj{r4p=!ZXv&|kJ<$g{S@*l*HEb^f6IZk zT447ew`aX6nk{v>qG^QEIH1D;&HnK~$Tjux(M$+`$9?^mdUj;@u2~J^_ce5hxc4uW z7F&Yn;|8B{gP1xF)mOJ9RT_kRgJZJ$&H#vbImjDO%W%Sy?3HRGk3yX7$k>IUZo7rL z9m*^=Zb-K3^x-CuW^EOvL+>A?e9loh2y`DgUlh{aOzo(mWPYMgfk!=5D)-H|Kf)ez z{wSv=gN%bvd$J41dy7!dZT3+!Q`er$6MUs#fZcc{{uJK5gKvl+vvym2eBM3ftOJ@u zWAe?ZNBVaw2;I5%K(ZV@SeigOr! zWDE82<66&0aC`P(1yrlnUo2kJ>)7^sTk(82ebC-SuQn^LAA3d%2K4BVv*CUP{2H^a zh3Q2M`E>m+sX zq;v6fFf{=I{8#EoTWXE}r2M~A3;+L1t#9aH`QIn?e?P*eN=bv(S#7@;1^~#W1OUYT zH%A!S*w`Bzy4X88|HA`4*w*&NlTY3LBjeOh8)#E@_8)~(sVm6Yq!MwCK=#QCA#3=%+_B$0mLm&=n*FH(q=07npR#HC@}rhYb(s0%j9(kf(#0z4?B z=1X7&6iTpo%;q{&*TF3jlgvzUmEZvTL8Jokl`TA(R!su^%rZe9;?BCX;0-7-iqSw^ zVi1T|{=`swR8B&Q==g3dw80zN1pP=@HYX++DJ=29B^I)?JfSY4WK=V0EM^uRx!~MU z$drQ=Gg^qpp&c|G6oh=`nMYs@{7K?bg&P!K)gGNAkijRRge$TaGa0DdTI3cB24DH>*yO*81KrVaCX>y-qd?LpjOS z5Nj-g*(<@u+go0Tt9sM2V`P1wMje2+CHY~E*}WZ*1je66Cz(HFZNfF(D}%s3gr}I& zb40iJDPJ^^9`iEDojXI;$NZ>@9$*}M>9PFjy> z6E|5Kn8zm_c&yv>FyRq3YX@HIBo(qdk7m)sT+L@)fr$ayMt+Rx_>T6mBSpozzI`ls z2?Q3?m!&5fgcWcbdYOb(j;#!w?MEO9uG5*}Qms2?222T#g-GtEvox=RF-AdBVGjj` zx&c+z-@b4L4YJ3hxsD*%a&p<=tWKBf8;my!4h`Ymy-vJS+TP@yjaQt^oz`6XHJNAM zu{1X6nzmP2*tfT^vDKg={T2Ygyt?%tqvPhZyfcmqG_iBkEpLQXAWk@eJZ5fTK=Ggf z#ik%Vh+zwubBNfrz3iIfeABcMc-A1;)C><2EJqL#uYkQPne>C1;daJUX5|Li7UlyzI<$VEFlmPx8vC?M^XBC3y9En= z{D}(+Zc>sD;j#UIUA_X%BNnRBv}A?CaIqy2fxAokk+=Y6tj6fu_pt)L)Ac@| zAxglkRJ*MiL!j}7J(fB%1vUIaa8QHDR*!7a95SdWKq-|)0UHVfY|^+zJlBmmDBC0SG6L|y*@sCq&8Q6GNbwsR zNP;L$81of__tm?LQKrUi_5xsr269a^Ns#ZXEWWz}$~Ny<)^ie|yOy#JXovGmg=83) zPreTg>M5H$>71Pb&w5Q{VJQPRY)3J3$g#sZz_~!scQB_x@>W!-9i~0xgs+avhLix5 zwH$-Nu>RL^QVItq$OcQ}XFz`n5&GRti}2V8Sq6fNQ*+}PWMf6U8L&$QI2@O}1|R95 zUtsYNodnSY^g7t7=4qJNOi?u?&cwVYL!6CMl=3`V*x;JGCYiVn3uzlmB-M-RlZqR@ z_uJ!{G8y=A5vDztStbFQcLhI++R`K8T=2o+MLwV3XPKzQzI3l2VCu`23vUI9D=Dam zwPv?7uiUi0jpH=?IreN^cIGcXhQ`rQRRnXLi?GOqb_}KGk7S}E%^Caj=w<-m0@z@wq*BygE^ULuYeW@XPRaP*}0vB574|Yn9|clLAV8zG#@k50nx#WPD+ru*2JbXjHQp`9nEFwf3U% z=1({;A!LhlP+8U)`f@JIYrniv2@&Osw$n#wADO6b+&#VpZlmrBJ|lvatPSqf(!Bz|6Y3A&ANZcy?_p-od-0r#7$YEb z^{$;~PZ$6TpSM`PiLurougbFn(Z5rCy03l-pCU6LUzJP%QiWQQ_(sMdEoU~)*a|bW zIgpJWP8%>VZG+86Yq`RhnFKdjL@HC81_{M$$b3hR%MDnKcY=8!GsZOlnoGv4x>d~X7J*jSw~BZ(|3y2%>=;9$ZK;pp zj=cjudLWNB%Y9*oDl9d7z13tYWmCsBayA}vyGZ>HM(oP@$dsT%RGt#!8 z0^Ilvap+7|K6kxu(rgr@%S1>Zj{Osu@c1p#_FKq?lKn*1Z}zh3EPQ6CLqN}Rt1x1t z*E%e~iXDpiw4wgiFfyRba42E6zJM8!RsQ*ez?aAknP2p(+fssEF_Na~5Ni`qFZRCn?{pi{mFab~M4%^gdNAlqf(($bT41@b80RJ}*W3 z*>g+KD5}ibfu@!*f7ZcnyAs&BYcoM%csx{~2HEE)8F_k()^0@JR zr~4K{cbh$5?Vq0~LgceqmYQ5q6jb3oDZ?T8E+(nWlp5?@B5!40?O6{c7xYvQtrfcYwfdf}LQG9tpzC*sZ` zWxyMAYspHzPv8$+g2|QypJULNbhBdk|L;g zZ%wEuQ%Mg?EF1|C^M1CLlOuUTyoXkJNZG< zfh*T8t>m;(gTi#=n@+?!WA~2|=%Ar>#0!yXn)j&{F1?z)AMldA73l{a?qC~IhkKr% zv(Y7=@UjP2bWr&CP+T7#&u710Pntv0{%pu7&#SPjbrou4^@{k5P5RW)vWzInVo@4K ztW}|sag0o$JRc2o4{)0p@JmP{Z<&)ukQ|EfO)A9%L6$nEd1Sd*w=PNuGK(>p$t&10%HBZtG@JE1h+(H$K!i!f=Tw#|Gram5Qm+C z+Uf))_rongI~`L_s;+Xn3EH&KWy*At)mpN6OeiNvJ<`xp*`ok!(9{Ivqb$0dm1!o2 z5#?4oYXV9Ypi!b%Ee(^IK54YjAlIi^1%8UaQ}({d#G;@AVf~A$qDlsxD!uJDSX&rZ zYlY&<5-K4G9tJX#`ngIl1b7rQIpt`rh$B#4**gNxxB>P|3iS~%4eoV#n3l<@@}@$4 z#eP7zrM!%EvyHSUMi-xPLUxs7t*q*Uy+vU*EbFEUS%M)YGCNcTz(N49jIz0m>9et@ z1UIC&Ab}U?Z4bbkR}LIu4r2yoV8skrggZAikZClCOPHT2g=Pj9O(QB(ppX%;vx+n! zL<(pz@HC~?s=|P|64I4Z<*HZaW6!FL>$lI6gkqt47wZ)5nt?692A=7_Zf!`-&`x9% zpS;v6o;*23geprLgA2Cv$^@ZD0H<-sHP20?8HfuGnqiZJLrj?6k zuK-Jk^y%@wL4!%1K#h9go$kU9l={T{$Qcc;3hDU`aMB{E-=-n;Q@Fj;-4_~=Qr1~O z70wB&RG8KkWQ1%46FwrX^$I0lkP}j7hu!`_tU69%>U2YK7)i6*3)?0C)D9!jQpQRi zY@yh+ap#n#z${?Aa*e!9QL|b|#sTk5;%N}7BA3j$)CJrbER3@gw&h^0iGUHai;g_( zRoCB=%ejrbpIp{#l3&Lw~V!;CeqOe&)#uyLif$;D&4^jcogqf{%i`5Q+l0;njjaB^)}^ zM>t>ug}T-7E;^+Wj^?jn0yAvh`)hsF}+T?|3XovBPo9d z;;E;J1W~Sdbu|SNP<{R}*avT5wV>drNo0V?gG`Lg!x8HnS0F5U39%C<)Q9u}2h%(K zd(a8KGbI^&7jeVkp@CpgtFOnAc703gT(?YZfg0!9tuz(^zJ(WyOq5pn!)=s2!LesT zW0U~3j?0^KOTp2oVMUU+*)V15(n5F)I`?ZS)t3n!bb^79i0iVZF5iAN0v-XWE8BQd5OxLMW>t2dP41 zKs2L>n&$YkL=q)eXfq}3Bvue{qsRCNqFbe=1+h>BnVo_9ytg2 z7GWM3+z>)6d%3_%;DLN8X4JZUBM885ZO6;x|6-7l{fX^PtV*g$SK`k$u}ZV!tGn|tY#Saf!$?*4r@2zcO(7YJ5TCJ z`jj<`3E_%TIugUfb92aM1vvBx8X{Ikr4B3nQqUm>#(ZJl(>MZmy3bp*LWhK(E;qhz zR$k|a#&s{;b_unA+^3YcT-_I3L0EM14~O<6;d$MmJvsOA&Iv8NKm7x4x-B}-2zH3^ zq1CkMYK>^}K0>C>*kH$RO?;S;LHLFj8YYZF{KU*5K(-x~mqUFf?F_u|6#CH51zmT* z7Ic@OBltKso3b12>$QFS{<3tsZmynjQgG7RAvPCpOn!b}WuYI$;dHbEe~txA2cTc? zNSSJDuA zsGM4xl+%NcMn8nTkPDh0xA!1Kf7bJCH3%ua4C0f(iE8KF`sA6v(!JjE5zfCRzF~|k z-gW=Z%(O%3DC*2FU0lO>x5PgcTKU*Q;nE^v=Wowb9ONwI$^E&r1Yfc z<;5l77wdPejODX2WZL58I2pPTmDH(Qj8YX^l;X)rTqg`4G8W_Tso9QX@DAhq5@)&= zTu)%^nuCU3(1JDB;db#c{WmE6 z0hO9Sb1%ZJ!M1thHEXsmLRhZ3x$+l<#$ubv+tx z5?)``Qqtju+@FW{?{epma+!KZPhvWXz*M=f@yaH1o>3q>>%IGCUPO24@0ilr9zAl5 zXUSTsF;aFt{49%%61i4z59@h3ms5Smm?8o3%0$#ZF#m7~^KLNz{+-{_A+jhw&7Ddti0>+(6JuDg6D<~%8TBxO| zEJ_)sec$F5R-)Iw8|}FON^c$a{5~EibP*bLY?2KgE06Jczgs$qzde}>n7)aLEKF>) zq?M6&hIN+v7HBzJ&cMnl-V`nhB|_54DT!%*%zPV~l{^X2m`3nO1Fzy$1XST}O#jzO&CUCp1C%7~s04h{8n7VoGXoK{3I4kW3n# z3q;^=`js_60B$tM;0IMCzW)z_6QV#6jkg)DgME?`xLxj(>S2I;HX?*@&h;n91Qk*w z*X^5cDXkU23Kem{@UQ`b0+JDj3sa=Z9cnERPalP_Z8W%S*g=+8`YEOX1Yfu^AAK4^ zCDBR=1m169qFA&=tU&_94Y^5hGaA_Q3F-;a5`|k`D3i^cFrYYg;x0q;MQJQi`%xky zMB^PSVAJVL2Pwh#w3>`Ecn+lGH1~$7hG_we*&N0qMxbve!Kn#;O5@mWWK=wQ=JXjL zRJD-u8R42>IY4IGQE#Xd)B8GHz(P4VL*#9ouGg&6zizz^Gk3%Ih}01pDudahA*MQd z9miuivrp4x{oX{I1MeSGEFPdfe*i|0{xme(ZIPBp2X2ha0R*608(WMn-U2ai9&I_8 z4;Mh4T3jY}Tqb5)*zgIQ$AG7R7+*0Xxrti4So>YY?jS1{{E*qoy=lcw)MZPPLd`fv zy;&BR5SlbVZ!OBBnbF!orrFkcG7%7p0PVufW2f*XjxF2f-R$rD3-mPvrqScAMp%Yr zQ2Tlp2G#DYGo{T>pkb}|HNo;Ked7RFn;f`^p7}BGPsD=L-6S(!SVW8js|$a9U@V*D zNVjU;;P9TI`2z7YPCc!6T5NcawJGXN#+=gPWb36I$Z^~4F6Pm$FAOGT+rW**RfvM2 z1@J{OMBoA-AO(F|Tj755X}s;GG??jSzK|FEc_409gFMI_qQC`1z+?-h#k^rep2g7}s?+#mV1>C%)mChQ$7as+(2_ zBf8K7Ou7b{tObe9KuBuFGLh^pEQ{a7;W7tDc7w4`k6uiyU8JwwPQN z9aj=(LrGp8Yk!+!trya@p~Dl{LDwaO=u(M<=X~Qpc4XljZu3M%=8H`le1z*m%A3py`&7oGhz67*b~wN9foN~|6WTprvB zQa!sh_@mJ>85)ech$JTj-`Go9ZFe_+)$b<38QzrHV4lH_yL+9=Bw+X>)bU}?_Y0fHzlwinn4wH3X>KGdv%!KVIF){6QWxK^7^&v7Qlm4}3&C*DS`wJ{AnCKy zBP4kj{pnjnUm|VgB}(dqih8Sb#0eIHTa{|t;DoIIAhtQCiM*}z;0+` zn4=Hq)bSxOk-E6+7aS5h=MJ?7H9Rs%ixF3eIl1k-XAw>uY&E-jx?MH={M93kPJC|o=w9~t;x_r6$CFT+=*AE{Z#J@9 zDd0JsG_nKbaQ-TGF5S9!^`YG&mul)2_ex+dgz#$Y93ebKeO@#JAQ+~4MLM2+f-t3c z0wQ-!q&Qf!8kej?no_bF`*D|JG2-P|o>9X2!}LhfZ^{X~+tPXOp;?!Qj3L3{BI>jbIl{a8 zw+5}Xk_t;&jRDZ%bqbK4jmaqvKyaGN$;Xf01c&ii4W2S8e)<9RI2qkcjG|?CQ)6&n zta9#868qFb?2)Aj zIa4%&Fn`sOUx@j;@@8mxBvs=~2WNM%XYq;mZb<8N=e&R{M@d>2Za{XaCCKPxSM=h@Pp;P@}FYnX0Oey~$4=8N@{=!?+m(u(RXzsO7dXXit4yqs&(R zHjUbgvidY8@143KP{jLvkdtJWdKu@18fI#7W<|?8Dwhh0fn?NyCZ}Cf0}&e4@kMv( z;93{;;{}e;+^m1J3g+YHh~M|4_VB+Dw)Q~ZCwlpo&ek|pT%g;Fv3caU+cseO_pm%Si?Xjv`BF7fl^X%w05&8A_+icI9FV&s10?!{RWa$y17`yo%X# z?iL}PiGi@C`JyW3fn-X@vHp0VF9Wu5JU3b^3wEREyEg(uFeDI||4X$Bpj>E9A$1$< zg-9Wcc*Ev#DtnlX~zzwOW4cGz;&i3Akp$>Q!_-mBi%-+#O&v!fA%AYGEC zFM(RI=t=QZiF0gjdY>A+H`$;bbq5&Us>tIvWgvn>^ZTui3*z}64#I*aVOlvwL|Obk z3~Rk{eaer+x(7S23cZ{5RMX5sSM^NtZgbrXMCVs zhC?)C(fXr-P28XM~kNr6Ucxzd7bH( z%Ki2BNLKk#pf``6755L=e)_C`b3xD?CXs@l;1I3b#f+ za&$TE(CQ(w(?q{TDULqdZO!k@;kJx%Dz>8uGN!t zrQ6??0br@BYJVYXN#sN2E$O}9-pbfi8k{GmYwD;Hdex9MmkIExyD!iRpwJfS0?YWaA6ayF%W=^Y57uwhGC8+-kaX)%3T zcXCrly(n%fS(Dr5rIUKwBGbDRmg_n4zFZ-jR{`)lNZC1SQs2DR#$q6%XjApA$_5os zrW%%Huy{ouG>xJk$T)g+FtwKLOml6Zv1a1msn}Yo%NpC(r?$MV!loAd5CG{vCB4Zd zZ6`bkX;eB<2t0eyBvcffbt-kY{UPVrlE;*|IG$h#Ht}XhIYZ`77bxtj5tU|&0&N&$ zD#Mr`AewQQW6F0&8Wl|`Z|9qZ%1LG^$EbgcYv_5t$x0xW03uF!zRH9JpeFCe%`sfkB zOtZwM8`P%w{qAmEL!2lUY5*gR>t`}TI_0T%7-P5Oa!xas(Q$w@T@_Q z?4JONm=OjLk&lYu{_sd*CWk{;;%w7rQ5v#6`k6+C)O=An(s_b~bFk%M2J)Kl&ZncO#j*F? z(Qi|5S7zhuUNdZtDtFUC<_{}QX7Z`QKHFvC$HI5}d%*&=dVAI_dT<#|D{|T6c;!m( z3EXz5SMZocb>ktd9gZH^E3OCReb*2L_5q@$6%g)^cf?wB5@*E#Z1Y1=pWn@<>4N^mp{isK!)!GoDP+ zc$12toM`4-&n7{+4Rfa5_J`VLLyOejy%MbZH;(I&e^f?y5w`cuEqFte-4Rdf1I3;yEz{lR(Wh{qapd# zGzw%p-&|O~AKuIRW*c{;99`oY2ug!}E%Ae0D|H`NvQY=1OfLJ@pV-Ft-OrI3-ALX% zNnXI~Hu&Aut&%V(Ka>#U{EHdFxgPuI4s=qCZ|whP?+YV{Hf}%!0BHX;KEeNu8ewa1 zVruiR^IhRsJ8!bxcl&{f<>r;Z4<+NClvXp5NV^PH(pI9bB|HvV}hCllW;E|=f^(qd#VE<){pJ@3(NJc=+mI5o=nPPZQUjT>DM~@5BdzT~cYr<|h^#V9 zl_rT?JfhnQWS~j5Eelrz%MjGkig`nKnvy~7Kq5BoWH~eC=WX1Cr3vmnOlJ+U#W4Q! zJ4u4LF?Tmiv)kL=yLqv()o^xpVb20Jtq>1QV1D-jm`3@{(-`xUD@CUt)-$1A1?I8k zY?d;thjU_P(T$z&#Yld^WXt@!M8Z5Li_?Eu8Fm4Pr6SJEOL88^?Fvc7^Z9Y04f*?Zy zZewYoYe9igz{T#T2f&m`zPt5#1$SHP)n@4+vF6g!80BPwzt5nTJ8rY2w+a58DK{o_ z=sfg3f{XRwpU`}kBWcUX&Z04dZ{;H`02=b|+Kq2hMz)O4@h}^WHKVZfRe;w{pi96U z5n+60&Om_U5Mf1Ss22Dj1xy)a>~C-D{+Q4tQ)HYXietjgMB3niC_)LG`RJac$h+fd zA7dFY@LBo9S#D=L7XkZYTp*d=$L>9_OuDsEMcsZZb74m9*viz5aubg45s_ms4voPr zRJS1!b}wqsdUos?GiTHe$)G~WAW)CKi&9MPjLpqmdPL%B2U%lmA%`sE<@gECkU2u2 zEwNzE{DT_zGrF=k;m;YiDCGz$#5x`PG@NyfCInvo$=>fQ5yR&N5vZMTeS#%gup`N0|msV8Z7I?^9*ZC zKS`iF5D2AaNsPaxe-W>23Wfd2P7dS+J4Iv}sdnLwTzV~kP!m`1Q1)d~sPt}(uAF9g z1_jox7F(`Q@i4OL_%%JoK&*I=vCn|V>688;749JwytqFi(c+DIgfuJ?dNN)YS7^|9 z8cJ?efIa9QK$V$oF~ZE9^+-zs{DyRijZC^m0^y`?cg{rd9_iJMDosBCHiSr?dm3v) z@HrG!H{kj;FOL^q4d;)HUBh~Y)3l7yO*zV#{sN@}I)XTDEOk|EGakTmCaFS*4)8ec zyaBpF79dxYfGNo6f+`{iJ5Tskzy8NB4m6w55JaK*wNH`mQjD*1k~=VoP{S}1iMH`n znz?b#dL&O-a6pJ9*mi(TNCO&4-49kIxCeeMMaL0Y-O9EUBWHRG6BL|WIl(vSCnZFd z$@1jjl_cX(#zRG&MKFU3)q@!hS8^#lgrl zF)bl-2GD02tr*tVAhFj@B5gxq#{!-z*e0+M0eG-(W3f~dY}+XhEnS0)N7c?wzLM;& zoOwHF>jAU?FIe>I2mrc~8M6Xhh^&bkQ^f{|obwTEN76E7;dtP@1(FF5ri5jh*18z$ z{SX2{fm)y3y9tt=?#<9Y!%>gPwH`igrysg)b4x9kVz>!THS_W;%eA23T~Df8om(RhSO3D~t|q5$PtaZn8ps zO((GaN&%lnin7IOccm9`+c9*;G2zsdQH?Ftzdl9!K1bb1s<7UDxA{@c;sofX0T#iaD_AIwiU>er$62E7y2t4@FPa$j~9gG*}0}m4R1#TX;;W(Mug_uq@izfOxDHheP_#-sqTreUIDEAE5vwa`JY89( z%s@g2yj6-eG(IV`Z{Ncf8L;=c*Vb`Kg!)f34syXz(mva9Xq^@cHr-c+vfDBt31n%a z`Exs5MZt-u7-b-9>gD0@hjFF~b`@7+E4|dU#cQ_Urr*^{Gr>lT>SK0#-GIK-=8w_LGJxQK(*P{BS%{nV3c?&H8FF;p_!8)B}yNm=$X%Vgbu&Nern_QmL)RZ@M z_`0$7ty<+I)f5`oT7K2zJBo^~3=+E|s#rJa8iwfFA5|Uy)c?G1bNMZYlk+##9oIB% zHBO^n`l%WS+{&>J283CvMu7BNOE;dK~GTNFeH70}rc>Yz3~EM($NSo% z4vm)>0*Qfcl3(9u;-R^XG20C__&Xb7IWIrXtk2?3M!`VYPCkWpIMRK`4=2@0xfBm_@KcJV)?75bf{XBo7Bhp}|>vz&~1AWcjce+cLUrms$Lq& zrd!^=Kqk=&l};TrJ<`+_Eoqo_Vcao{%o{7$)N9ypu{(rDV_w7Fd+x$AXVz;DAtB(< z>kZ2jovO}IdYKa056=JP&XpZ$)&R~p-0m$@ewUh!H)2awDK@dwnK73uD!$>vUAEKl z=JX9O9mErdQrarLE{kxj<~g+7kVNxBznwjWcW!2DQl5h<52PbcyxsDfMdGRY+c(aX z9Q|OA(J$17ig^F(s%tohu1o2e@D1H5=zC4AbTzSK`nnT92hv#fDfjfJVqHE*NYK@a z4z+Vw+-)|E-~Yi<=~TrxSiM^JatI4Z332@WVG->;*kw{J`=|uJP&0MiYIwT*apbUk z;&B^v$0glNPS)Cv+xX%0q6LnUs^qM>4&_x^Lg6o*MJw@)NAKJre?UfygCJW}&Xar;{;F zqdw~dasXd0W*o2UE`bu4Y$X^0PVt$Mn^{a>PbMnCB9}}cR+NHtzW*jUYD|2!eC(su zz=N4vU%dwcFQW-?*3tzvUbp9I+`MYuyfK3gO26Vgru@!P3yj8dDk(lV86zWIllg)c zr`LPZbJLd$58_mF!_}D5csb2ryy^T4=W6%a(z1Xok4*DjsamT7fGqSVqL#+H6qFI* zou1XWNt2XBU)kmrAy?IuP;(FJw`l3M68K#J*?gON${`_KDHA)6TUTCCz`a6_Wu;5q7VNDa_I87 z@Dp^`d)-f=`s_-t!Cb^ChaNx_-j$*`tKR~{sy|E$;T7fDqP*?od14k2lWqhLZI30w zQG3mNv)_=5-#^un$x*}CbQmO#rxgG-;&uoz;(Kt!fBN&U;0>ZBU4yoH1$i4 zcmmdCa@2gfE4e`fA=e@ntp%O45)6UP-j3$5<}bjUT;11|*#8tnDYciHIYrB0m)vHFlS^(qoA@{QJlJ6 z!?EVrvXwS}(^S0(&j*jepPAhFXx3&y`JP#>Ds#8Cg{!QJUYf4(b?F9CG#BOfR0uFV z0)@&)CCf&|7K3h&Uum+!j>wP@UZN(wQ0U>ge}_Y@4d_qMxJ`VTrw*h*C5+Z@f%Tb} zER-IL^gF{b`Jm1=Wcj!2J`7sYtfWkms^|H~r?z&z|UWwy!6(%zztv)_uhU{bhsU z^Fj0A>;H*J|0f0OFAf-$LID6!5d1HtU?)>YS5s$~|C`RRw%?jS{jSqzbO@-^ap;un zngB$1x7_Hn#hi&cF5`qsa|$}2 z|2BowgVke1ryp4~Gb*ea#bRjKWTZNnDWQ?5<~eel44xSBo>sf$ql|IYOo6YNHSNDt z1u{XV8`&Bm6vUvB;hc8wLp+sKuaQW^0Mm$Az4<*Hx^H{lvshT2%5a7NZC*JcB z`ghF9Hte6=8IJi9lDm6Q3TutlUgzw2++6+cV$_ZI`L^&N@UJ~NlO=oiipKE5k6w4} zX>-GL>c!|f057nX^&WSPIg=S%J{`&2&9eT!Xx`j8UpCiz7(A1YO#+Vb8Drx&SZC|* z6RLfQvvRg8m5Cn5Pi%>fVBQ#d{4wXj*{TyjljhK8&fH{n;Bi=ath0rjEC?E5`U38K z!hAcGCFi|5Yj4&D{T;}dH}A6+SmdnXZ6~aH>0?E0*{nU#^|DT0yjjht1BP1;_K5bf zJ{al}1)EIsx=PRwu+r?W9RyRqG1ApWXSjdoxt1V$#nUKJ&fpk`6tkvK-0f*I{ateSSSsRJDrD`*DR5m$HVT*eUbn%?rCsmbIR=Hc)7F1 z+&Ft@9`bbT{z`dd_C8`jvBc$ z9k%{T96NY$a&odG4jO1Hky^n~g?)s{taDxg=J8j26gmaRa&i2DV826FI$mr*j&5=K z7-#?(ux6xSL){eBso=l}ne-zPEm$wL3a?i|KI#?;Qe|IgMiljrszw;%d5E53zbzi7 z4XsCyQphn|@+K7)^~TUFrX=K{R;p4Jjku~Em?0U9V*_DnJi(gU#PBjN~@h^7#pVy2urj^dO}tQnAS#3y?EZAO@JM3i4KNHcly-4-GEZou zhUPo@#>q9$uN;6$y}av15mFGO&+eJN{x64CGH=~f4ZgS%;zCxdJAYiogC~OmW7a7y zq`3`8fugdS=FIITI+#^ykNNNU#CUAc1z30CFa&N699XbF3ngztf6Z;if&i3=rYuXT z4pM^$DQT&1#?@M7Wk$}Aa~j`Q^wQsw4UA)+lF}+F2r)H!?Qp~}=p+fInI)PT#^FZd3RhuX5@iaod``5H28`ukUs^uZ&P~ROPJzc5q9KO#NNZx= zu?wZ-VhtxR9ICigRis!xzHBahQW}AVtUpNT=uid=XB77Sc0UxCCB4L^?LI>ODd-|f zDTJM{b8^Ily~{QH;5o86&$0w+04vw0aE5*C1*=>&2T8MwA?!tIkpSSscA*?a*~g19 z<|H+T2$0-GFjj>6>>lV3tjDaKh={Vxt?+#S*rlFx#^>*O4}TbBZ%R0t4l@W3(E>P= zZ|^g8#zZ=K?Usd<4K#hgDn{(2t#=f0`pGyhptWMyiKdoR?_H@t|_lae(V!&mbZA7oqyBrGutv#8hwZCWYjVinI zX|yXO|C-4bABkFQA-qDCr-yi%Nsxx=FmOqT1rvfpZe_CL`h!(yd&CZP@;hL~r22@J zgdc3#aBc5BrR5qU_U)YSm55Qze}@k*LW>74hKNp|(dct=eMnRg?%t(sHC%`H7q`d(^Ma-%2L3gE&93yZ@ml9H`oY%4_5x%(zFr`s z-j#}%`WWYtU92*Qw;*e@dAS6xqnnuggmZg=#D)*APIY2*1v3>H*Yu#~XR-()D+9v) z7o{R8ug8FAwO?CoBQOvV!pBM3zupMKI*eo2@^%#XbCuPjhkV5^jMT_|xI&$yYdM-I zt+aIdp07zmq&TK6`t9BWZE^hc|zJNZ+YG<&(b-+yct<+U!mUwVUVbxP4(zbg;R zH#~nC#U}O#zh;L~yZ$KHEfr_QT(Vi0Vk@%VdBYUUj%DG}Oi60Omse%S$31lpjFha&q;*(#19V z&goiE-K4nL8gp^=bnjLPw~RM0^vPA*DV>bMgnY@tt`91Xv&|($V#LE0Q4I8s%Nwv9 zq>63~9Igbsz=|a#4;qN3RA-pGAOnp@&rJ#Mb{1>fmBv5(ffoPO9>Ft5TYi}ho9OUJMXoaNH7gM-e1Zy?H`HO(4V;2PVc@z#3P<0V(QKuiW!CgdoA@?}6Z`$mgnx>)R4^_ZuS=E$OltGz6q|h1IKxtA zD|nKrMlgyTfu`jrEM67shZ{oL1u-9en0J@BxQOO!B8L1*Xol$@vmo>s>El0{0lPA-MppV`nhpbec6%V8v0V7c?HeF2mPd+ z@Ix!4pFmY7s`z(STAoX6nyCo6mX+Cx`6X+eFOyx*73Q8Q{=o`xSMeUgAHLv4H4h@h zus{D#MESp8Fh~cDhzTqJKnoiHfXM&G6m&9mcK8qS_n$AQW$U~(hWIn5?f6}{yQWn?fgj=MW=2ap%!-?f zAt|(86Li#4@D2?7yb!$rZM4KdH^sA~W|~tvA}_3jnleI3*+d0T@!1rs%+H@dG3_2I zhIjaoXTtfJtxC$UVQPpo8W>N*a&Y?p5kfaD^@sRT)=4B^Fv{IYCKegS zBtaPlExEPk_7eYxC{lq&F(E`y6NSV=>5G8k`LT5c7#?Dq08VJBHU_v7KtLw}bd@08 zO{Of13b6`<#J{tkgO3KBmi&K-B4MZzFPbETI-+wFGeK73ON;fr&*lyyV8fgn@1(QF z{FGYB%<_c#nvjJXQDZh(aOFR7MIyiqQq5^0o`iJJbWjoVo@Sna(fAW-L@ok}F@IP> z>QNMdhOU~DknD&D^dXdhzBrO0#5z@@XsGZJfq~RY)|UVjL79z3wBtOaIGdt^sDWhu zI!_bMea!e;uagCyN_)dvWt1NsL4nEsIeD0B)64HZ^6BMawGIEfg9WGH_l(M@R(`jO z+Q9hhYMC7;yvo){N0*K^10L4a=tR4;5Yehd!(P66(JCg#MyFY;+tflwgFf-M^`9#k z|0B}GM~q(9%Mq@A2vW4rYh<0Lc>|@+9efPqWVv3~x=sU&ouOkl@D}$EDWW{BNdtR1 z#Iph!0Wgj97~AO->1;=nj(fdouU-cG031K+b-alN(FJBJFN3hcp_%2=x`Nu!IvW}- z)hZ9ofH~2z9N}z9N8@22#wbWS+_(6LZeWG6e+vK;CRzD#$O9HjE{_e?%A_70d#gua za1d|qWW+uD%JpM1J?xTeif26oi}_8(M4X5hcpAUXEA7euMpqvDQjUaXz=V+%x|3ei)kJLxWh$4UlW$_!BK`cj{wS+o{VEV>uf*zp_ zDS^HRrX02s!~F~DJ~gCJ)Pv1}7}_oSTZtD7LSP=E3G zxqu_pfMToB^(_R)f=73Fwk(7+iz_z6A#KVMPlmwEQ_Z?Y`M<3X;9E|N8nzJN_4ggJ zgY=@6v1R!!HA65@I*Z^9N%{#N@8lGPcP*eY=o@zVf0FV@f3^egaOx?(Ek+E4VL%Ld zw}6`88qjCOY^VS>_b@_Zt=hrg?2_4t4A2jkbJlPC=rmY?o<&p38;Ti0LAkQt0=16$ zpf`4AGu}E}58*PJJ%YdK07PKIiYc_3c#g2|0(pR74~Gsh)0f12wIf=Juqyx@iq);NMCY!zi^V`Ga#h-K?xh4V2$t0GI3lc zT2uZq;k8>qmbVk>esC^p5J)Zfcx`;C)F)+1$cud3BSu8i>`gac$cN!STA~9T5o0K< zu9pO?9|c6(3yN6y54qUs=jsT7LJ(a5-w}!5mpW8guFLf8M%XGuG4dZhvxXVdES3$G z!nw~0@CWG(6;{uM#FEyRg#2^G8#r0C6d;)42+brv=n7=kh141NODyo8riy}MFXC!bk6txm{gIVY$HFtl`{`r?=%_3UCgHEaHsQZ z7ea}=)H92icZgl~gd6kCE%2Wz{>iQGa^AVG&r$LX0Xb61nQ)p(w2FPZ+=!DlZblcA~_MUnV*qJc7!y;k#{A_ z8Alo=k$Y9uoY{>p{;sy?d11>dU9t23^Ud=LpL}_#&d#m{p^BCJc&nK7eq3!|hOgD% ze$j?)w+U4=gt*z=k^?|L29s2aPUg-3=BKoD`3JU0Wle|ThpjKuh>#(|ug{+O^f6us zZ+}0fkJ=g@o}+2+nhv3L2;P2*mz>X=>;#nGcJ>*f;jxhq5mM@8ZF^$^k>l5H?WR%o|J=P7*^|Jv^5%07l zg)zC#fis$MxV`y4fpB^Qzi$mp5;bUngJ&IY2F|3difE7*woiOV+_^L>(9D;ar_|nf zoYTD|2b;0;<)JX;91C2mv4=fSqRM)~vx14tx$IQI2s`N$LP#WJx!4vA#%H}NneF>o zss@x80JEVaRC7$QfCIou_Qy2;Xq=N z@hFS?bv}~z7CmJN_6vD0CaKbhvWpewI41Y1o|P386w?H1CkgvM;U34lMjdy=T#3jq z1jQQ^sK(-ecQk}v%v!)L);Q?(?7igl%*dJZ{C*k*`I^+m8*6~nLZi^) zIIF8^7SUl5_TC69Mz6-&gOT`IPY>X0N<11 z#7oBD*yG-T^y8TNam;=ThcWv?!n{~#pu_eb9tGZ9cR9g(&M%3*K2Ez__h_l?RX#Oc5l-C zpBtas_Sw{AMbFp!b?3hF)>3Y{@n{K`4VZh9?Mp>bU#8DsDRI)*@!qNgc7;XGjB?FApc({-yBF4BFMf-MUs1FxEqo`VFAL&MH0?~TB~m`c3! z2NT>edOQRyj2f4z2BOLF?pXc8KJJSUw>OOr zqt?{9VM}VF=$_1lyd5dntpi%IVUz3iNlUTFvsr!sh51Fa#Td!>9VMIAa+4r6>igm* z3AaB$(uSiII#7=4n8<$z+u(ji{{IuxXUz1~3O0p%!YsSGcbpaBU{ir#hgQ(^OeH@>N<-!>Nnz4L=G+`bv%4b83 z>F?WNz~}F`g)*hymI*&wvw(t=7~U6|0)xsnA3gS)?iluHUAxORi;U*#4kNcQFU3*l zg;tDf0-xM57ngeEfj z>|ue7xBH}f3X3RL)+?-;xcL7TP09b^UW#S}$MW$306G)@mqpY6fwccO!|__%V~@A) zK2w81If#O>7;{@HoP`aRQcjT(MRxtOG7vzlMCnRvCw1N2Ej4AocfIi*|K=V~OGTlP zZfg9^<#axo$$ZS)XD>>o$zMGCAw;j&@B8@y{W=W~>v(H*U$l;v^LN<5ZxQeP34Oy~ z;N!i{ecmj-X`tME>@3jBlo+Ow;SXIp8Dt0FjS%FBZQ_Z&QS(e0A&MwBN)0875>2aF zl$4OaDdLdGAEA*%*A(Uvdk>aS`e4^Q1~5sKDb{3xL=X`Mv7|^d?235c$%!(}@`)@ zZ9wD^1!IvVqZZ9D2i&T)J19hQ$AubA0KY_$VIFqWq6D#)7m~9filNBvg9{#w!1&Oh zn3F-)0;QP8NEN_0EVBp$=8xn~*aRm-f@sJST2AN$Nzg3upui9t@LLu@+JFz7IjCJG zl{WJregrtFMx-5vd_;AIN!VUGA-K)rW6Er?&Xz>8?l zH!w{%bMi1;@cObX(Q}!f>jEz9=)Hv}G+j)*(8m$@>kq)N-LC`JHC%WGFLL|!Py_xA zxL4rIc;OljyU=}>qnrCgx5GA)U7gr*7C)kS+ru7YR_q5n4v29aSL9DM*Bv~^{kG`` z-0k=jIL31wOXtv{j8R8Rw-u~d@9esY2aqsLP4Lq|?Z8IfYj|3-&3&840$Knt9CcH_ z9i52LXPuvmYPRO@zkmzy15muwj{)lyV1aM;Sq5FvwwvSX*%A#8;P&S_#zE<%9xegq zAA>St-?V)xm_LF-H$%z$t}xIAuwwY;S6`6XgH~?y^tzp4c++cfF9z&;Zl%CGE&`Z0 zOx-@Lc<0P&8P)#t8=2_6f%~m>?Huk`J?A-Gq_x_x9nd&t6E+VC0I*09*y8)dYMuDu znNDnbaPI<%vlVXo6R!f;qpk1}asd+qgB&L00^T7NkYOGK7Ap}#2d&Eg8}4T5T!rBmlLvCCdR-&P~v>FnE}=q!Ay7=$r4$N zlcT_v3vXWSoeFl`7?CzjG7A_4!3Gfe4*1rvy_t3(S)tv0^&? zLs599$bs)T3;=o|)pnCdhr@OtU%r7s8iy-G`pYRxaLgh;*mM~u*a+&h>XXy&o;8={ z{S!mv6J=~k;`u;+kSD8c$s4I_n?-gp7hoAqVFFM**5MW=fP0)*D^v1~O;&AMzn~X_ zNzrSaqvDn&fq6vJ$scMHa+x+@Rlg5vNAnF5Vsh{7_Pa1Q{EyW`>$iNcD2u;P%+;+v ztZMS?&rhqu3RSkf{@K#F7c053kFT?X!=1kt!%CB9 zl6kFS7t|&A)5r{k3^-wS`8W#^gzPiG(F2s25M-{9bHh&ouM8Xr$9^2yVNh0pBWi&w z-L+k4lMM%qup0*SwRTvcI%{z~{Zn{f5~>HyfKo~QTW3O8;ZW4z zT}&M4q6GVO1!s!+KyTSTc|&0Iw!_Z?%4cG(lr0J*Jc||>HP%+pAn#Ey?6fHN5e;M+ z`EsO1$=W544vE3)(!(FC*+N1p{6#{8kDyLk@hw}?FvA;ck80BQh@``g8a$i_MPh;i zlAXE{J_BwXTHuVs#*D#{Z8{lbk4^)u+NQlcHy4AC^Lx|cvhIio;}xD9(w0QU4jTP8G)ab>X`0-q71=Ls6C zH%ypL13EAAk&6jfC5YQVME_e^ds`(r4a7uHu(yXve^fewiL#-3@2Z=djx7zc4vQ3wSWK!#Yk29KZIuhqAi2l6sgKcRGz@ zw>@UY%uJjJfQd{KDJnl16NqEY4a@K8HR7vEzV@;q!;3IHlMGnRjC<(^rh_wgN+)$@ zj#x6x3-Ri(i?v2Cq+6yesadV!k83nF&zaYetW!aNFAEr4-w9G7Zm$+#vUt;ShI2j zw*pBu4QZ@r$l#Ia$7=ZtT+#0`3bVOrviB-iZ3YcOC5wy_ zsk76I5@|}N#XY7e6xy`IiMO=apR9Aq{+QX%ZPc^cpeyEx6ZA!}WZV-{%Q$E7Uixj! z;6v*7A=&fOtQcbOId5ujx@|D-d*&&rupZL`0<*(&ypLcYi9%j&sUY;v@C&USnFD{v z0o&N^L%F-s$!$SRg9uU>4X9|7b|3EowopBUp~1&W6wYWeE&R0_muL6O@rTUU^MN|4 z{*%gX075A(vxUbJ2c>;$8tj4)L>!IrN|~XFZ2SjP1ioK7?cKfY_Z|0n0`B>Yyo)|` z4F_j|IX9dODgQdQ8Bau0UXf=`P1?mfu#kMxs51zb6Eba3mnHsz+_WG&^K0G5QYL^R z4y`(rjUm&N1jVr6s?@01qXOZQVI&TWq7j~7{2c@54PpB#DM_eU7yj#q@aRiesCS}2 zO0X};1>$63H&Y#Aj&H&Z(1f4Nes`tFGC~g*jAzYAy;4xtzhR~9(U&AstpFHwx(tsB zTgD$nbnVqF)zL|))%n_8>ThU<=4Lc0;zI9Hx2LO&)}57$wP!@12h@!W`y6L|DOF@L^DbWGC7_J{w^TjesQrkS=*OOQiA{_a!i(75&_KfaZ1U$lO;#~z zK@$jQ^DhvN6EB|Ef&|x*+!l4gJBr&RA4NXtYk%0eZc->( zlOm6_TN!~%kN6ZY@)Mohr;(hZR{7EY5>g@KamwHXz2nMdIz6oMvkA3~bZ{UyxO$Ro3YaGS zd}opsg=sHHa=anW_9SeAZ%-V+Si^wh_%>lSZ#eN?m>{YyAk0IoAi~JI+?z4`MXM#4 z8vyba|F{YR!?u_KY5&+d`&Erof}l8>4<(Yn#r}vKGX!MlTf`M)X4^ZJad}ReV$4xj zlV3W(Exv1zA{dd-{!;XG<$+Y-%NH7Q46hmu~T|xUoZ{niY1#+6cA#r(= z=IcOmpn0#k!pqSl@U>rv-=UC9bHawW1{wQf%5xe+e>h~m zC?$$bQZp?eqRxaEN~$2jl{GRXhI)o$w#``7sJT!sKs?<`G9H3w=O<;L%$b5hb#xg= z6{JEV(bmqEkBnl>0d1et>=fORm0$`<;WQO+)s7;kEQ!tzWgk3(KF7WH)ZuEH!$1LS zL_Uf)LkrkiUc{Tf8<$xNRbo3%)Nf&YbSuSuZCO%WSz-*u1-}g^l~kINN3zv?-)qDp&Vuw=ZaN0%k4i5EjHFr#7FMTS#%v;37=LyvIatc7Z@&H1 z;e00g)=CBm0CUCiw$GNtO*eJolaAXIp* zk$~J!>v-v7@XK-AsA)H@ONjaw*mksC^x9wUk-yw&rNk~~1HC$3wc3ZlAyzTgW00l3 zciUXau~(H*Y11U-PSt-a*J3j7^gnkrEYdSplkXT>ewO3OkQ?85VJ+u$?kLwpzK$JB zvz@fsXk)bIX>ZMsz(fALp){vXKi4!2L9d_^Pg#JW(^6o0&OVV9@C0Nk%c_+6hB3?A zzkuVjfk6%gjO00`vI$MFO;d-rZ>`$6yQ7hC?$V-Byk$7^0!>G^dB|ez@zZEbWY0Ke zjlbIbBl>z5f2Q#qQ~5ZtTWxX0ovp#I%Ut9FTmyKF?4Qqi0~oBaWRN>S(}~#jm!|hG z#H16eR6vdU<(|e)1niL4m14)Hc%_N;O3_J6yTF2TexmB&;S@_1XIPTI*%Vq8XPTm` zb~O6)+QsS>xh8V)?aFnjqy^W%EMWBzS-lSQ&w;%qr>xU>Huw+T>nPr9wik*oJvi-m zixE6(E>ov-R6O-l-8f@%)xZ8l0Xfy*rl86r$1S!DnNmDcT%Co6_r$nL+pW!Vt+iHl zbAYI+^}?w8?L3}~sr#!ht7ICEn^US@M)f*G3b0FXg_5dJ#o6#&`;?&)4QI7;gL>e1 zaM?NB?h|2`==aI%gUoDpfIINnS=??l5vSoE0oK}qp0a9h;COtrJ(Ie}mKPWxBq08D9g@Q;VbdH!l(TB|A?qi3GaS~n0l0h|+m zFUY1-xRBeU{R%?8mH1J6@eTm-@u95)i8|;Wf6!f~7;ED#!FJz={j=%xU@Db5-aR{? zH&yn(Sk35cRv1DF7UwqbqOxNe?wokZTvie^aiq;GA;|{&qmLl510itA29&Tf&onK; zSC5g7f308EHo6bhWI5xx#s<$M2RD;E+tNfj?^c<<81U}jKaODwA2g+N;`fmg-p9&D zd^3s%{)_~Wxso%PflBV_2ETU`=sp-V3?U(PaA*BE=Fsh*N_NaVtKAJ0-^F|BXXEOP z;{S{JpUeXNg(m37S@k^RKDHD14Jw6x3|h^iV>_oJcbXG~wkE_PY0wfu>vX9_uZLQq zitQcNX)!TfT`p=Nf~}gfbYfD28Ud)+7&Mwp5ve3S$+1Ry!i5ycn|&og2~WiIjd|lE zS(1TVH4swpK(43_iv{92m|$m>;N8+`SKcNlTjWuT5(Ohx%_=4A`+gzjfPMXQLIuWu zR%~ZTRB#g(7U*4t)6KMG5G!g2~nmjsaSjcx)&7gnJjr3@BoQ zO()cKH_n$Gn=3Wq!1M*E?>%Hjx(kT{Gmt^YVT{OsqtP8pW5ZOwGC-M$zP8Ca$g<%G z4$OZQRN<-tTa}Pm4-T+B!B99QyRO(S%Jlc(BuI6LQ2|49BFoqpw2zuFq81MyfIdWT zAexwYK7Ko3LR-seco7t>y@;)>^GB=4ep0ZBeW|6Ob=)|xWryyXwtdJMJR8V&JwW>!d`r@^5*I$wlxNgIaWP#?*uwJ5>G%1_GZZ+L^fZZvm~TLnq_ z-|6KwBL_?-p^mmq`7~8hixQjU(pt9nB#cUST375aBUw>?q0VjnMb&7a->_gz&lhao zXM6VSmFaKy))F;GtR3;phaQ_o!o!$QCkiUq=cQk%;iN(Br4=WQdp?|5n3@i$u;>Nu z3%SDD^mzh7h1pT{_U=uZQ7Y!wGVU*$V@dn>I$Xc6JYiSJs#1lGdCZ(s8|Q z@fNU6uj_+r4H!6_6T>w89gjBAP`%`Ui=WsdtGcQXI6o>R4th2-2bPf5_oT22-2G)K zPC~yk-~(xJN`}KV(Z4Vt-+r`0`aUX}?e}Y0c-EAkrr3q8c4tcWrD-ZFzB>GWs2eyI z81w>0rNiM`WOeECS0my36RhP2(UBbAG?E6ER?EW4rK9TUSu(TU)z9Ow;AL+3ThG;*wp{THL?uoiK zxavlvp2M`@pN!lpWLjZ+AxxFy%3-R)8^Q?}G8Nmg4|$>O2NC|w7BR^( zi;PN;`B~H8aZsB8CgLoU4YGM)JldYe^9!=6>#2lzT*rc#aO8FtwD5U8heC;tc`=Q? zwRYDno==o4-7rf~JH4sp)Jd!_+wPdIJAT+tP0TnoeZy=5K03Mt_Y`}3lL3F`S$M@X zJIqTuB~vKxefKLl;Wb-@=9)w{G=Lg-m}5R|^#T*!K{=qhYJF}^f#aC?ZjmNnAZrKL zmU=NdYdw0nk5NI2OKE;a5F9UF2S5<6t(@V-*!!KXi%SN~$5iI? zxr;4duCwXOTY#^ZIF#;h$gkRsbM=n|mst~6wM0ACVK7aUL6@tA*VqSR>py6!We#3% zEc~nqt4K7&TU0kQq_VgbXd3`1bSVa?xwQAK$%i7KQ0s9!E}QUr!-=cyoM> zKJ3rdufZxF=4MN53UZd6MS4%W}U8W(K)R+)S9fbdo9tm+(|u1RW)5x zgU$S~Y~ZgE+EQ*Z2}w2N^2@Ru6&(LMOP^Rd3Xy37`}bHK3pcN&o|Wt^Fab&2$Eb+CuW91ZCOBEQg>Is zl{^)vL`f@HFL)^GRGq|`9p6XOv=w4$w<;A^T)2iwmyMEWYK(Z(i4%G0i3>>q7QaH;Y7L4l2zCAoY#Xm-(UrL`zCS9VUUb)NKu{Mw10^6#zN9_2dUBrSyNlm;sS7tj1c>tjvUW%LG)+0(VIK!*Q+s>je(^=^ zVf>+$G(1h!-NU$Um8ekJ*`B4q{3NIru60B(m=aYPwC?FNIKf3#vZS6Wu}UaX`Wu@l zh8*7TCP`9wuIe#zlQa)b&Mbq)oDiR;=p9)tNSON$hX&yRm4O5AgS;Im*~|w>+AME! ztzcF5t8+#J1Vn%cJ_@UB)ZGijXzY$?Zf;(U^1H(r_gBszhwkbN{#ESzA~XUNn)$; zEqmeSJ+|k*-5pP3p{C1LA8s7$30W)GMQ}qP(}M?}swuoFnxLG)i=I5QtmRUqP}#wL ze$d|YMcR0-!VdLUq8$?&qn&tZ|F%S|Kz*A&2m1s4|H)3$mm7^;|MsuLsQfSOe-~3b zXM3mr?0+7A=goUqF0&7`879_$$22 zc(TKV682_s;g|XvcPFF#jowUFR!w>s%vmYTA?3Wkj=MY_Zg(-V9rz`hNHc4i2;wvU zt{^#1Xri>S%aKDDb375dCc|hB%`&iV_$F;M@+bSAIe-bKmtfJoPKY4`yP=IJ@@azU zj(IgnAN037(r#Dgw>cR)EQKnOYQhbxG_rc=}|B{X`cpq};cz)Kcj zIVN-n(Tp3OzF~1CHi+OxiaDToYz6`eO^7`NsmNrFB}kwgy)!WByo^~B04`AsPRxPg zez3(K6sZgYMSurm``#nPlaoy^$f$^Y$DuU4vHy0yC>@nhhEy{Va$-LOloR(4^gaF9&Z78?l;>z^Bkfj0J9ZyDVFoa#d zC*^4g$i6=w;rnJV#uqaOwq`QT=YK6@cu3dRu;&Q)r3Z|8-aU+D&23r;**HC9#y$Za zkhJG$uB{xrGW@+VGJuzdb@O2~9$tva-8g-L37ZuII{?IZjcT8~`GCsaOe=Zk=`;9*{Ov3a(A z6B23+DBU?)6zBw4u6))@3xoy&CQK1BWaDUGzEy$oJXJmL_olT30N3MSI!#)nUsBmc5|5wLSb3D*F6A>S*_)*9TQNAk#IDu*TSf=3oGo7CnRur*b#3B0N_`tiP$8Jn}9 zXT#zQ#w^`%iv%p#lS^TUtJu+c_X`E;&09GJ+ z0?S!B&HvZYV#vYC<9MZ*Ku_e{D|EL*o`{130Ski^>a8wCvrRAbtIEe<04rCwxu1Gs zUnpi#B{LqlHSi0r$;4R%z?oe)c>Jy&Pv3N=Oj{%xBN#p~$Pgzm9=C_SH-#^=0h+O8 z#VXNQ$qMLPFQpK?+X*HF4u{1JS!0RNT;lve^j0Q%CKL_p4sQ{2LZm4+U>+T4oJx9=5*{7*<)FUevN*`8;5a4v%N=zXd8WrG^h#ez z(u2`>h_FuF(y`Vm+=gr_>h`D(K1j`ia)BfyjBurFtS!^ICDxd9GgT%X@6SYc#*ziM zH*=kbN9r%`9{yy{&|kbS?!e!Mo$J7xczM!zGGPyrI(&3Ehdx<8%R6v?4Jr;Pa`8>- z6)kKHD?cTaPXZ5;Ux&ggILz z@j?8ep6qT7xj}TO?}q=w*;|IS@ds_Ar7hZ0oT9~CFB`e?+nu>*=APNj?u^0@Xz{&J4%ihg7=S(Q z;t>>s>&N|Vhm?L@>>IhfbLN^n#=DogpEbO9O|qTq91;QTe&O$YF(}pS`UXc;MU|KE zK^-r1PSxs?+iMOt<*6sGPD2!dL7^QKYV!pCg0!j6#iPB`d3we%1=wx(!<>IJn~baT zfjZu6g~fPNjGOWXJRTlaxsN}N`ramsI-5SWg1_z{S07Io@0wN#UU&K)uK>nYq3aLE z{Jj{m$mFXpl_I9une73Wd+sHE#XFQXl%tCsg zs4zx8+ut}@dhunA3R}4>^BU>PPabsSW<_ThkYiu%vIhoN(-KZH4S8*B$aW-Ide@kv zZeD1oMII0Vpg5ECDuS=IL%iy|$ex6dCrFUFziqJF)7dQ%7S%iWb0?-SqA$%v?Q|UG%+g9f*goVk zFaNO+8Omt=%u=4w=z+QG|AwHOesR+k@CB8Rk86gfr#?v}RYJOFVJG+R_maDvOkZv) z4hCZoRZMvIn1$5p_~J*LT#@u7{tk~XCFj~t6Xd`cC7cv1j?h>B9^$XB{D`VXsKVSQ zWK`d_ef_5DB`j}i8eJFwfUKPT>#&D(=YA3>COu;D`;8FM5tk{>Gb{Bz>9?W@o7_v+1u20^;_ zg?;GGMgNxbmC22Fs=nB~;w>-uCI#9ew+NlTgD?0^NaRD`49ozXzGW= zC!)s%oA!SOhVe|Iq!ta-f)!M6V%cX7#Bzj6Khaguiykl2yW_k~mzGqaGRzkeIQDf? zvfs~zoJUsCDSrLMUFOLJdu_n_gpYp8D5z~jE=+ma*~f#I@uw#q@J zPWOd6srk~Ut>5@ByIO|n&tnFVov1Jv8KW*%}fIVYF@L z#ifM@7`V%vm2PI99AgTcKJ!@lrD)chyFDC0{F0kmO+lEN4PW} z9mmtT7v@RNr8W6ga1u6%97S~5cMfa?46P)PX0dKBfuU{NCxo;g`1Sn1F=0B;s7?BBY?ha|EdzZcW2u)x|3E^-cQ zBgKy6(R|kbbU(g=2#`-vsu#*V{UKMedDfyD0=n}5?Ry?p| z*l#F>oN|FbJ+k-^^8?-F+@BYlr3|6vlrWy`%W60q3I_F%^97zXS8ihHFZv)7FR)#b(v=ps6jxj9;UayP)?<$ zc&w)znLa{T_(6^@`%7!dj)pl^G5U7lJlksnEO*|CG%IOtm-YsSS;!ty%xj` z>dUc(QU+2I@k1Xcp-!LXtMzY8F%>2jZwi_2S0;aZP6I}MM0h|VFA_~vGOy&@WaYS6 zVahHfZ+F)cfL>f`v;Et+#&QpOvB`{;F9xbtb~oI_%!s-hZ_o9kAV8cmR>0fvZ+`W{ zD|(9L2atKg3|45+-;5`jb%!I<1$y|p-v4xb-pq2Vo0plZl-4JtzU|PQcqYX@>tx;; z<~X-pRbDsLOOz?qhHl7~L_dTuq?g>w}{<%L&q_ z!mSPygYCFS;Dq=B2awX~3B|s7Qji_f?<2l<+US$9{iNWT`bx>n{%`C>hiXRC*B{^I z{_&ZcpGn2U+A4IyKFiGdSN;zCFCK=mey4eO;^wbX9kH@@)vZ7yC*Zs`@Yvt=mLG$K z{M%%D!{d0$-{rkuGUUX%I0TA6NF^&Qr$Xs*UM=GNuBCg|?v-(7y_GeTRYB^Y(oX0x z>5QkmI`M`|ZP}tIC0Reb+_JC$hn*dQY|G7JrM^<6`pUO39Z6NWe({W9baaLduBm(6 zoV;G<%4}R@m`#e9QMJ{})qs3-Kc@fKJpCN|En%p(NWTbylGY#B=fS<#I9neSXoHuW zxFxO&#>fX8`4iv2?`9OIov@Aj^TI`AU4*;JY!tdPShTM#r42Xh*p<6&d}CVi7eHP6 zP5R=u$T{mjo`3uwYD#2PiuFn=w6bE@buIsDBhPG7f2}=#DiioM2$9y$GhVuTXLA+1 zeQzGCsG%P`*0`6r!1Sgg*!TO#lfeo`PbwHz3)duMnv(MRV9brCr@z7_g4Q(FB;E@cAls& z96k93wT%3;%fo=HHP7l$KJnGUrvN7!K=&`p$yy6+8_#wW1RwNwCdiQ43)WsqBLGlU zMb6G*<=xj*g6nb2MSagNWSbIB=v&U5MrFXnMTZ8Vq2k=GZ5T;=HIp6|)%$i;%sU-9 zA}sYr*BDnm!HmbAJ3Z$ip#dswWtov84L?<}7}OItFBediUV1Mp3ozeBAAYxy`(VHG zd{-T0_ei?DjHM~wFM57Pd-ZU|$oCw!&u``|zjVVPn3V{IQPO!UP`{DRtT?hXwjysb z2K(g}q#3Pzbvq46v%9R_8T{Eg3gl`BGXVcId~EWVx!M{m@O#_~n=x$$|A3!)8fGo| zIM~E5I58Gfs*1L(Z&C8|2badSsDww@=CNCz`wN~Q64hW7w|~ezDi#k4jM^#}>9lp$ zI{MaqlKrA((ZMtH+ogyJb~#45sxJ>BJoiXYggsh(wJklt&YRIFgN^k4EKV!YbFyiU z!m{kfa?GnwI$Z8iJvTSi?yaRpsNuEpK>i&ma;5#kGw8iF4Q;8r#VHTl!~_=)oRmHt zVL}Ts=jL!uQ9V*Pl+emjnU2y(D5{H}N;@G_^y8|@P|ZgtOc896^WKgj zU%Izl(X&ssG}`r>W~0xa678{MT2&sG8iaIZ_T^JrOaD;kS2qmEsF9-i_^i9leGPPV z8sBxDv%5gFMI{TS(o(b#(wTGlT(k60l8(W1kG`R^>vvDFQPUHxtu>-|+RtYQH*vDl ziFXhEm#CZOC4WH*Izy+Y;*o>8`ZeHWWZGt%(5E3_r{!AZ78N`@qT+?{1NQy7yuAH z_~105Ga=K?c^Nv)DE~L`^~vqfk)6@hznJaiRV+g_2==2aHa?0|&OlLe;A%1mAv#|% z(G_Lm)4pP&s5}3gX0hVZYj}V|fLV8Wf~7EUzgCpYQRmsrrwZ_91jt2WATh&TSIr8K zXExIjCqZ^*`jdLwb1SL+xODEe2ID>bFyomM9UJm-^!YutxNPVMTaPE>2MPzGR#Dr_ z(!>+_J(%gogm<^V-Nu`l0Ft0x&Nu3JHnWVbTg!Ab%uVT3PfMTU{d@aNk~QX}$bO;1 zvk{D}oSDzDw3Xc)==S(~hofRW3#6JT*A|ldB7EpwX8FMLim_xZy7G&Kb(0Ur$cNWz z#~q^j_Ptz3WTo|3X(ms|7JEP;lRG3;Fyk|V`xEt{fKrq}g!{Lccj@Imjq<{G`(Ye+ zUnp@Km6T=_8k-9*nVwZ?x!64`LLzvoGx^4B({C6K;3Igz*UQwNj#>xF3;Yr+tsiP; z41}IeY70`m{rPwKR`cn{%BxSZ@A_-X5cyo9GEuAY|F*;}p+yEOLvH^nxHlL3WFZq$ zg!e%3>%bmgSb!0aO-&9)!uS#gMQv>Z?{*qw@CYp?nz)iEll|@o-uYdttWdNW)`PVAZy7LZyuiC%|6d60On>6b^^ldXrT>3=QWbi153L`b0sE&$>0#ZNd+ zHSG;-A9Ed_))vR^C2= zg+51M+l5B+#);bkV;*WJ`%|bK*zVbb;hzkRhhBWs-L=rM-a=gZnu@0NuQa<;U!kn0{i5%%S94AT zFxhc=!W2d>CS>}{%6N!^ZBDvq9YUUpPT;x_ZHWjf-zE`S0W>Q151X&}@gse$`>644 z4tRfGbiTK#86_R_>$erg_IR{D3>Y@siu=!?cbq0x6w%tYSBCvnx8LrBIN;(m3#_;}bKx zrVkT(1YEcUCxyo`Uk#GJQdT>x+j;Rp__0owva;l}abrk#_Tyq&vC<$^%a1y|hhUkz zfvAu_@tMAHMBb=E>J;>V|O5O!Jq})vVAi zVQ7YIGhRnNQS5r7Qrc|!M3HZjQDDJQ%t_ZnM7%GTjB0Fa+nLy?>i#fM+?|MmRA@oeVA*qP8LS(ROM?H6$p2Tl|8XjVtS_01xi-yJ~9s~-U@WPZJG0)rqZ@82QKx^w_stVN`B zF{HILo(sE2TC-DwGndeUC6E=N4$x^nAj?hBT&>zeer)ewr$&iz)2O31s*6v-v|4b1_(dsXN~&B0Y!! zD-v$)?246s_%j?U1tckTi275vpOeogJ8hoid%IJ`y8?ch`|qM~lIUq<9@83dNOhDR zJ$`4pJi5TvL-bx8KLI^{M-bnpeEfgnQVWiwySTTs)5w?Ic1{^p-F#Hc;k%rAx&z`S z$UH@l-!VzBT=*drxSSMq5g8z$P~ne@`{_+Pb;-AfFk8t`__MFaby-@XJ^VaCU~$@(>D zh@WDx!_{0_nczlo~$3 zjpXe3*-h~17-hhGnosDkg^o_6k$iaRhe#)sAAmbqZ3+2fP=;gF(xu`T_d9$t#XRP3 ze}taX`{*kH4mMKF-jUkNl+4;3W1%PSL~eSYP(C}r>?t<6;(a#fOGHI9VKwN&1g-0; znKNcsH^~Ks<}gO;Dg{ocP`0UG^9&PjAG^oYt;c=Jb2hnoBAv7XGaibdUut?WqCn-` zCxYk5((xM{&@XkZitbH*I=u9JKW?k{YS#ii8B0<)ar!#LO62(A=V-89L7wmOUgcT(&A}bqHU!;6DY}+rIpZeCOL8hLpea2I>)Vf! z;e|uFk`w>$$sqm5gv)?=kJfH|iaS_q8%<0tsb~WwtWU>x(LZ z>)xT(v^IAV_vSZ0rF9#{b8P(Og1u(f7tRqvOFWmhKf?-| zd~k?f_br(mQ)m5xsjir?TxV{P7Juk-BX<4je~E=XKHtzW`IVIg|Fb7H3Aj%z|1UdP zne%zL*?KrTnVQ);S)j*zx$~O4czf75JMjqezu^IkiI|EA@i^Ezd4hSYojiHnoewg- zP4s%HANe>TTsFLhc1`c(ho6(lM|>x3XJ!wlGV+?Qd{?u?`?_a3fBJh^Zg2Ncu6*7| zzFMAqFOOr*hpA$r!E-|1x!j}|ovGg2BmQG93dO7Y=eHY|HwzKX*_Ym9BS69Na>Nr9 zQ(k}L$!YD*B+3S&X@wxI!8;b$2xuLPUE{Z>h|&UfZ}zN z8-Ui_!U^qa|EYT;2ED)OZCUcVWJgRPDVX4jD8=b6OOoQ?E|h9`{`+dOWauIpRp)I( zHATnm$)Yi`N?>g3SU&>U=C|tQ#1=?UP0_jUN%s|&d1`tZv2|(Wh4M-0P(**-v46cR zd$hqbRUQbH;E4wZlzJ6+jG0|k_M35KVMV37k`(^+D#r6=(g3REo+X<%fYyu0 zajFZh^KpKNt?*~>-VEoGwrxuzN->E%eT8O{H_d_+kaZ%azR!P}sM!G-nAO?bw>>?7 zdF}JcGrxBeMSOZreEo6g+J4iQ4-*WTT?ezTB^+(x`p~2bEO?%-^kZecbyAEmj z-2e^gDkjtY!C9=+hE%`N`|kUdnW>wj$YY_0h@#6Rdpb|w(>p%nB%Am`y0o*+vGlRd zYn%5AdVG%!(j&(!HM2)A7rx%NWRGJr(;B%r93+I0mq~=Y3gBaN((yu%&OLZAcFRr$ zv`V19F->X3${&gC_OYG|54Yo}LN)=a={S_Mz*o!*&}$9Iw4 zrIk9#Xu2EZcFrvpol3gzW<^ zCw$Mx{h=gx<;c-8DuRh9WfKyQo5TB0LI&sZ? zCs0#r0|7hSy@c_>SqFBF*DL0&r1kKvY{Wax z3sw_AJcNen!b|V*SDGZ0elO@WeO0rvkqz`@*on1TQuj>Ofb%&7^>Wz?ieI#vS6C*O2>( z(I=%yC!YeF^(MWDuA&$rL+bhf$z8eNJh*JotFJPn5i49H@f(?X{F7^=*y|&#!_OC^ zQ?EMw)_tWd&%2FV54MigE8!BuxT3)^^!PoO&OZk|r0nF>Z}bLEQ4Gf7gDhKa^0K*dZ8y_W-Bqek!3Rv+peeI%1B)XcX17-gzQ#QnKwT ze|xcCR*c;%5r{UQpN^JH`eJ&K`$zuqhx{-UVeeh7BGKa_>4t*8G3x5>J&|(}8OoPU z9*VCs(n-g)7Z`2kf16}>@70vs?xc6Xw6fz2deCwtz15MwR^+-l#_R>&M)ziP=2JY* zat_IuY7zL7B2UT~Gkto&;Oy?zbkdDJjNazUUo#ns9RCmvcpab`z)CKApmpzqQ)rWS zM$?{+Q=m3K$dY}#;U40RQtkz>XnShpQpk*R?#Y-=bkw0WG${*P6eYVaj$zC?b}pF< zPC>+gCDq3ThDG;8 z4uB?Mh(|Zs{$Vi0y>lHR=D`l%lavp1$TLJhOLvMS$!^moAf62|-K%vF*JGYupg>DH z;}}`M7T3QfOo&)um)Y2(HVjnjQ2IshlMH0&e{jZoMam- zqmZjd^%{V$#sn`d>EUT+F2INVqIZ>jTfHr}Q^ty469F{~aQKj5-Dw8(H>p`kr_80$ z3`s-Hk|G6-YO=(0C@vSchOXNfA6E72og2(Dy_<3$zH(gjmXZ4X$PFN8-|N)n=PTDe zTYW2kcb7+@3DB8-oaq@k6y?|+G}`Ts80|vZe5sWem7rQ$jpx_y~ylUjtio7$PJ&)m2W0vCQx4d6Xe|8w-@zK zptrTxKr71@;>v1(%s5k#Z#&aIt73t;(wp$hXlO;Eb|FX+#hRm5MSczGlHyR^-qxLB zv(=k+MyJ&e2TVI$68a<$zs+bQ^NSIN@)?>mRu4daZGP$1-|XW>kmiLVb_9;R|Ivr~ z4K~OPWpT>uO8xqcJ@W0txLm{6%lU?n1P1x5vvy80?`kTGtu|V+S^RLJZ2hJ&b@c=} z(6)m7Co9%ia$gDLE{Hw6(4_hsUpK4Np&uD;931wnsEwyCd8 zGm=$Js9!=)iI_TrrLd1JN(jD*?)zP}S!v$Ld2`bJJl(08wSD8VpZ@38WyP94|2V^5 z?a_5Cg5_xEIZ*m_vkb!5TS~XxraTn%>~}^0RRYQzH1?$nx{^daEbb(B^20gka_cT+ zpTYFeyYJLt{e0uHI979Gca~eU5T&WO($S)DOAnG+hX(k#cVsrHc1-gwN}S$gytue) z&>aq=ujWf!*tZ4ABt}H-X4{QVCGVklvI6dD%>64jktRk@p)!an&x zP1db@r#79)!uZ-@7F)w@c7^x-xJXi6!X;A~DI!<`>9u(-7>V*YG7!I1)kKsuLIai0 zPXNw}_LOtR;*aUG)=#TK?Zv9#7weXNKDU3flLIUH?3Ei}z*0 zUBs;lE!$`-2l?}+>I7)xN#ZAIT}JO9QsSZ$(oE=5Sn7}=l*P-doNIX3rNw%YZQM(n zHZ|QLJ1r$^Bg~_v)`Ya#qjib3tNuenmU1jf<-s0 z5trtJc$(d4m_yp-$dZZVAdh5(mF8&%nZ=-2+azu3W+ATY5mCKIOIKHyTpV%TR9twO z(0K8t*jE=9H@n>7BQKW1mb$z+rVZl3M@Gd3^Mi1WA|^o=0WUwdzCcap`7I?6Nk%qH z3qt`K%E7UD<`S{XU-WIunR`P%e>GIZrZVlmet1;UfRRk%4PGwB>l`qZh`J)wHNyb6OOKs7bI@^MUyt>+>ao4!kh#_Vp3sy-pnxpA* z(ss`j{9Co5)I%3zt7e)5t6K_1Gany)ftqkvi@AO`ApfqkV(p^z*g7CS=^)y3A74Ub zI5BpI^}E^a^Zwv^+Bm@sptPZOzQRHDl11oLW>POTJGl78N2&~JTAIN-!4@G_F*ov- zH{zueWLcPH98xsOw&`PblUI*_TVu(!a9u}kYbxKzv6lRrFyeSso;QV_LF^_FM^pJo zuGl4KLvI;t<-Wz-#v~Z$3U5M>$%`4pnzuFZDr94oWE2#5IEDk6`qeiw*m~r;)-^P? zvf2Jt|9&RN>=cQ7l&&KhG@=}PpWH)2_bBpnJu7n7_{#=BF14^QqyK`F(LDb;~zVJ+)TiCW4bCqBay+13A*K1gAb5m~E>~Tzo3y_+cx#9MEv#+&Q02nW0 z>noBaD`zpq!I7vhyrfo%WA@eBIwC|KXZs`5u-6)-aEjc{$(d zMdUUAvW=I79APlIvQrY$IMr^`m&F^tK8I~7sgGzhNY+hG#aUw$FfnNr21jyy8#K(u zh)Dh5@7b;G(L!uZBI$3N7ZZy`owm4Os)l=EwWr<6Zxloj!1WmQHuprxne4J7M zJzEt1d_C)-$j32es+gJ$Yh*)`^{8yFjESwjbqX9;`X2rs2uFdaM-mFoyLL z$HwkW&vNXXnuhwY!lml^v*k<{;$*Sm_@_*MlL*4^LX?c7f-l}$r@poNvJeoiY+Lu; zmYuDLEzZYEN|480LEplhtF?PvLr^$UN3X*V|I%+a&8!1rp2@%(`W(1xGap^NgKM$> zMo=(Rew9#wIlcnZ12Mo58*9e+__gV;Nyy?xhK0IHcf?DaN5-;8H&c`Nx;KY;+SLTU zZ-ggCu_WVpo@eH;X+qL~uJ24X^Gjmm54elJ)bqCxZVZfRWY~KBuFFqh!r7rZS^!s_Ne&f)19Q5*j&Y>$KF<=u`drjhYY;+N7E65nNRsO4!p0B zH1_09?3EL}prr!`#mZy+0T46YDs95MTTQJ8squTUUtVi3g~oyeCae3 zjOF~iw}pYAp29!4KMArFE=Dz!Z+ln{yOEF4>AU9|`Vtu&kk@9g>l}w(^y}>mCZt6@FLvdlk)y2r(OXIn-G62De63g;DH(5zC-1} z9Iordx*rMo)8v$#Otq{FHP;_~`ejUm_>{J(7MU-M%NZEJ5yEoE*@Lh;qvUKaZ-b`A zsi`=nUj-9U1nQ84*)D^=oC{@T<^>uLO4Y`Bp=tAeYA|cXHgjXqcd49!t}P`MwW9k8 z>EPxb5#gHcuNsg#)!}vqSqsw_!Ev8HawrN<4OM%v>AlB`Rojya&5TtP5w`$Y5!>5| z%*0cqXNd{kp6ZXj`Le-2pP65Gom1lQ__HL#g+%A`#%8+vGuq#uY>g^~E)T4zHNuBH z2v!-qd^e+RxyQi*7)6&A zJkmKa$j2SowZ_XipO$;2)BGM#T5>R#Fcw~=BA*(qr&A9F+YYLDY1h$IhC5ikZTkIt z3q!fl%=bm@*vOCulX2sGP59BY75W(yQv-b;wLE<9ymx<#mc%vUY1PbCYiCGG#+A~& z(dU*HoY5M?N=$k)?>P+n$@?tyvJ&N33xjvd%|A!=eG=SLJ~^Ct<;Ajhg~Nzi1*@|_Hc~PbDm%uzvT8OKMzS&S`r+Hk@VcPN% z^^W(uMnV3TiF_VoMe9+Tbk^+o6pGj!`ZDD_Ns>>g>Quscy*4?Y%l;xOcLtM#i- z<76OAR%|i^i;TST(rwYhZgxUE_034~2>t(PZueVxQAs!{j4%(Qw!H+Te)Nf16|nUD znl3*TDrzdze*(Grz($D}b4x=i-^nxE4!se~jN7&A;OFr05gf^_%^N(Ly~6`)2(`c_ z9rtb=J?KVFf2+jL$_zNwa`6jF#vY!fRkNlKB`9z)C1}TEoyFpn$jG~h@~eKII+J=* zS0?n3fU0R$dL3fMZ}u^aBXu`f757s#J?)Tea%O%}@t*2T4Eh=+qpz1$?fk?W-=5Kf zcL^P?B5g_%K}0cAzP;dD^({r7lOJa?ndwn3;I@HUwt@xQItrEQ!9%-XB?QKCO$=CNLIEyPQ;CTiLWOL4CO-d0JQmV_b>Ymvo#*PM zD9yg@GHy8i<#NqtYc;NNjS@tuI*19kF*`G-R|mPJP*UoX**)___OV*!7mVwhinbB2 zt4%3loS8t5Qs#>BGz+hc1lwq0rC0>hr)!yh%0S`Hs!@f5>Lw3V$H*Bkl)bp*T<6`3`mx~mFK9(1`a(ov*oQTvdH9kw9-~z& zG}7K{HQIi~ke|68Ilvx+!895oZ#zW&4;-c%l1Fl5Bu?Bm;FqK!imgz_!1PjI*<+s5 zlERGpt5e}}$>ZlISyod1Y>RW=jJ4NtpNOy{>xu$*cZKrhSBYXmsWASr{5DF{p$Q9O zHHk-jMsY&youWho0+oP0u`6_T3X=W8w7ip%Qj<*;}73!nK*6N%s+s zr5OhHF#c@+%%Gqk&A6wkwGj5JcHM|Zaa-Q}IA3I_W8!RPpt~6>^I%{^gkNCf4Wx{< z(s!HOv(fJ~xG0~|;hZgo6<1^(1Ng=ISUM`d%EXd+<+$29J~N1^_edjy(k!N~ZHm!a zelj#9T=2Z9bwuq;4^diD;waYP7Y`|*gkfOa(7^h+_61gfVE9T%u(#@s@t*pi*jOl2&ewIF**`fxZpR{yq#pqC+ohKpX%j0TCpi+$x~3ovfk!A zGM3A}n2DY>a{hivEJNi+M{^$C7%B0k$wJ<8%0Y&DsigbgziYo`TXK31dqVLU%dRt) zWCaD?ZE82rtxc#M>D)H{GQc<9AUWnxt^-jFNv?T{i=itNuucM&>)J9T@;!c=tI{u7 ztk91G{b7r_2VF1^c5# zhpOtnf=Eg||J)5lWg}Vk0+R0z_i^^$FNn!gzTK>(xKFBXfyNx|N!q1-j|0&zPKuaZ114^=VwM^t%U`i`Pm=QUCDuMD-)AJ`x#tQ3m=sn&nR+=X1i-4tJLqO z4n*6e?uX@PU2os}%9yS8xwTgx+i=iE2Xx#nMeuOvJIXctb54EYHg6tVX@TS;?87&F zM5Z#iaZTD%jy@0mt*X0SGzP$ZE=zCQcTE*I*Cc*^x~u_?pFdM;YJ?7+?*d*gPIL~P zgyRbTQQbVd9-l~7S?V6vVZH60 z;!6f!jsM}Y7(3)*0eLi3 zO%^Jy#g71D%qH~mTeAGMbv!`X8ZKTIo9kmJSV|R|@c0mAhRLUOqK(O?bD|<>FJf&=|`d|l=VK{A-XscG=~)SCeW zFM$Ga-6WvE61NVZe-Vsl#t)PP=4cw%NhmU`#0F&%i_?O#w8d$&0#adR0h>q;q??mz z4*R%Qc=E}q?%*zZqY4ZvGG$5*q>IrEH*mW}OTtZ{bd4562^yE&uL;-4>ga@#T%fKJ zz+_DXet)3A9OBinVG`b=o?-KNFhEmAAFeNrZnyyrS!81$+DHI%JhrQ~{3n$+aTQ`0 zF?`-Lti$d4;`o@wtz)S}7k+bPTDg{p2P#+-#sgKYktd5^>bR7B$L0VxfA8l6fB!B? z2Gs0|AcN|54U<8&yUIwR3SIkTOK}y&foSWQm||mbf`7rz&$Wxcv$0Pq>&>AT7<8dX zfu>!mWLq5Z%|EMTxX3Q_@IZNM=Xg%8)GdbC=w;)^WI&~^9x|YQ7YQ@(g(6;7(b_ff z?4@Y|-mSn-|AOv-w(bsvFICFG zJkbBA1;nS!z-9P<{6eoQm=tKzWljc^h5N{Os+h!No5)M_){X2Jy@&e*avN+KbGS-9 zpj|7*{C~7h!X9nOhV`7UT64MjL-DmwTritB&=?e7ViP1iBySR%N)XU(|=#wE^| z*z74?MK_;?QW3uguFVjSZ*+#XfOy$75g9O)=l!pQS524k;Be_c<)xuv5_Y736CEc~ z#fgp=$>~JLg`}_CU;(v0U*MP3$VQoj8v}y;P^8!7s!M~O;f$-=@*QNv1c+;W&eMuG zu=wl@#Lu4eIxBuL7Ga-io>i3%+rPela34W6jPZ|)>TUy9UF17z(D5Pxigo_ZxQgB( z_zuBw%9GR_zfUaRQ2_DdVg`(%Ds(rES&(j(Wn|a?*iWU3lvV|1fwb8z+q-kjAQUh` z1_alCxXspK$@=4+3cClbZKS}?y(_fg>q{LfyfO%3H*xW*B z7Yqe@>J;C+Wx0l1D}`vOz32LavK$Kx1lOPqcVCG1L{O?p06MV#AFrOltKVn* z@O6&&Y+pR!Iq99yV-o0{fWt#kH=4h|tp?5(jvh(t_b=2svxy~FqGAdy_cbAI@W$3%Az;J}?tw6Xl;$o2nG=5md z462yzgjM+Y{j~@@56CIj>;y8!Z0%mD-es)t63=E={Y1xkjG#!kq<06BxN1G{>mLr} zNfS5&&3h>AaN{J5s$auB-e~d_C-8Y+8rV_!DzUl4scG=;V~11n^`c)^9NNiR1Q3B$ z45sjc;}g#YSTI;34QAdV&4xDnF{(I$ujxf%^G}3*d~^I_|M11JTOdnT|Dn6(E!mRk z{DagQ3o+joR1Gf7buFO?PXyj`w%nP_DlH97l_U@MG*||aW-qO1qCI$p6PY#xTww;? z5SgO$L@O4!JmZ*C+7wMl?nN+?p=&Tp5eg&wa?3-EFLCPnS5@AoN3&_703tw7c`19J zswJS=za93hrI9;fsAnHdVL3}3oBSLroZvepw6=KR1_9moam`|E=%C%$UgC7Ur{PwD zn>}dH^>ZC$63>pWLdc*dXxdrQr|%Vilw_*G+}=|Dg{D7Ey*UIr)h!}amyRU?P3U0K zq-Y7ya7t``2+&v~+1`E50WOD~yUQ1S4<%0<=UXS9MI;-BBarSn&%=$qrNmf{HqYZO z;}TxYCM8Gtjga_(exYwh+}vkF!XKbOG#B}oppUPdfqs|Xg5Gp`qd#Jt{-* z=IPF@UdLnJ@wl`D653>^<8p7Jvdw}ADqOqiP}#1*%lf@WhS%|_tBlM<=foYavWm{~ zNY`cYsXL%*tpqQtZ0#Bkl)Gk$2f9lU=hHiJ!@N{Px|1X6F=Gr|ioPSbz=_{)$Mq8w zyE5@WIcs`oPi~gx72zyGvvae~e{>8=e4vrP`H%Y^+(*4jzGIIGBG8$%3QdZYGMe$; zmEZUGNmG;A!3Ez9xSBgm6TFN6BlXhn2o7-g_hhasFU;B1c)Mv3Cgei<1c*8Ncx002 z^Y-4S+!Y5Ej?OrHh0QVl~eJaBK zPVismAn!tmUS;5-+JR2SCuwYGV44g5W+0dX`8DOwAF(Q}VXqlML05LLafS!R zqZsdP;10h#pmMDS4-^h3amU}f72;)s^KAq~^^Lp1?ui0l-2S}cG|9Gk=mkDly5AZm zjJXKCVq-^p5lZ(WhVy_7Eyw-q6x3TZF}m0Xp%p+oU^0E+Y|ah0+}$&prZB#~dnzOg znC9)>56sb9Jiu8oTGtaiwCX4Br5@z$lUisG2WfvEDa>V}EJxOy0 zo|BQqn2uKam!r@F)2_qfHCeo@(4iP_vD!f5ret)r0Y2Kmi$3jL3ZI-08!kcH7IfMy zPVoNP3fh2r7diIq{v)W^cIsKS zQiQsg#_6c{f8`GJ6xSUPy+?_{)6E)pl|rPW(Df>53Q8b>Z+nJ)mB4igZ8~&qs&{hA zHsZ~0#}_Q6f>1Zf{;!7W)l5NYV-m=p2Yb_(j^ob(kfUP$Fqz%}22jX89rCpckO}my zStd<%cFmO-IZk*f8}J1bxt%NmaNv-I^1v;_zRE#qkXc~{^Z_jJMIwTCP%7kl*ety? zIXVK4Wuc^*d|v`y&96o@WA=Vw?3NBIVmu1Hx(~Z$$_fHwN{h-ZK@RE&3Eo3-n)u-2 z1L$S_L^1ul^jY7o1j8hWOd()iI30af=#>HtMN|bthJeXxSEKi^QN$LHg7AVc`>Nb!Af6=x=R+o%FtFeEn&nv0+7MeG7uUaci?7{l>84E)*sJ+vR3J z8e_aOjE%8_)|D;mD`6l_3?gj2dqW)l!sYI_+p7k8Ul`#%Wvuh~p{>&j}m~I!xv}+ITVl;q(-ZlW7Z9GjhPUw{g!a#D@VHDtj1|2psF=(Oq=#OHqBEpP`_+){ENR}{ch6iG_RGpDv zF?bcdliZyi+NA>V(biK%yVRX0Oou4&89J3XGoq0)MrXy2muRGtnLv?Oi|D)u`VXmN z^ve}G3o>;)MdHC$<|H~;qbzW2pn;&H?&;YFw?tSB)IGu?3OSG#5F9)praoSZ1 z;BXS3PxNjT519GU3*}2aGWv|gd$9S~#e3Ml9H@Z9?d+`&r6ZHjxYVQ8<1*^m;i)rl zRX9cPjSRo(9k<1Sg>O-vJ%4$|!R|SWzx|IK9`U~?SAe7wp&mXFr z;ZF~1o&VPXezHc@;rm?6^5w*s>GIhmHNLm0bZX^i!BCSyGS8d=6o7^Z@%f%x=0oCW3EJ6vK9Dd3i*QO{nk4ta=ZKs+5_OC`UuaM zD+G^3ZDhA>F7BuLdz1k93=Ba)q2?7lv;zYGi2%?Z0EGZRJs3{pYKsgYD33(wzf1r(+M5C%lhfeP5?1B4Jj=ynRX*L^X@04#tZ5EuZx1UTmh zw9SEk$AJ{e0m+hki5nY;+YeZb3ou>-;~OxTPvJS$xqvBbK)W4a)dL}}0O!2Gs0Yre z01IP)g*;$=Pj3Og{I^Kv)BK?L>Ko7ab}Al6k<@8sNqc4^UwN1g3z>@QV=ezzSr-56I!Y zE&vb!01QamOQ-o!0f{BJg_l&ns0su8=eL056X074P_qX7&I4}zfq-;@u6;rWfD!=6 z2l)O#&p_S)+Jb-(;$U5`eK*`&~#si!`iBWWv-$Fd`y7P14wTGcisU11~3l)4~XNXCbR$`4*0!X) zR)O|^dhv#E3U8=>3ELkipAb+F-D^j&J!wW!A!gk1@c9E)Z`G5YY*+Syr%se&Po<3^uUu z`~ihLK;fkd`p5tXWUvj$pbHfMRCK}d+L~SvBw&dz*>(j~5rE+f#QXrnd;`?x4NyEv zAjg706Wn_NG%~;$Dgds%Tv~`9g{u$W0a62CPy(R4OEkSNeZe-x#R?~ zrCu6=h(KP6;9U@()D%gQo3vLOxI~&yaw;+DO|nY_7!}l31NN3}%LenyK((iciahRQ zrx@sb#it(VAyQNPbP(04Lpq4$6cat<+mtLlL}N-KNsn&dGifZ*B@9fFl(#Alk4|(k z!F%Enp9-d1QJ;d)uY5m&C#ChrN7E}k$liy5BOHuV_!L(5)y=ttNHc{+xME z5~ALxO(G)a$fzc-0B>5>LCp6n6_=At`JxO4%@C%VLd zY3V86@4p3MC%Nc@Zqi(MK<+6G&0w~Uca5E3dFlHI@QtijJlI{@s{%X^@`?qk%Xnpi z(`CGB!O?UO>8T64T-yEDAbvVQT_E)owENPaa1h`BvD_&N`eT;;=;)&B^hrX%Eu(k+bsupX`Y%zig#XF#-jwkEr8Jc%%6)?XGsaJ(5ACe(gWwc)n%N`uLJ?8x0oX>R8_ zsdL=MkEF1eZPCDyfi5`&N%_t8EPSD*5I#avrXSL@pR|=w*bs(1UkruVx zrEtvZQjnJLC70Zz57suZoLhcj^QYRLZp>*EtHd*((I3T>nRz4QB2kEoEK85U^Kmkd zSK;S#gE)2)wXC*dO3^(Ila+LTt5dmfK6mlG_CsIU*lyVg9dF(I_SL!kx-!X?dUQaJ zU3kvs&U%gwa&{vsIOiNX?Z??K^$>M=1FhL3_t{7L#0#oypmUq`UeK9qPJDbR#Nx$_R}Hu>EWudAai8>68dy;!t(x5 z2v{g0#fUMbJDo6X8y$J_T#h*U)P-N=6&!kSk=Fz9++=t@Wxw|$F%_n1%CE1A@Es)a z&acj>$!wU-iG$-2%8e&hKr6yUa>@#hRmXeo9XLfL2#%&-TG-x3!myvBiUq1U1t#0W zPP2fK{gs-Do;?Qn`dz-&9xip6Hcmi z(oW`6s}{gcE3mMu?R=?%_dCzk={D1bdUaLM+yqe$uz~_e)=lQ&szI+#x#58(1++vJfOFyT*t_uZ=XAwpkxH6iyia{1~Of<4G$MZ{u zI9G#~VzIiQA=T#m*-><9Eq*$IY*bKvH7o(oxm)()yUF$abZe>|Jqs%>25+b+aage=d1PcU_#t$!2v0>wesMCtW}wP>h9(iL_8hUK0StSS%V!!K#QgvWvTOiBTdDRDcT~KR zJT0EquPPQzpWK;qYeP*?diY0l7n9ki*Gdnq_oZd|2J}A6d{%8DNuOQ0GF8z?npk}9 z7LFfWtC*pIcnZ8+{gXjF7T1mqlW)X{$fw>X$2SrQc+c?>*-`WDeUN>=N!&^CxxtxE z+I=0`>-`^=#FU;0RNa_5>2o5iQJGAD9$MWw9mFH9DhF4gI}Qm&YXvV84DWvF%0DJ@p!i zrZAK_LMhQAQ}Pn^F{?OZ!r~IDc^(au@r|nQw*%R03ElQBcVCz^iCGp2n=H}Vqy4JC ze3SuAV@8HVko4GJ2vx%hG~3M8yz7T^Z~qmVFs^>}FIR2fAZi-XJTk(JIa&Pm=fmZ5 zSP*im3H7vO{p`N2vt?&##P0pCsM{R`H5_f>z^FZ4)I2smjOWn`E|n2BFpYH~^{bP@ zQIWMXaXXVZ1Rk@PK&Mq2C%wJFR)%fht{Td5XTJ*`F*&;=v@-qWrIf{p3EDHEbB2O8Sd4t=pYO9(i|Ks7_iYAGD{Gv^Ogfx~%YXR$Pd7 zPb$vnGWLLAlc|p@3C~YI5G6RY41C5iWEgB-=W6;{eWadXwRT*Hj8#!g6#W~r5ZC>X z?s@7iGSFXh6(35SuS~|d*U)Gy09DF9V-l`e#8%~LOjGx3I5RmT)Ebar*gL;Yx&A|~ zp@dah1bofX5PL!?ef}&`O!g;%E`Htm*c@SpLjBZxmUBwp*xu`}4AJQiOj=!@U%?W? zG!Jse%~Y0k7eNi?aUOHOz2j;TMFrJozJ473^cT10!tcX4Zfx&`tg^IYb=gHsxXhZ@pz7QV#D*CT-lZ@u%J`I;(*3|g{4n)KG=qUM_+xLm;6N2aU88tKN5h=&QX#H9 zR(K${PV`PO!qLw7B6CUXj|8-PXFljx*1g}QS35z^uabBAVY6{{qR7*tyqfYtc!KLD zuVcBg7SlVwez$v7|J|{Nxm>O0=f&*Ls3`9`#(U0O)yd-cA5nYv#HxxEWm8ZEKRYck zex!=hlp9E^6!r>6QI-on3V6vs^q}?lgGzip;W$fB+(E5SMOJScOoi~Uf%&h7uSfs# zpwSbdMdRIsl_GK#Vgs_&WwuoJ=(GS&zNx0?L)WOD!-ehMPOa2yZzN&cLly;ak<{xk zy?ez~GM|g!1s?kQ`U9_hzqCi^3d1!O_l1cvAvg}-Yz8g6=(00rCEim5NMJZnN~3;R zUK{q;xKzj41{%9u{ApnX{8@fdalxC7S#t59_>@2clOW}nq1RxI1>74A>nQa%gBtmD z%tAg@P#V~slZo?-HQPzB0a&=?Cxi@zTFT!-QhNyb37xVk1M_}M92&&_;b4W%(YB9r zymInHy$vq4Zr_$kS&v$rsKXh{?p4~SeDDlbsAT8%gC66V&u=S*i?LgM=_H*lsUS>B$E9wCgTL<}k~KrrJ_NyluTf|&|2-b`O;2}92dfuGX^wP?Rs z?iVS@bv-(Oe}_e=5D;OXVj`y`~XQS5ao ztNF)Qy7b3yzRaC`Q=0KQ-g&5iMJ#-k+tu+G#mXrnEG?;|8NtIPeg&5}uq%n-oeiXxYVT+5P@OBw!18r#p6X0c4OJTU?`2s_sd^4T(G0?y{1``FDI72)8B3N zz4uUxgs<)s$PD|xeXwF_r?9#~ibQMQ)6vWqJUKCIl8-!lzGX zBE-L^f+L`$zVe#<$jZ0a``~9W2E5z7k?|ny>_6WmJWlH*?8AXW|FfgjAdB|U(1Yo} z6AHxgOH=`Rjuexw;a9QC!*}C^H(ZEByD4?G2&f+<+|i+*Eb&Gp0s8u$%C7MjF-=Gdlpb1zKf>x zAcKDjw(IK~sQC9264%3Una+s(3qMWpqnSJA|V+BQ;=?1c)7b(6DBKfb5tv*zaL!Q&!h(|O=ku6gL=;w>TY2ayY-{4FcE)AL>1 zL?T)J-mizO5BH3L*Gs^SoR~1fZap3PPAYaaJZ ziMrGg0N=S)RUjq-8&@D?_XZe}4(-dDy<@7f;X9tTEWO@ZdG9=14O`SPEh5a0w!HZ< zOa&F(i?4_}>3VfcS-mn5+1;wj+!Qri^|MtKvDwLpsl;y9o`<=me9_BI!`dyx{9p5{ zE*4(*h){oQyRNjgBSDNb9muM)|AGWps3B+DY1M!C4?;gBDc$AGW()!B6dKh*^1M3@Sy0G zT>Q1Rx9PeoG_E;c8n&Zi@St0YxPKQ26J>jL3M=g}rgN}5WUaNq*k`@(@Et9?I?w&Z z2Nep5;ak%QT3LSdy&v0~ujy6tS3_-@(Kj`xfE?Ls9BBJp<}lUw1v&qFmy=n#I?12C zWedwKM!wOV3B1)gbVje%6JqzTw33snsK`>5kWl@=E*tzEN!iJTv@=w~3Z*sl$%q3o zNwLZ0{q5H!*(M=GPRnl4#OwJi`4@p^qT0=c?!q%oL)*X}H%Xc`{O9=RaXSY1L+&^A z#&)Wb3u_HwQtP<<%e14jRExnIXlx|}oFThhBMo-kF> zX4i$n;{u_A4^vC}L=cg0ce4W|O6nU{Tn6b%*B^VsTB{Yz%+%@Jxrwe{tziEUy~BQF zeuqa#BTT}u%lY?HBH_=ZmK=LWOCrv{!3 zt>w_ol*QWVLuA_cqaxE5x?W{dkjhOkb`drBQ@TCsTWq7U(0QE>>sZ|RiNijM*Dmi8 z;^?sZWR@&D{IMmp2pU+z7ggMrT5b8@|4rF7xro7+Dyo5J4b^^H@7=xxOUvHBIBK~4 zMxOk%pBv^1o33ns`m(|?Oc@IclgZg`vZENLi?2FCSw(^J<&WCllYsR~LbT&4WuF@7 z*sN>uU`$ET$C0Z^t;7uwW??qo~wy^5qo8d_ZR}R!4|hMf(OtUl8qOCY|+W-Jv0Yo)-i^u(6(P zbBoid=6aL}Y;=`wB)IXXnx4DOe_LYj?2<^yBJMxjE=<;jq1L26^u>R*V#O7or=R$9 z*TXvxOeZ-6H$Hp;`^pmLH~M?&iAOTtcAWfD`#$VY^dq~GY9K|WJ!cI6aIc7!$RgFQ zieihSmf)y>((OK|>KmwMK2M~E0U`~9`2}7bwi|=SEp8zFU0(6Q7pP55dyQ%j1eH>o zrBCwhS7W!^W|Jf}mc4vi++H$e^D>IfXEn8Qg9lRHjYG-UmFyZqf+#}-n*NmThSwGJ zvB#V-t}H&oZSVU|%$pCF3cCjp0-1{jU0{;KSKR?~gg(dkZP#yG`|e%27YSa?-JdqQ zJ!uiom8%J6A2$EJdgmvIUmbopzhs{twcmca(rFlmFCi!?*#g~#hiUwr4w}2rB9$f5 zzGLOP;`*}XA!%T@FPp5LHsoXaHjYu;-Qj=9jmdPU6w)U{C4DtEDC8px+RUe`Vp*eJ^`DCV zrOIy`c#{u&X&8bipS^QRGc(rBMqYnoZmAGvRrpzi9CbqQZ|Fo`}z&07RBZE|+x%JxK}=IL~Ng{Ihn!znVE;5?VF8 z8;?5vl|Oj`Z@A6e3C372$nw6*CjL>)do@HI2nAIlVt2ga;65O8aq#?R-H9`FPTqAC zL#Frrnk$Fu_LaKO#*=hXeCUYV;-rMHIsMrMvlCvX7~W5exP(F26S{+yvXaPQBxRLr zv>|_Kf*p2V1;`-Y%@Shw9h1ww~SI zvOfoal?@1VCwQ`tKQhYisNU3#Oyg?q)Z0Cui~iP#a8%%k{*#6YC2D48`X5d>orsAazsa)`;;@hB>tP=bMXe{v0Ud4Z zOl{4bW`>9n`+u47TqZ(6=6KBroErV{+{Wfdn{VSSv^+4@|Ja959!zv0U-M6% z$Ez0ZZVtaOhTHc)uVG*@=?aG%{@Xcr7}DyJa5Ub!BR*J5bBF4G5}Sm7Jd@z}hq=yq7&|*w?;vyqI7x8qRsZCgTx~ zFx=W{k2V}$Hp;$ByQ>&~=cA+cw7@)DPF1urEKF~aTy>=SsZOvtyNpc(V8Cq_`JVgbgfC11kaH5isfzQspum={1&~{{!3VR6i1>h zQlm~d^*_VUH^v--?n3?DN7rY}Ba*_X(-*_q3LE#A8{@)F9(jK}^zfF;zu&$h$v?sB z(~ruyzB0L<(@a}@UeYx4&JP{2y98^?`lGWit3lLULSMJo@-AOP>n8}t?5@fACf6XS z&5)B;+b_#$$jwFPON=~)A1J@eXt&=4dFE%S7KBT&Sv4I6ZLq}NVx)}S>8)R#jIvKu z-f!vykK*#1%PAdts^mo)1&{xzS?wU9`Km(RDYXVb*_24YJ=}jT3t|reDeN2C4x(>iu;rhfDIV zW{xn+RJgiG9xMswI-b=iGpe;mj}Ck& z6sr6Cx*lZY#ajJJXLLWDXXs%Ty9e)^*xl!`e?QO2dkW4(3EcnB?q0g6s|#ekC!!gA z_vdXLO8F;+hgOD`x7Ei-Wu0{$RSdTeTJ#lpLx-TN%Qc$?IosLQy-3E?dFg~}x+2sb z&8;D9i30mz(Q%StjAK6xjHgi)v4ZrZ_f^`)YMaPZ`%)@M7KDQiS#^8wl^PfnOHn4j z+Dt3kw?;otpkLd(eaOd_SWkz)dB~Sufa8m-$3xdieyc@tz)n%S6Biwp!X1{PXdJ*r zlZi0(LxRVIPN9DOr;VHxh+WDNwyV!Hiae$-rsqGucTo#C=cJgPD?`!JyT%Zt)j+-< zzMo=nn1cxgcs^glXT|K7+9&hE78DO zv=6??&kJB@gMX_TA%R5?3HT>vzCoH6k^gGSdjFcm#YZD^kL{BIYIZ>s2YUlX+)Rqm8}@Ox|E=XoACNtHm{34ulQ`7AnXt+c6uGF;>X_-tM=-zTfJ&DJr|X2 z`hQ}xZbHa-88XOV;aohcW2ii<AiGFw0YMGZ6xTK+k)ckjGpmX=eF*@%+WDnt;7% zNRU+i=W7hXOS|{^R+ydh`N1}XjgBh>rCn{x6$Xy)c{T-cG}p+@k*~sn^t{MvtQm*- zk2LpdfUw@HXu5`Vukp+zO z{3t%^_K#ntBE)bY=U#Og3VmYXgO%x%LGtD@g+EI2GSjlC@gE3K>Gk16w~g;BP)gdu zLduCjTue_h$y$hz3q~uB|DvXYwFDkN9yMzxV0t61ByX!%$+3D%+_$x`mk7KW5$0i(DkPWvI2Q5RNpYz|$6xvA$=#w=^!*!xnx7^0 zkjR^#U$2W7Dz!BZeI7G*RWHZqhMwN{i#&|DnpRpiF0U=f%UP>WTv;E*Um93bP;4}? z&g6D`7Bywh-QkV;c}w#$sa!p*YeLwL&97Wx(5M{(Yt+_Fh!0oQ$<@~Hg>Mj_Dj^ow zzr;x^g-mr6c1@YSAfxQ4Fj9(aYX)ajW0K6p-3Gk zM-Q5}90E$**MBxrKA-7+UIjjWDrX|l`)o05+%-J(iBi95B6L^e^xyK*NIHF2cY33B zCfnMTafiz4h|ig5hljDgM#Ulq>rRg)$#7TtchW1+pMfsCJ+R8mjXjk{qX__dBvBW2=RFnh&;J8vFDyW0-6%1Z@HQ7t8S@wN8?YD%o0UKtI&9A9@SSO-iyGVgMERcCO1JOOeAOAsd zjRuDO=*xUGx+w}1D%_L02b;adz9+lgobt8BBqZaB31r-FgDTTMH@t_BtG zR-~jntf)GO@P8)+Lr3j@g%#yWRoG*n6*q?8T{jCVm@Y|)QU0S6b2s)f%dCYJ$R9MNMF`2t@ma{W-9iv5Wj7bN9tZoS24skl$` zkB@uv;wfV(9X^JEPsEru>fLYe*dWmmQ>Fh7&v=CX)4yo(yGHtZ@6%;-d)JV@v@b^oi{!y1{DwKI3Fu zQA&emRMDfPsD$2)H7Thm%S>IH96}eQk^Rb1;+Vh4qT)ck3R-l%aO1o7ua(tk?OzAi zZcgT|ryc9;_=BIG|3L1s_gGFQuk+5xc?wMo6n_jBaOU;F2tSqhjxJ1)Dvh_El%wwB zZ}UbkIMrHh3^fy(}@vq z3qPk0$`ccOc)Z6J3}eCJIG2e@KS88j$~}Oi8tD|AdZEVXqZwDU|IuLLP*ch@>CTYf z_<|ZE-7TF&984!^&o~{y4pnX`Mv~O*E@j}bl8(Vp*|~zhE&LE_3!gUDAr2yrUwIWa z``tjtvG5+R4+jIj8NiXv{+vKOOyB3jh$lr%h#|#i&_EsYj9b{=V-{({Kz^fu#TBOS#j?ai_C_HW31XTyIs zbSsZZq{B~H`AgnS`wEH9A+|OiU-s~+8~=;sYEd&QL+{_e3x7)yH$6TM^T=6k8@Y45 zH!0PKVB@>8@nEp#Gf=t614A$?KC5 zTxK6JY1KrCT3$P*0wG-6@GiX|ZTz8~y-qA)CwC~Rm~84&Jc~YYr{ zclFm&GnT%!A}#7KDe5dru%s}xr>7g3st;qAtJ)G*A0_!uQ!H6L&|br$J0lwnJqZj2 zX#`G@UmYrlqfOD`Q-^rsZX6)GKE&QGzv+r!GF3cW@jmviFhe4*TIy;2G@eGzl`KxfAh|3 zSj26EBOU$?psJ{Mwce4b6{2WGc>G7|)-mzRIZxIA`-f8~-qFoS+SoX?Qw&UtOXN&1%12`|_g_dAdW3>#{Erc-kOlpT zF>a=<*9(Jf&q29%5eOw8>pA}f81qLN38cuO=zx7>P0JBwz7EqoKHp$;DW?og4?BN( zPAswNQ9d6p_x_8NqmT<(No{s3M5ytbWViT2#oK1tUv^3JV0~PDmY^r)CBGxBuSz<# zq)4%68z-f&I+>4b{V-Fh%W-Hk!WUY{IQ;Kcy?(B*5{cbfekiMU`7X@GD@r0!hr z;J5K=F3UF8F;?;r<$kapmoiNjjw3V88-|`O;YH!~nLp+i9O-QTFz|tUsDHbbPSVcB zSB+lvqji`qYZ8I-hROw=d@(%Qc98^D;w17q?m_Cy+}bv8Dt%+eJv|ti7TK(2uTavD zImyU*i|!s6hE=h)d!yq$V+H$yVefkt|0wT2klsuN`QiG+=2PF@zrM2y_woJi^Ypu? z^|H`>V?8i7p}WTMG-d-~e*R~3hZlX{Inp&Id4_c3FPW>|~K4xU8xS-!Tjcz_=?{z$*<<)r!?59ioDa|&3b)Sep_AFQ)qpIvFUG{1N(Ek z)}rrVNRRGbeX7siOz=_eoughCx3Ey#64kvBP60PoYPs2W#pgMlOu8h7eze@iZ1KxG znW$!d&-_v58KcpzL7SRG%>wULw{rtM??~Kw^NTbd42p_RBZj0c-@tXF7@B%|7RVL1 zv?0`}+kSUJ2voBUy3tQN>NhD&Y$~le9)XQt{ycXd^L^W(-5{o=LfnD~l zIaiwosuA^0q+?MRBaT^oO;2?3M>K4Cwc`2BCh+Rj&#wt>@TQfOo8&5mj4EHnk))WrxEOwL8}V{cokg&D2(n&(gY&mRI^S)q1xK`)mMhGP;QDt7l$|bu<0+8nu)|Uc;GJ#d!Yc40T^d~@OIehkfO|$BIka3dfh*S>STD|;}9Kt zB$C=c!%=0MJgdmFW;L;{_c;SjP`3zVIX0Wlf0k2ok50l3`s7~psF&i}&hyNijW~1} z!7;nL!ci9&RL-@|vnI83VOdg6BL;9mh2zzU%MVTXj^Q ze2`}_lyC3fw#csZISXz(dpi53LnyTAuUb8?(V(+N%Fxm@~*TbTRGx zmf2p5UG+v{_S5aKfLJi~jM zdDK3n`Ce`vrs;&k!f96>`R~(C5H1M^-8GW=e4KgY)&46BN$GtaiWWZ|g4R_5T_*}f z^cEqMlY{_RrlhJ2-h6N3VRf=fwKdLs{y}R8yU4Js$e^o^Dua%{Izj8R-IRCBYJcPT z+H}AnX-}i#zx(E>Kdye;_gK>lGBED#wxuuY_v+C@R}p0ct~1;e^8~Is&GLjgHRYQj zhaXiF4tvvX9CTBhbGZmwPIQr5pUtwi?<08}C|;M-8KCXnkE0J_=oDmqp0GZ8hqHg zzw+3=cf@d5QTW@4>i87&)T_=U^^i54{U@^_$V?VbAjvH+dUYy*-a|n&N2+Bp&vHd5 zuv53}h|0S8V*2gBpmp?LLDaXeG88RGd$6;A_&jHG1lGuN2V31^OAAEKTaX*Nh{qUo zqbam~We79x@!2QgaOcYZ3AYPnO;XtVo@mg zqKSUQlQ;9zRB3;OopHB%sluMSZAkVDJu`OLZ8-RsQj1Eq>Qec)Zo}MT7})f(g*&P~ z3K2zZ5&ifO^)Ok`Gy$(=_HSnqN-zcc-c&X}YRpKFns0b@2BV+#xXYuwYM;~zqtN*J zv0M@Ncv>+mHiJ2pEfc@e;dCS&e$ef&y4mSja6KRY(PnXb~lmT}&$Q6fQ{_v_2G$d=9H1yC5OV zjU4JHN*`rl_m9<~(1P>kNG_pdjV1(@u`vg2s!N~Ah=9(pPR(;HUqxj~1R6PG^`?16 z<>U04;$DnN|7j@A=%g`wYxG8Pq(4_>WEamh!&{2hETbnUG~)_m2(Q={=OT~EAnoHp zoRqBnrg3?%Ij4&$N4_R;+{@FUP@>H;lZ$@)L{82{W)ZQtalJ0$HU8t4cp@ANP92al zbwyeAKmYR}PoZNNdQmj=`?OJi<4JJbLPzA;4Qc7Fgsg=#%RTr}pXV3V#T99imS8t4 z>%lA9GDT^|S|q(0-lX>wN9-1?j8$)QI@OK}UakHyEfS{owqR;fCg~wvYC;7eSAfUt z5tJknRS;+hQln?C4oQ_ts%gu_<`ZYmxhcxxxxr%$AcVemW%}R0T7kUr*N4%^kXwA| zC}DfiA*mA$0~OVa;~0wX5O5Q$rz^Go|mft{+U%gpB!W}d9253WDa zU5&pI8B(SHVf)k361VkuC!)^3TL7Wjgl>0S?gc=vhhR}dpZIU z@aGtJIjh%+=hRAi;zP;Oo^L;z{(ekBPT`7w%XjdsdQ{(BV>CFdj;jg#LEkG;L@9eaB*1!uPY-y;E**)34efx=HxS(S&kvuHn|fu;8{NH7^(J4noayuW)9#}OycJ8S>WQ_yCJeninvHgrH$CwnWmwu3QISGSgh_FJd;{(XH-IOnrQUkz5<^OTaybGBShez?(pehZHtkw+`| zv+r6%){#|D^WQ#p_y6a2uKlW}8r7d)Ma6HQ9=fc$qqnK;iFiwj2p1He<9-#`DYXu^ zuPtUs%Edmv6LZY66#d525+Yjdiyo?VjEnKfK1u-2^XmO{m+9<T1`~llY8uJu8iKG8@hG7OPP_u`5pI zlN`&q{=1uo+>$B-lPD?27Ale)Tc$ZiRs^fYA2F4w`WQ1-|BIrl4r}9S!UbB2ySrPl z;I753xc+cbC=_>hO>qfcptuz%R@^N>ad&rjNWOf3Omeq3vpchw-6zl7&HFCnm+4w} znl#4R6zV2-W*kW3Sy%HcY(HUpiNSCq8_{0YeX7OE^;S5|s|8HDjjB6?8=vnG=f6Q7 zsC6m>WM_>yKv70E>FJGV8j(id;V)Y%drj(A`cz@2?peuI_wLwKQ@vs~R)jXPX*7~a z%==W6VfPrV32QuGh|M3r`c%zb-56AE3)bcu9X(+l`B^r8_4#A%DqV8HGTlQ#gGw8#oP}>vi1QUBa~1FAM$^9C?NiJj?}FiJ z9lI*R%UkE)vzR;c?=*}h5%UE6fBfyS0;vm{#1Y4SeHEU?JRco)8&+=vR2&zisLtp- zj-pV9TvDP!C#OUnFLol1uY{R(G)O~(1Vo7azY_*tn7z9Qy{t8&eQhp^Ky;zT7;o;C z6ENE+vF&u^DtIa}*Cv3F7KfI(B2mSU3V5uziA# z(TsRnShq#sZUoUb9VFm6G*i8IkmONpQX9Un|XJf&J)Jn1$As*dQd1=RzA|M(9&7OgdwzLMKIqE=D#cZ6#HlGIeoM0;g? zvMrRC*1^}1QS)W~Ci-#K#B@~T>abX~5gpqmjlzD(`+GUJ9-B;RZ1+XiKu2G*&nI=` zCgtr1ZcphE2Cnq&Pg3 zbT0t^m8$|g?cNcWg8g`yxCTr}IDMU4L;i(%wI_)`qS25=70T7o zVc%|6aLiJ@J~{t~-K1cJO~nPk#(W2{C7cNbuSYLo0W_k@zz0!B-1>OH?=iv{07+fI z3GKe-A6yfh8$Zl_A#tEzZ+owrMax7|b;m?q2J*JR!N?~TpO~B7nAF#fn_Xxsg)oLf zL!0W521(PeTc!)eTKOZwoIc{8*7^q{50-$Z`7f)0z&B)%?(zal$0jYvM1kUl3Z40b znG%Z1Mv`N}?QVgVv5kGBZjf>U1>3TloTTj}kuwXP315drj3wVskKM1q_pg<$zPY~t z6^1p+I#q$lHggUeNEk-XTmosLHd@N^2`?;~ens}YeM~`XiBIrd0F!v&^*)5w$cSZ>q94_|B{u^b?PAJ&8*32r3&+qHu=*NDrWm3heug zvs3PXv!ftPGkDc={I8keFf#BUFxn@j&<@$T@W2bhKPt}*ka*&=K9PMUqz7dFx#PhY zF~q!in5D7xS-JU3iy(Gi7;dX1@I!!}|Ll;ysM++w%V(o1XqcoNDbpJJsFnm-8J^-J z#=N!1R#D05_FrV4jvI=>Y;^keJ{^V?)I8x692JoVCxVQASXZrFrMDUf;zyzRLf*N; z>MFTD*^4TY;ZUNl>IW-Ti@HBl2kw`q2Aw(=2d?U|o-{Zgpw8UCv+ z4?}4MKkL3w_l;fqSZCGcbuNdhJtUI!rDw>@>ZiDCJdo=e*E-VJBWrO5*p(6nsW#7Hr9aAAOOCZPBnBnP8wd-b~|#*gUUCA!g`OdQJR5jydmSu!;=* zRny1>8+cf*BD`8(js7SfruE})8PkMQFJ)i1=oZ7Prr-b&p7`eza7+dY4Z(T;z2k>< zjDIC(_Bh2nv-2Zf{nCWPCN`pR^R-noR5evkKmJY_wnuw`*#CKXpd6rI^|jlr?11>k zs%(AG5wkfA-+JTo=|em&dzySY)Q83J@4GQgt9C*IqxkFUUl!#RdBqD!*06pd zvw{Dj-8*CLXUGR;c;tVJH?~=~;nlC5v3|M033jNBCvtc=#&T#YckrWMAw~NgKz%}f zlQ!yU@(0dR*ni(+d*I;L6?XQaBk_!MD)j6Iw_baYz2~i#St>wD_B562qsT?-ieG!F z_WV)At8RunvV!7O3vI5{FYPYc*spA=aKJMX*g1B)D0`k#=Mw%?rx4EFW{T<`-}Y3| z%x#RrIaM*C5roup?7HsJkc!}m(OIU{EQv3CBQcKu7_OG4x3kGG)wKB0vlu8=t}SKf zw%bIc7UE#mKfX~q3@L*Ol*`Oid5gOVm$a(roMRRYE9H2j|FH?UE!#l9StzTC>IuXp zBwUwu3#@<=tpD7yJJ#F7)gjEOTY(p7mX(J3j@a3AnK`S^;_ zGhgk^gSeO-SB73N1F5qk*2N3=IH;dRmPGbh^di(uuDui;sw8VI7gX57imh0FKq6di z6Imi|s|#ylwGne-0>oLfHFyGI)^PCl2OLBn+v>f*rx)0%7hN6&RJrA|a!p0lh$|f@ zZJ{G|jrl@j2n(G)cjV{WGStG1eBu@$Eqo;WROPmRE5?xu2J$t)h7$om+xAj)Hus`D zr(VKo>du*SS^i<@qy;R^!!jp&GrgCZ-BvUXB5}?hwGkfCeR}_~`;;k&B2eyG_#)+? ztq_!}j#urwDTH?Zaiisc$h0(fG2z=C{wD(70q;8S&x={?PsZ{M{A&2r7K1zWyZqFM z>5A~Q)|uX2e*Xhn-RHKO9hEa#4*pS|E94)fcHI33&F|Yvqn2itTUS<79+s)WO&{A< z9x^Q&;=B8Y;E4K7`#Bm~p`m=fcbksT5?$6(o_dRM){GlKP+JP;i|^>VTK(^Kq~HHklaN&z>M!@vet$tSd^G!S zF8HakbRt;kJREV13t&vYU&J0`Cj&9Vy;&9SKK<2mS}|0m1U~UYgEXHZxFyB5wi25+ z$w74bUlqfi>ZopH6t07mAcWCNG6SbSz{vf#vXHLhGVn>EJ9r0mh5@ibyWg;a*T7;4 z4J75}TKfR0A%jr)LYBi@g_X_!((G%n?a${0FChSCaBi$HfwbcLX;{%q#+EhDSlnHD zfEW6bPnPp?5RXP(h+2|}D!K(x8kiFIW{xAq5D}nnh6u42_`}Qx?zjctp}5)q>gJ={ zpJdygVCg;ufsdjM6~hcaC-@YJTi$1YgPM>aPP!11es=}Pb98HYdhC-1HswBcxS@=t zr!*ud8?S-D5*mQZ?X-pfF(rcp`a+PyTN|RfAyMG2a3?z{K0GXmP^fmDPI?3Ru#@4$MW%T7f%A_6-_LM()h!(zkihWt!RP*4DF{vf>Q<#C zQ2vnnD?ZRLW(gl4sRj5%v!8ww?4yAIvBlgs7YE|^is8_5qtox3Eimrm8UV>&X!cF| zUvU?jL2=f)Y6@Wz{%a$l>riV5(M z7KlsOz+`zYjs)g%LfCi4+-DKr7Yz3)iMxf->`UtFz%=y#f8^R<^{c0s1oZ zZ9_14Nn?N_0c;w*LwPope>DvgR!>;UkYb^z2APvm#c&RPbXU)~;tvu-=Rq==o$d*){G=T59 z?|44|7EEP>{vzL~hyw$Ab2tH$jqf2_Z2J!Vw{np4XrDEOf%Y}&sMbzGOMzr?d9xyo$&Zx`-h zV`Dm?TTEs=IKL9ifqS#B+07mgZr}so+=AawZ?3!$8ze0ABtHT2`$i{(J^-4{kRgL) z5TGvvFZ`*fzug8I;=;DC)$b4uM#8-blq0;+6s6psAc|hnvQ(FY7_sdOMwe3{K$4?{ zHNsjI6G&yjaDo!ZyZ&~6dCShov|JZt%k$3n;Bh`M$1S)R#jU)5H~wp+u;#$)2LKYT z8;+&KwH_e9an9l#RT|>Nu&>(xDl^~^2KK^60^0SP{OWmCN+4AL*RG(!w}$pWBg3ys&vFuI&#W)~!Lb(;;Pr zmx?EbT(gxuhrD+0CB|GnPq}(#M`NZ+JAFe3BQS6s8M+-e?p1bU>LiOIHdLPO`4#&G zD(kb*C@qZQs^ctYAd#7>BJp zJN}Nkb5&U|ed&YRdT0Cc?$+;XR0B_l?U!ZSw!m~(cg8$I*q!m&f5J99%AGv{Q=Xa) zN!Nce*BnK=7xG1?0!5;Bx^j+X{AZ2s2#Kr+>|J6PY#E!HNbkA50>$f)rZA0Ta>t_@ zz&Q&8&v%>c$UicsGKyF3w*Uf8qc0_mi~rfJ&TCscj1K1PdMK`8uX<_fd+RUZOTTYQ~6)MU-q{5`&%F5@o|OWTu?0R-q)~d{o(f?)Zv%# z7#|__=e(;T9Ad~YzJt}enFXJLXw3L$d6!UhN70D_jm+trfCNyI-e32!%Ceo{)uM$ViQXG4`4t3tH1Zr2#Puow|9{j<`^EL`Gd_w9M$hV z&+=;X;rpn4$=x96Qk@U16}y5P($y24@6BD@38`DDc6wxu+o=jN&>#7V=k?8KCUqKg z!0q4xFd(lGzHDe%h3?5Mz)=_c1fmr#u(qngOSrI{dLM(np4o; zc%qz{NgdlF)Ql>%)t9aL`=q}0+=X#pSz%UrKwu@;B|YfbnO$3ky?yfOz@JfPGrUw_ z{&rQQ>vvw8!Hw`@zI&CVP3*rWvtMtVfezisl@uu7rWM2`T+rvk27$)XerZCe{P!Hy z6(b@L_`h@OvU%oLyPC|`1%=z)Zy%*S_!2yPDl@+#Bhyc|%P4;a^_X{5!v4(>&3jc@ zA+UJVAtY223Y30z<|q)C&k%{kdrEn&`^)^~aR6~$B#AN*!5tioX5NFNK%4Tc%i>Qm zsW|kijFyxbRDS>b8|5IV5lM6wNcUu0W9@xqz#uuC)^1?DTvu};+|4{u06B4CB6UXRVZ@Bu<3o=-S>Hk^I3^}s3^+gHX7~t=lg{$xr>bTuHcr= z(Uf8bqp~MPM~wW|V9z-?$I=q}UhSY_RMu#!(dq|uKbA)H*d4I$BUif5lUC|O*KO>> z9qW0*K_%aDVWI2>b_8#H7M4*miJuqqQwH=d{^rLvMKNn}I%PHPxS_H=sfbF$b$eoO zz9ay5{cZ@Mrqn__Gpu#Skm}@v&_NWvmQEb2uZLKcmW0*eUo>OMCYagDd#dV6>UwU? zq0qD-3Ml~*No!85ZhT#?o-8qKqbi!bXpVPlG&>)HZatRp))7JvQ2ye(gRtf{@20lu z`mwS;H^|aa`07tqb=L$cr7N)#brZby;n+s6-0U_oH$m;*^9yAagUR| zoN(_{ybazGK|L+Op!#U`RkJ4G0|S+9OZ)L~aa0KhMm(lmM{nvilCE?{j|fI34Gq>w zOT%}YMoss|kcO5^Xivf|-|uw!Mham%oE{@BX1X$KWmXc#_-^^Orjn9WG*5-9UQdp6 zuHEIFxb0YqM`VNa}YVi4}v6IxsLN*w1Lp~FJLXkdq%3p4tB&_x3By;re|o~;!aSmh!T z8EY(}ofhfBse^s1F#OEL937#y_cp}U9jQE`b0Ha8ZS{Ih{k!0q`aSuvqvrVA5EEIy zK=@*P`NZC3o22{#!m(j&X6MpA2kL`t?-TFd5g|20yEqbxq{zGQo!D>dFo!};ar9(j zT4Uk)cc1)EaFQ3-KYLL$i0$-yesJpSM!eWE^NzLIK_vL6lOBY2?f#eZ@^*-l6-25Q zQmPw-q!D+Iq@f>M*4r{=pkvEPR8v9ZIEl8>WdDn&l1jN4qt13Llq<=#6YUcvsaaEk zixF4HJJI2pLHE1~*(Y#ejbI65)3v30bSb_kb#j3LiJZYn<9s8s@I9mu#umKSdw4>kwB3SmDKt8RNC>6%B9PyIp|KYE^xStMuL1LALAyZ1?4l0h^&w%UreZ zmj`CGIE9YqQKGMqwOK_qk0zx9WlT4o&?11nM1mM8sc)20(6>7wuKsc)cr*q1fspfjaP zQu@#5yd#$uV`U_wru4m1jW+|1@8!z6P-ed19m@hKm)Qc?V;^i;bEf< zd7-#Vg%V_{0S}8_idoufvZ6H5#c)p|zi{YjOOZzwbmEFH2vQD zuc@W8FSV%(9dWKMg(f1zdkc*@U!c2^+MWT#dryst*VBFRsY4<~?VE8hWuNT=uP>Ym zsg=8hr~4pRAI!a=Qlr!Vbo8HT=lG*2%GK}OWbPjR!WuhousXx)sWofz<6Kd{rrOWF zaoybo63>lyDta79D^#fX=YCn!I_R2zXhypDN%1*V)>X6S6y&-i0xRzHSeLt#Qv2n* z9ZWwbX)jXX`@89vGTZlc1RXm`wT~VjRg@SDh}cF#HZ6kv!&o<*zjx}Ta)I}FFW@*Y z2)_H`ts=JDCB}9lYenz*-|?^8F_vAWudHD_jvxQ=hkumcP33(Oru3*ppGDy!AWK{R zWb$a4R{J^aUuNi7^w>bdjS zmDo;pq2SX9E=kh2lT6jWde*AR*Xq1p_SXmiY={SCUQ9_q9==sDZP1Q3uTWe{Y>#G+xjd?gd%tWtPcR(=o0=Vk3KOe-%AB{j+ z`1C!B&r1Uuij2KJr~<&yOD5i?o(3jk10yyI)dUA4&X7HmqAUKvv!7pxM|4JKqd2vk zKDguNSGUKB9UPdgc@sY=#?&x#&0-Xz^Ea`!drHje$3NVw`rI0tQ-)=iXL+LpZjmHgq(N`k$pC>;tDt!fSAwj<%u= zNt)l-f0y+XUj5b3kf4E&<@vXDacfPTif^SHQURTx9LdFMDxZ(_zfo*v8`3ZDyFjEE z&qVYpsPg~bQs3M|Db!3gT&vBrVt$0f$gJL$aqjNA#s7-qeJP`Y#6q1E-_Xu7CmyN;ANDy16L;G#U!R)X zB~UI3B?o|WC})}2T}L3f?|yQEgt#aC!q_q@QwmF#(?4#LzA%+jvE$@iLsI|ZuzjrQ z{>PU+O-DJK(Cf9^x!uJ<>h^eHbW~O$v4*+S+NZ%9<}-P{A;-;SLgkd$^IzcRDG3st zq%LL*RmEHFU;9pZuQ%S96V3t*eGQp-?M`GTt-;!yQ-2~m%-?YWOIoes=rT^VJv{*Q8m(&j|Q_ar!1;c4uMb?GUdUn{9!331v6c(o% z-51`-bORGc#T{wWM6)+Ex(f7vG|>yuar&$b5DxZ;TYRoseV6NyiBQYkd4)QC4JCLT5E7a$cRo(QDf@3}*R?e$q(+#`heaWTmR6nUmGn zHJ|d`G;b2;DvNS31N_5hvOGAemp%RUS)FgXQ)VbeXtVY0L{+v}_s@)7P8NIVccjYR zGf4wN;Jn@qF;N&<-a9Y)Q>m8i%63KGWc9!KktNgL?-xXZ7hE*7FQn#B##ph%9ZrnO zHJcu#{c?;D&yr&9+<42#7`tjG;yjZYV>vRdz5f8^TWWr?=WwaSU;QJP1)0885ViDkQ3 zLXSS|?cISK*2Ds!3dyKT=8FcFQFMsw(^YN^;8IOcb_SKqE=!B&Um7+wU7JW=9k&^! zf8m1kB$p8aPpJ9)4QRo|n(7gE*>TKh%CmOea%JzygmRz@H>}9Jr}0tu(5Uk9iQ23H zQsO6lVB03|89&b-A|=Uy!-)!EL{#}UyhW+6Q9~6=m(RrzjjaLya&~71rKgX2`Z;baVgAK2~ z(6K9>XU=|li^rpJR;&Z*Pvq|>iPz3)^{A}bNv=4WdB*?gR$bbTIJc36e#wsC>$hiG zyF(3O@5$1ysO^P!KR%V;hEVV+2%(^%_FCrQuKN6E;%1m(TN<(H;@>!J&1}_SsfG1a z9SUB2EzJTsozj=zsZ`k&OXN=5KPuG@CQ(9SU+Ii}-xj&Z7a8UdL{+Ys$eb!r^S|Q~ z75!n86BW{EqRA9wCCgG^d;K)t#V~HhIF8Ax=;nb2hvFEQW0x0xV4b3nUS8uvMz zI}@i&<1!7^bU6h%W7Dttb|g9c%Ocg@+=`L9zPPrsZrguU!qcbjM8$`d&xgmyfh=hK zO%#G8tfDV+?g(}n5YsnQMPyH znkQdf8w&2Bl+1_MP7{0s$a{P_9cr!jQUZSp2+W8Kk~7dF?aD%Ovl)k>^7?Exw3{)_?pvHS|{MMPCTKhx5jB!-I1TXsjiHB)9Kft~4#|wcRc~)mmGse6M{aSmG z^j}oH$MrrDTbbXa7a+t>oEX(Yfxvg`J|LV6F{s`$CzdD+3>3N_S&RS1kCyFcDQ6e? zm_o|qM<9^NFJFTM<3vzHJ_xkyc<%u)0!pX+c>IrVsf2maTC_bvIte=qqay8PFO%$9 z@5O<@iT@8h&(uhh2U4@51NkNb-%7sFl3zw*W$}+Qi~68ja@G4L1s@Br;MIIgvw@QD z`nbrys}GG6p0{nBq$42F8V`X>fd*vze0i(-Xh=adXT5}{_w{o^&d*y)r43QT%DV+7 zji@P6c{>2!p2@l~30r_f64BJkMeqYx;zL%#KRPlq=|&6UGO?J4t z^q*<>}&5Ij;zqy)y;=dhn9U+ z>X1=hkM|-$VyVAyg+hjYNChm_BPw$xYqyQxc&7SvZMh8b|F$7vXPVnmI&)iYH#a(VzWvTUXjQ*#5t#jxDfSQISX%lY ziI(sq)Gs+H9-mVKFTV0g?2}}A5yrN`S2@(dfPTuJH=Mj9(hHod(GP3VnH~&KBNb>C za{+{m(c(=xRiWt{6KbYzs*ehz@bGE8?iYRrJL`FCs!@ml1;8sEpGQ?#cQ_0!tB(@p@K{L_>62^2* zCwCS0X#$_h0|na?s~7shAyil?XOa+U;0b~B-6r=Bh^m_=@Fa?MAKO$0I2N^+zBFqw ztey3X*B6{C+4brl1V^!!^ zv+C1!!aBoMCh3WW9R*~Wx|GK{K)EQi%sp~8YoG=JkEq9rG_cJP{d|&HBx&h-g{1-X zdDrQC;ii)f-Kc{d-BOTEV)aps4A73};_&mw%m9vZdB$(n_ham-OM>Bst)N6hDNDyp zUD~`+B@srv2*~*w9?w{IS<+Git-6h;O?2W?Ijs!vV*C;V_oIs;1PxH=9Gx%Z4TFxLtG^q5&F z$*@~3)lhW1Hf^_7330Sa8gah%@awQ%<9pf$S01OB_a8IYI%&%~V-|+1rZ7?s^~5Y4 zIT)=y*(91FO+)k=&>XQaaB)`;d z=MJ-PV5S-hZ`c0Xog_%$aVvWM?}=c^mx|filS{oRC?^a8{mWT){$pmC9p;|8lqJ;v z$Dhr*iGi~jGQBelO>$4(4u=?-5%Zi-mL?gN3YGHM<1t(7NvaQ50do>Ful|NX@=Wko zgD4I@150ZPAm^C!P)AIn*Z)XBJfZ-SljeGc`bM|kiA(aC4+%Y3Jmie!6C_nhOPS>? z4O8Juoxc*7x_eq}=99d?aUAhcvs>#uXC)e*^}v4{WT!g7k{%i0AY5ucqGnRI9n&;? z*Z=TP`cXJ(DY=rR;YV0vm*huO_#@VEo);IKd9Eq^rDu!8M8gV5Hs7!`kb_|^TAD1A zBR7=CUI|Pn76CcN9_46=?bo(bDm|p3eQ_TF40EW`H7y$Nk2Wt#mQyfB@8_CvxYcC454KeGU zI|4}lgec;4N$)j=g|-HDWDV-bFZRJ&sFZpyNGP97T$;xf(Co)x+-s6YFBI$ zmb0Mey<(jLI?e%Se$(V$>^QA1I*>gv&d%PukR)*_^Hs878=hFYK~}U{*KPwv5_`r$ z<(RtgQ4D%eIl=i!@Kh2vvJz+=uLwP@T!o~X#bgShNXTL~k%W*zUWx7IH;UzM(Pbl@ zFUaZuw{O|Z+~kOhTm7ySUbF3p#^v6v$PC)Uv+C>zE89Hd4H}?%xp$+`g3j@ZI{Izp zY}oMR-$lfUnvle!K?swJS+CJBSn%h7K-g)&3tYn70^B&eefs&Gl+~pK*U^3JdWf8qr7R%RXx|- z1`?o(VEw#c=rM)+{*I8u_yrBrsKb=Yov!Wu3u#cQS22x53C{{A4HfPhLCr<5W(0RA zgS>c*X4oj)HNJ@atXq=i37Wwb3B(;2>JW=F**ihnL18r5QE**2w}3K8l3~!5jW?Ma zsO9|=NuIb@F_GkrQQnJ98}6BQODvMSQtxF7NgAWP9ox+x97}>O4v7L#cN zdTNW*6cR=Td1*GcN*c@dvxqJ24|gPO8nlDOrieSAxkBm^M%igWKM3-Ug8}K>`^q3{ z26?u$BN-A@0>`~zTq%+>26>ej+|M@3c#+f*H^S!aVlgBUjFH*>-YY4`X72#!n%-Ur z;)6!LLbKZDr>Csk0$Q2n;dPf zc7&cJqq~xmCAAqwAwQ127^$BmeHCE}*l;>;iZ|+^U$CO6dDxXWyfqO%0v+ zl!js}F_%Z(22rFy6R6-JV#5cZk^$cNp=8dT(!%ZFwhF~?TVB|Z=5>%eGfcs&XCvaA zr2&6!1h%CPO*o&1(cVC(*5IC$PaM|V1%YT`A1#`^GT@ig;q;8!_DZEbUPmw?wm0fE zcnIec@b3r+2ri}yim)MEc=)*R@SXQzLNd?F%-e@Qy6t_od`6$ldK6cF1JyS(Tl*lm}OG zS)78o7(7eCy9;^vTo1#h)Se}+*Z0@pp(cO3b2Netv0Mk? z0Gl-bZ*&#p&InV`mp)L|hsVJ1M*SZ=h8lPb|3yHEJPCQS*L9yIv4r8G1XrWwEyjH5 z^VxW18LAO;OJY7t<@)c6Mb$@-v>17eFpKD*C}(!}@)rd%eJ%)4+QDxu@8|sO10?-D zms=Xe-e5L`&ty~F9RYTad;Ua1VT|Ae;#Pkjvhr=>LR<$;;8iw-Py}FzP~F0S_XqO8 z7-k^xYF{965$Cbo4NUq~$H$16KG%UttxGH|oYNZCvMv0YM}XyA43#C}N1->c=>gQa~Q!AjV|+MS5Ro?#G;jTgSyw7e6M zbGRGh<%yz?rZAnn_n?HeSXEm_jn}n0QzW2jm zv2%r_1x4Yd>qk&#c%8B(dm4#bIfSNHvB=r$BWON+O5f5wh2*SE6|)zm#Q8|f_f5-L za=DzaZvcf|uIn|eS7-1z)UyITVzTw{`&z&Y&W;hY#a7-CgZx#2x=v8<@Xt%tk0701 z0~SDu3W$$wCYL)>v!xM9{+~n*5t;8NN`~PDykD$@&yWEuY<;#yR=O^v(UXpFt(k*INoU*SOSsEU(jf^y_<%v8>u|zH4}0mjo?2_k-_BYhtlO z>BXskM7hb1CtM;;JQ7K%=oz`Kz#5p#Zqn;)``d4DxH**YZuAp@Yd>~2e~hcu1wTFh zj(rkSh-!TRSVk_nZIZA`_LqP5+`L=fuI>&HK)XaLQAYVNaV;f_cInontcG^F7jjLo z8ReL^O|aOFbWX52WR$jWHqvP&re=}Vwv1KMDq7xv>K-?9=A!JbkEL~QBP?vRCc!(C z##8kRV?<2WS4ueuIb-8PZYzMx#qxN3`toqT;-m0(tg4ly(BHIF#O+wgXnkv}&haTz zYQtf&f6EJ;8u2~ENz7uQ@JzD?JNi)l8A-h9cjLu294)!ymF1^;caIj7n zeZ_0{;x(&we`Vy#ru(5|pv5G<8C(AW`jUyzjvZFJ&aeh@gB_1Ws_^(=-|H;TGj*sF51S0#A{7ks< z_4-m#XW6uOJ}#u;0QuqCBj38_2H zTnD2i%~j1OmVaKo*ILQ^%PgXEl{Fzmz} zj+eJ&(L@dw6Li1)-E<&KVFYsm>XdAsb7tbI+Yqsf*!RUXZxTMjSJPPEJ@hfg)|m(R zSuV@^Ad27?B}eZcwXf6#Wf>8FnE`!6X#Ck+Jke1 zqvZi1S*86UaHO$|nFZtZIUu>x=R#7M^WYplHCt^HjLqjYiK{(&EUh*g5>l>7lX4xv zrcuULDlhnm2wodptxn=&ob2X26!73pl z=E9R)^|iy9f&?US!V32rOF;LA;VE*%!0Wt8RDGu}L zfxmA-y-0Szl8KFAd|{%1Wa<1jY*IP$ z_eA1&oLM-lfcY5>u*5A}$4lc?S!Mp4sHTaJlTx;ix*X0zUBQ==x0QcDPMG4?a!DXh zLsO&*FG7*@98oYcbkUR)Ug;n4Lu0^9e*`Wwb`;QWo zmJ&81WRgpdd`Ea`NG_{RfW{ zGY6fkQd+&FN)hjjQq=55$`|riH9;k5S4lfK7*LF3bk}%q+Eq5G!s!FIhQ0eg8f1r+ zUu<*dV(pX)3)l8qqydIh^7~Zn@w$3gbd=dMHs+(-E`^CpM)r%db>;d+M zlNa)OD=~K#pHUq~0fkV=*0har&2}t4q~WB0TN!*ePD!#!Z5{=@Ddh?9K`eVzK3yOY zFeclU96UG7k^5!RotUsauli?V#e9p)hXPdXw#n&PL*C;Ky{b`PdHnRkx}%Zmf^U=K zsegd03?)w+@gJq~B2J59BuP`?8HhTGi-7*{rHy4=nXtwChHt9ErXO-ihGY6YcCvQJ zuL1cgR(N~OAk4cqI~g_>i8ee>aOL>r6_|~6@xb2_(pXuJwC=8KptP6#eR$)w#5>zX zWZebAhH*0uvH*T#YBZGG9r6@~j3lmEzlwe~w-c-G@ z+(bpWR%(yzOY1U0J}mzBKJvf!4b`JMN0}V&_tWcKg*t=yQ8v1k%Y1=t1|E4!lU(#k z9(n)|DhE*BKkh&)NyILnzPJ1ZXYOBPAQeOT71nsd*`{3KGrRKzlJWHY^VcZRt|%<} zng8rOn1e_Oc3>?8L?h9iQLAJDQnd|i2`!icXoXGim-D;iV)!`8q&@@;q$kQelw z%2SZx^!HQDvn@|m3hDbT8=aXjG@hS+yMVX_PM-{ojGq&tnX5H}!B#Iu3OF03)K09(S^!=JFfotKX@Ei>0^+>yRVc zO6ue_0%OY0r`Su$JF|XbL1U1xNmIM`fQqWT;XO-iz=|PY|kcOZ6dSF+Q z&r4i8FhljGjshypvtx|t+-!FC!Tqz(#0EHcud0f+z(~~1wfhOdu^qOtEjooT)whZ< zI*2dq$r!6dJ0q05=XTUPdOR8y&1WHBOP684Y}a6e0mDGYQ3I46jj!haFdUJwf|QhC z8{B^dl?4ETcHU1fRoOSSEcGqYj9QmRBA8Gt7f{pY!DX0vC=f@WR%hCkT$Mv*=y zfHhOeh7-g%$ygp;-xyoA@nQBi<*!Ko;`qJ9k+x?k@k`@C1}7+u z8eL`O+`=>FL+FEjopvM|ka@U)VXVY|b(onxYTvHJbf(8k-oWK@#$+b$jIHHLkSlNm$kCH#fDj$ z(9XY^)1d5^i*xi?6r|_LwuXa%{)~3Nq%%=-Eq}G~payxfj9eDQF$XnE{^{iCY+%{w z&S{@fi3#0hkVE|{DRXS5+)_P~^QB);_RuLklW`~mn7Q@tUjQ^zJG5|hGi{(hoO~Ny zIzN*`R-Djlmp1e(?2(#_Gpq5dl0_oX)s~SpAnx@e(ab8;3PB)e2fZ#W`VN1-G1@E< z+x;-t#B?(2Wj_XHkVY(-VJ+^avyk6XPLO<0x3Ds2=g~f1KDqthcsgxWZ6A&KjWf$k zrr+W%rdrFTF|q7u6K1dj+J1`233N4`6_3!PbfZ%8|UFs_$ZZ)8Mi3G0}z@ zXgI2#W=#tI`%&uaf~f_`KB^M)_i(y;SSh*PGKvez;mg%m%o&&Q+v62bm$qNjO6_i# zW~d)D>w~fm9w<46T@xp#iZTn6QHYNXMHamg%y2UrUO0bUZD^6%;+7{Q0e6&yi()7O z%ve=)Npfk>03`GKmQcN9k{cehC%dBC?T9C4q8{7f$&W#g>N#fnVcR2PcX@c7DLJb} zYd+b1CxZLWlRJ`kD*`2&x)M~R^lyu08f(C;5I#@!Dekdbw0}DZ1;+9-s;#77M}bwN zmL;ID%xe-17z03iut&A^1O1Zf(tF@&7JCo#YEI6P<$t}&B`h}y(#Y>%lm~sN>^)5+ zFZz!YtFP|?VbzX8#$h*+K7F9VHr(eJybZgL_^|UKp>{1aQjAM)gR)}mY;wb|zaM|= zp?PR--vG3`Q?+n`QwR>*s)1v`@eRz^-1TOM`J6%2T6BT;YUTkYgp)llNkKcR3t7mw z>v2?j3DDY|hfkm%JjMu4+Y3 zR?t_+9W1LE95c0OJC1NJuf;!ur-qDSN}{v9_9KH~cW!f>I%djUpV7C7m1CmPwGKm)R7( zga5a$mubLubA8r?tTo+u(RjYXaPPC#od>a3(uv`WeY`%bFUscN9o`by2b&C9O0-bWx4ea@A?QQ3(4Y4?aXZ7h zKbP4OoAMT$<;q*EEo-s*XH4VGc?||Oe0_Z&7vso1C@s5j zW>a4mLr~iNpzYaZgPvbbz1P$f@-!$pC!c9t8zy!3E0TVV&B-1Giz$Dfa|^jv)?k7?Xh zSMdU5dfgvsD`VIm>O-tyy-;IX0%M^ADFwrv_nF{-VW1*oAO!WGt9NL6g6JqC^T>{J z+BDw8LYb~UA9LdY`wHt?nN58-O*ia<$%+e8T&9$hE;v>~nL-Rn9>|7Y+HO-y4yVV* z)JH5iB<-Gc)f=urq3iQa5a{<+Ug8U;5q#2$#XTOzLLYN|h0PN1HRw?9SgDDQF-^3C z*U86}(gVID9gljFE$P~FH+Lv}Jjs;uONLl3^|sDqRqX|L>Nju& z!=#Y2#) z6{aCMwYKBJ9lL7-P`N6;0Uy*&Yj3`0+Q(WxN6dq2efb9!7;Du|QGHLm<@HVPW3;w> zqsk|}&qvkt?!oe#|q3a5E;esi%v<8lS^S71xn^5zI*K z;08d-OP;D$`7*Dv|4FQ}ZM8Y|I9u~1zhbl0V^B-?m2lA3y!%2+@GhC@S^~SAHIaYX zbj4lD0QH3c@fIGUNeJZPAv zWm{M9c3j5>5HBifY|Y2-s60IYu7EMzP};Z~e#i6QtMGRp{(H%8)pn|VTKzpi^Lvrz z_urb|>ovbiv)@zXcx>Oxl)Wt^?KwBWLepbBg4%|r_jYIIUprWru(fo0Hy>t^O)%&z zvZmfc{kHKnVmD7_kilY*{q+Kc(cH(-E3LV|sj!<;8m#qTd?OdW_G}4rH#aDr(7}y$ z!bFH@!K~bUrj%)bd&HbpfGb!}rTk7_v>nsn1-$*9g`{nR_t}nc)8Y<&o&ASx8g#Ue z0lfsER{`|Ko@TIZGkNjD+hFaITE+$K=AoLwofii;ucu~YhcY`k3?A-%u1raHHH7kQ zywJ2qOas+!f@dGj1>rYY*aETm!Y_GT4p*b@N?>C)UhJp&!qA|_G3#+`jS*++yKr8x z1Z%V0I0ik=Qo;$*;9RP&!CB35qE7*i*<$vzrsPO_o=uTLP}bx8U1ANLIg?x^U>#uW z83K0Wl=$Wh=OQAsz8!UF1nAI5dr-Tw38HRE#+uUom}t-hPdvMRWUHFRz_WY>S?YLW zvM&#Dt@&Iz^%&pRrm7a5!SMn)^|owoWHXvC78yy1PVZGk5JzE*Q&Y;BQ#p2(+omkd zx{G_|`t22tw@mBm6YB5n^xInbq#9XO^Sis|_h8NMO`6{aG{1dMslVMcztc}>$0r|- zp5psrVY(4$X3O%?{)YGpyc%XrS3Y5YC14m$Qry3fURd4}vXu#sxRTC8DxB%>UHIuL z#eM+YwI1lMt?rH9+W6soPzpz_hj7$}rbl|h#!4txjvml%!d=S^1LZ*m`BC&+KIe9e z1#dccriwPgFYaz(iJ49OU93Lnf0h1I58HUiwg_AMU9W7x($| zbx$eK@lsp~6`80!$!8Xlc8mK}*+AfiNX^Z7Qeie@O8GWZvsHP*`Xi$`r5E{2j6K`B z8PNA=PTuChTFY%~Ah!lq?Sd`QuN%peY?fF2uv1U>tzMuqgVB$c%JW%@kK&uE=>>oB zh-aD&`El(wdaG-2b-09z?Gd_ z#J;0q-h*dMShfWF2{Tw17Fb0$^j79?!C~mHHpmyOK?2bcHbqI=kM8XsIZ`2JEZAo9 zdMS|@Z%Qe3mK$8`KV|hU8lg=zj=8BkIqa;~|H%CfA!*No+J&UO<+4e>>#KK0`fcMU zKios%TpqK7VOoO`6q_R za2**07i>W&i)cYXMn|cra32+@AT1y?|Fhh;q%Dl|{eQo|`Hbb|t@oaD?%B_|p;V{n z^tt_%_-idISZ*09!Ss}~K4PF}SwPPQpl4fh&(vqn1&*rUZTWRh!rxb-&XT{}@ZS&& z-NLbgrR$~NQ25$y(5c~;SEB>jeHk4peZ8LGsp3(bfhL7`K0Io8tynJAr80b{IQV=A z{zl7tgns2f^!#?%)OMn)&hG4hp;)7+I7@h?TL|+o@wQ{gC{w~+oYEpY3nvvLd#8e2 zg%<8j#4BqV0AFrGuUQ2;57A`g;PPd_QkuwrSR83q!Zi-|A7-Ne@V=4gI}8raR~DsO zenWN{!eG)=98ATeM3gC!YUcB~hO00C#^pHnWsSWpe>aRall^>8tV;0nWg31ypfj=Y z6ZZ3Ek}f;d9`D&LvfyH4i1Cl?Fav(;+j$q6=dwW(Uk7VgTv#ht-??`E^624%x<_P(~Kr|EMBp0JSClc+sA#PIn z_8-ljJ86~7FhS&(AUKgsdWeFRq?sSef%%{P2@_8i&B6I?E2&ReV$f$SPjhh{mjb6m z?FH^D!^Im*x!0G%i7r@{5IEC5o*=Q?0?oDwhi$>#*-N>9O z#apMc&iC4>m~W;;R*=81S1KmlM7~qP>tOYKx*2F#oUs(gxNJ#(6#F%M6;H|u1Mu_T z>{S=2vBBAUmGr@&ASA(YnW_!WRb$iU5kv=ry+3U;9Zs816gAC5+mTFb;9n~Lq8&|Q z)2b=xpFVgb{Fye3C?I{>D*3O2FHXyq|5iHDb-24mtsCs7U_;&DJo#@W|2qxSIpv|S z4l|wdP@6W=NoPvs8T_h~{+UPP>MoX$PI!zo;sU^aTgYypF!R(^yb+5M-b$%J$!c^l zo)VLm^IBeOoF;+A*Dk3wSUO%9Fc%YZqStyrm(pg{J5j9?;ZF@Wv$l+0|M)ljOxAe6 za`$KGnE8TS#1voywhb#3O~x{d5KjA)M^O58E7jN5_Tt>Za`8bg*4N&Fe<}4U5IsW- zMnM|u`bRUZ%f6T3Eh?DzbG#)42w*{BK7>IdrxWEL;-Vj*4@+bH2*(d2hHvG^|B_7} zsLyEZA8lfIxyexTlM~75*JgT4@Gw-9q-wB8SK$Xr@k}h`pBLj;`z`%m?khp*S1DWI zOE~kD{yO%pkViV?aN?7+q$o2e=Hj`eX)I75>YcwGS9Z_so0O4v$j#J~#R`7<+91asLK8AnK0< z2tdO%4e_`ico)sy1k2a`QV0OG43SvMU7hjehPNT*ZWo6THpB6S$2%Ow8gB(;hZn$Q zIX<3Ez`+$JT9z_!zF1GdxeC4_W-o37oN08i-;jQ?-^YFeP9Qd*7;nGzBw7D(l7taJ z0N}IOf)5b;T>L3~^3Ayb6(pH0@!8o#VXQEwtf#=V5`Kw%qo_&0Vt|LlPc-E^KkBy+ z!t&b!7YR0)j-Z7k4P4R*Ul+>dUnhKzE|pLg|M~;dwd)EpY3G=_kV!>aK{ouv+;imZ z;^$8oB)`&A0|xemIaYnp1|*Jp0?9>vIfFdAh3RGFi42R96B!o3HRje3AX45MLED6Y zOc58J$eD@ce5Le6RebSYYaCAX5dB~=>W?0fMl15}uTEj!ovjZ#NXnFyuXR-=v?Vl7 zVU@n*kjxt)QE#$}20%Kg*)9YWtA1^`#oobEY|zY#X}EcAN?#f6CGWReJaPg{i~COC z2~Rx1m|*{uXNRPLM68j%W$V)dKE2}TpRP1N4X1J)QtzUTkJ8m>zx*MXlv3_iB9GRS z-N2bgdomcx-(K!yDVS0MMwIw2gU9$)29I$q{Hsqh{i&bw{=JAv83YHi(^7iEqk8uP zybJ0)2YZ1XTk-dsURJfX0;bRJ!%4Jf0m15be2E#a7X0;FFmF5&9~YYV!2EwXwA>7X z5ucBPn+YpP^j&P(UjJMtHKER5SSJ;EtQXl%4m3_Ku$}zYcKq}@>$u;S*73nFtn8pm z>#gI9<*|O2?IkYxpnmCk>txot*0`ZRlRo9wHkye=qz~fWG%=7Sf#_mnleiUd3aFRC zKClk{S%~XkhnZf4L1$BaiZ@vLlXM{!y^G7SVg>!&EiR^?!A1V)5I0Af;U;(u>JW_| z*z=K0U60Z!Lcgl$dCDNKS6f?FH0;CmhK5h9vRrJq80o5q7Zclj%ariH$g^DpI(eyE z{1a}?*`P+t^T@H-ptdyoqj|aT#(KZD9x1oC8U>Mcx!dpiTnQgf%jqP{aZ1ON=-OU| z&-m6?{P1ZO%@dYU0No>E$wCyXh0@?c!ZRqfmi7%Q1&g2()=HkV9`d9qnU_hc>Cp#V zWKv`?_-L=5=@FMh`bfH-xXbCFEua9F>}szn9R=30iWiGd0|~2@j8E8I2i4n{h{;07JAl!5d4^xh}a~L zxUy#oMVAB7u^9nhL&XFbtLaa~l#+@PVnWeMH>ZG=yD^=l8zz%nEKXZYL~x#8ZVg5Z zh8%iULeJ%3-l#Uke3lwo4k-C|@WtZjU5N6cC%*TEBCSpI<@tYZl&0F%DmcE{(o;}^a_uTRJHNgkH0K=d|u+e|7kWUobT zD-3cglH|6j9l50)k=!;(aue750~Tm(N-lWgrSJA|J8``KDpGBIM2p zF^PFVT7J-IkQnBwGm!ytHJyTszM09bG8<@9lG-;HFsX^V$2zPWl@zgiDpJ>-xG;da zc8M4N!PLd+2s%-hWorM{mb{vdMe@4b10F1dGK|h)3PJc`r2cW)hbhDr zos^ODo$6azkwN}#Hz*uHa{f#ZfX6uDe6SI0n>cM~rq+sN;NZzl(T50L?(;VaUmSC7 zh7%c8#o5lp#Qb0>CbQjO_mSkG{-=g2Gn14VxLLijs3_a;>4h483$043F3N6P2;&j& ze<11ch0lQ=d##iC;m8eFy#;5IkZ5o27Q5bLZ2m%^*e1R_kM0<|K~JY zdaG6^t~-p1O+)_Z`HZ3oCKuoL?Jd~CtA2+#7)j>ycGNe2@tD|KV)-wG`jf8GH^^?sowZ+NPWEP?`9HpLN&*kZ)2K5A1_St0S9e^oanPvA~L~P zO?0Hp%B$Ixlv5Mh=FG#5+R<+)=Q8;?HQJ?)&ibsg_wA^7J!f@7zGjy?y~T=CQU5y_ zq<%lHmm3$#MRoETI5l_;yQM4f`efXH`GvQw)zM>(Tr!R|*2&AQW1toLPyI^U@$fZ? z{$oFw$O&o*h>gp<;-;Ol2$J?we-lADn^O}eRjRR%IJb+YwS-79`u<#AWdRK&q*mq; z#qZ)2HX<23N_R}C3`8fDV>(OD27k-G;@HWX{93&~+Mko2X<;5;j3%i+It=)1eZaT6 z0yH!b-~ib#XaQbr z_TSN?^}s3GqQ1~PLlmmfXcYk=ROOILHX70^ zNxslI+EKeECvM6nk{6AqTJ(^Xk2@lQwWz4tm&K9Q6?X=-)oL^!xJxncfM_SRWgh@1 zPkCx(!1t|R+W_!V=G5e6t_&_K3dbu>uHMVJsAoPRG1i%?;lQ&H;kOH#igD;P-97H+0!Vn2tVJ&b<{nVe-1b-M3W@5UEWn z*DoYeM!7MYAb~H(MW``gG>;ncIgpFO^x^vf&KB&l(x^u$MvXR_9MCfLVY~;$HFTrC zyOrw|q9NwoWOvgcCU|gt0FD-kNb_!z;Pz|DdQz*+9u36apT`)QCovSzfEDS3jd9oR z8~{wQA0SrDd)ZncH`ZBksN3pV<97cqCv}E?&ETl9ZwIUCZdB8;EGz3=(A0q}!!I)L zZ|rXKKl|1w;TeSZd=Fv+@ax+g6oF__CHj$bHUy$0%Ka@(urdJC>;2L0K%I8~^-9E^ zZ+~!~>e~bQC=g6frWSYz_4hQt))zz?P*XQ9TkySs>cv~eYRHhQVY!iC4}AFs%Rde8 zJ1^mVAK?1{MQZFRLaDF5BHdq?%*3lS&V_Z?_>-!|@M7aO&Xpb%dI!PntZ?68)cu?R zszB9)ogBel!mWeUZ8n^bAHInZQn%v<(m%w6d9Crre<##K81g+3JM%5D(@4yZY#TXN z7Nu;4i{TpN7K!2GJ6tr+k!^h9tgpn*dx>nn-Qpg$>1zEL#$_*VNOI=YGzX{Y+wkKzo(`eo=bvi5?kuvRY-8&_h6={D* z>wObSk2E5rydU1w69r-8wlTu$Z{YfaczL}5kOXZlg5*`NA5MvjxH-5l! z{6f&dG0|Bp6e*ub`$oI~)QOsFELzQoDnptCvIvzC#^@hr8_lrpwBJyfz8Kg?wjZ?O znCqKB)Kd)7XOusBtSNWk-r(0N;}Fn{xDWWg1b!at%XL)MH0cJ6@)2dgy8;^DE|&qV z0sRj&1ERCbDD^ZQL=GQ0NaS1#QU<{IjeaxE%i^C=9y$i-v;}m?30Jrn@(}qg{7tRT zH!4pPAgBNHu<1AZhD28&_E9y1pjv{USUW)5z>5Q~TB2h>ttC<_Yb|fpQ0tf}hOBhs ztiHImZG7KuljpXL2a82Nu^m&HNA5$gV_Lw3%r)4EFEy7Ii+@we?`ix`Q7CqNn?PeZ0Z9 z{kYW8w8@%b~VwfTt zZJ^`aD`Gv?ku-OLh)h$XIey<^U>w`WozSOrPerwO|bch`3{b zD(JmhA5cZp{Lx1|ev076E4uij_ffy=y|4n8W=Xwp+e;`?7hs^(ncvJ&3pbBb45KIw zYK-EhNiYXYBZ@tkt4>{xzJ0N_0vY}!53ZOV<7bgJ;~V0xxB?drtc+cLU?)^q)D?{; zsw}$iB}_P#y>I9=)fScFkmfqrh!~C>VxZ|)rv8=RI!RxM)CA`?&ZjYL5o_;taZhq` z?vc8Mz`D8vU}cEbP7JJT7+4v8RyD6c@FC=sa6R%oIMp8(G=gwgtjqC$4OQz4%V#!s z+u2m1sCBeT;Cb57Th&5%sZof}5t0(C&h_wQ<0_jwO2)@WU|tv5_mnlJf`ZX5@h2AW zb^!^}g*eD`_y`QT3oN_HhTzGFi&vh(4-JQj>6Cqmu-Jm!7}H=tOr7Ic(%He5;NtT8 zZ@K;MTXS+SpTR>R0wCJOmtG13qNq5g%izt8$IOCt6f7_uqwvn?-#CXTv_D=kj6O(# zSzrBq52>L8@!%J#R)BHV!7E*vl-ZuqSc7_Q?`2d7vIP`GRT;Wo%zjD6bQ@n{{Tgex z5-WjAU!=TpZppy+C*&QSWjPy{V{t{SA4Pz)WeBbTm-b9?h$+CUggVc1xPSGDb^P~F z4F3yp9#M{=Sl}_@Z=j&4z7=41QJTd#hZrF*Vd=2`GFP6Wo1`BW22x$$WOkeccNv#X zH0acGyyKn!`o;Gd%Zv-l8xT8T^qvezF< zkOv*rC))f;L~m{}d7qs5ThB=YeHEp7Re}n?0y*er;<%^+q35VR5c9H}f|YS5y6%Yc z?yZ<2Id6J9&dVJ~u6W)|DfOtaDWF7*bW78EN;Yysmg%B-o>Yi}gW!t0*m+1=MN0T2 z5@`-UX+UT+feKWV>7>$AvzAs`!KDAAyr(<}!w60%BiJjA;O%0`bCjSZjo>AVf zjnU}cN%Bs~h0o@)%sr?F+sOARV3zq*tvDk9-#|^^vdgtDFz|Z)M$+(#G7vTa?JnSE zEcv?`c5l#AOky8zae1i+6Fzqq))v(6LOZc5kf5lXV+r`fA6f$bn2+Rok?|*)jF;KY zeV+DjLtK-u1jV(V;*AM@N1_@MYwSPoh;mH7V~=OifBulSo#T-ln@4dxHkP_56%gBn z*qju*#nuU?1HI?uuThfhN_3yUmzKuyoRaJhjj)aMJwBZweUIk_5*+An-@)M-H>U%N z8=tit=oJA}ic>K(1Nvg8Hhs)9EMm`vb-Rx=O#>Vu@OeY zv;+j>drM6Jbc@LP3z(SCi>scM+ScPsES|S!iG|NEmbCF$e{Q?6eu))tv{id{gzk)Q z6THj~30~%n?V=@d$C13u8x3rq^^Dyd{aF6O+R&kzGDU}&}F;OF$)TSi=5=6hj)|ThgG;Se3LiZOX{x5;2^f8ND zt+;pAQ-J@dxat`T)#u4@w|>H6>*TGAZTJU>=(pG|-b-(E7;Al>3_s+OPl!}a%FSxq z6(BVboySyttC#ZlQv4y8*GWpUKQ`K%!HI-Pa2`zY3_R}%#QHAx`+mS=+x{YoMj+|M z!TuaP{2{nqy0RZ9%IP?#vR#}fZzM>XQiSkgf3(=$*iFiV8iM+(s?QzVLt%q>2^bG` zSWo2!MGliKDeo3&8=Fs{a>E|KU8ouV3OrN4HJ;$#UOc;%&xs%YVgWG!1CY_P+S@l% zeg?s7x`h{mQzsyIfh@hBzE%7fbGcISx7LLVA^qb*Dd#>q~wS zDSvgBgmCw!F1WHMc#PGC5zH~oz*y6$4rhV~(_>a(qRcwX9~=B)LZsTeiKCuCOG>!hg(z)2 zgX!_)mBy1qYD)MjblXMw!9eshTr78Ci5h-ZSu`xWxi(1&FN7bVAKhFd(p;473?8Q} z8k}7tl0ps6a1GkH@}+WPjQH?e6D{y^K`$%GMw=zd(@>-LyX$wr@WUIhu&2vP4!U+( zN0C;ioQZaTLOi{HLy*&BwFi<_-zsGqwtYm9I1JwiPiI;(%=SC|9NC8_dUZy#D=8^> z3JzL{jG<_&3rDRN#jX0tfc7eH)bZ6r=iKZg{trIq6;EP(jeX9|AvBKxS2byjA|-Mg`Oz4a z&4aU(l*mvzb9IJ+1jGNYETX3=tXN&>9i&92y)P43t4gGUFdI>@n5o{#x}XZGYe9$& z$!MO&9Mz>phfoaxXwopL>p-#TTaRAMOx4#^agskeP@GS-ljtQ2UhEI8bO(HJ6=Ra4 z@pZ{8y{P0lSqZ-fps0GtI|!I8YKXf259&CWH_{)yt`yzmpCHF+!&l!Rz6vWAxGD~t z>MebrDdD#zpQ}OeZCj6?#D%gSZDlziUTiqlM6#kIZ{_#;)}U!+$L+8G0k={PCgQd9 z&*63FEDNvXcQKJL(inUK5=I^Y33!G5*w>n9P!SE;ta3(k#mmicnS@R*3?;IjBpF2SfEb7#j!{|p|BF8$KWdh2W_Vx@hgs7f9q_vW6 zYfGbqe@!Lg^(YZv3i6|I_-uZBju#WWrTpDZ9E)X@O$2M4mXBhsUHpON@U^8BFFJ*a zn6Gp<9%sdinQoLY=2IZ;>G*FOB$k}E12UHamN#wo-SVgl4)p_jfNicm^5@u z<0Hg(|Ci`zdg`B|pSK>f=!Y~Fe2GOw<;Lg}h^YLCL{u)Lut-Fg)uR+G!R)PeS;pXc=KxJrx(ic+sRR#ggKFkDk=use`>4$A~rNtiooCEecQv-h$ zugE~o74F9LHu$TU!RsDyH#Y%yvkcrB>z_7a+P02yS8rec-Tp7vfA^F>xBh1);;+Px zKbiI|Ct*XOY+L|C3dANF8mh zWn?Mg^>UKDf7a)5Duc-5;4i1Oi?g~KxKhF|Bfe$tg~m=Kk00+y9;+tKN&siSAkx0xB{SYmH1PJD>f?vnAunKzE-Q*&JO1Gh&6mi;cC=>O@~%C8L@4eMC+`Kx z7P0bGR)P$6>JRNV{mkvz-CvVUb_a6K;&}SIGg{-~n6<3Hzndv@>@^XHMyc2aRe9HX z3{;@@deMSbY7hqeQ1z{=xLb`5q8yQv==_y7RldIB2R&th?5BtM7g5ssEjm`hV_Q(L zQyklG#<7)fI;QlSv21-ni?yybfsXI{GAR%Y2D@wq5Qt9nfNheYq!0A~iq>O3hQyA1 zuGYX$0OTWL`(p>H;WZUK{nE|UjOi7Coi6L)mVoA@u!49}qnHAwxj6$`W7W9q-Nn9N zf=XjH_X(stQLK%|RGEt1svI_Uf3H z`ic@>>u5^>_aEr+OP@OuzBHWfqFe_3r^E7PPqus6uLnDP*__0etvtwczP{bsRKf^i z^9{4)8!C!zPZwdU9BqyMVk<*%2*qp&(E77{u5lXv`P3NjnpSJ@S%jTxJZ}8EaT42? ziR-JStE1s>mNA|&-#Y%qc7B`fxTo!&?`-E^u^nG$-D6lPWTNOLwui5spO{Yr_P_QG zmN2&_q@cC$b{&Y$r@%jkWvtp)u>alu*cj7u;_^qwWKd>&%)1;Ggc5$;iPBQa>;6Lh zm19GrQf~0FmK#lpd`jG;O+v6EFAOZVm=>|j zxZm+M@5<|@9>3zdFtX#6L*1##bLP@b(w@?P>Du+avPrv>y)uko=g9o5aHri^?3Y8~!Du zb!Gbw3fwS2>+2Q2U*V!kwRLJ;0r`9IM-Ag7xlMzGG;Xl4Tqu!*=E`P&0((AkRsb8o zVE3PP{YZf|oZ{CGAv@2@*Y>Fzki)uS%-2}Q1bS&(93AKgN0P2n{*|6ab!UyM$-hwK zpomR;mQl>6)rnK;L|gmad}&knQ>LO^xf|L~yS@4dayaLXva+^Hr=`u=R@$#LgT0 zhs$9w4?=ZTAUYlhX;)jYqen7zT5CMC-&t{5`(>~(0mRE?rhm!;o)96We za}^m8vq${UlN-oT_h~?DDuq5Lka<{9Q5=1Zys@#AYu^A;_kC7T1>mCa``H+C_uFbp zj;k7UjuJ+%kXRkzOq~)E0K3TS{F+8Gkk(~^hg^yJg$RFf`NOU8#>Z^^3%840JT*<+ zJk44hKfXdW?M$yDkb@bqPFwe*EctI95qEkT=sc+RWaq@ixQ7u21u6~WZFrfoi1IHv zZLXl4pC~B7-=PLPMWuD2Rc<(2T21#Ft&iLHKd|-BH1m<8H)aN+cY_$XBCC1GC{IPF zfUi*r#~jqs8>v>S3pF_N>vxBmoWTB^`JgEK)+*rz4o(>@a#Fiu7>SG<7c{ZHR?q+x zy+71&sIXI0VW<7o1^ZQPOlF{Wqgwk%it<6xXw|oQ(s5>d^_Q1HdCCRW|L>QnD?}T9 zN#7jGrJcaBWn0<`7Pty@XgRFCw=h7OuZlHUoDkg0h3kj+d%If6qVhdLV0#? zQ*wUIQgR$CMNO)+NtqVolS*U&98eP$%r|bP@jCY)Byi?+gJoPVNm(>F1AfYq4r9`T zGyJ~qx&B1OBphMM=ZS9kufzieQYHu-5N*R6l=Q2cRNwO8R)i8xEOyx`2ra6G#y?xI zs23&l8G+t9#rmk~^b2ZyBj!Jh(fNrO*`VAYttbn`Qicu+Ui>5C42w+qr5TBfk=&xi zS?IWnjfC%|{Ph5)2c-h;hh+G*E&jS8ZaE`3CfDz8H1AB_`;7t5juZtyux5T{t)Q1xD_Q&6skhh<8guMNw#GTva?Q`4*bATF;|F81({l2gMOujzc_J5JD zpCvOYd=J|4^^a|nuTN>Qlsrj4wOmY@P{KXYorg{`nX{ikIs3=NZGJEK|9tlQyvaRE zP$5^j8O;bcCQ(Rtof2swERXO|{KDpQKbc%SOR%*7$<|11DEX#s7Jv9>7dC;Bc&_!d%#1%rSt)cXC=Yr#~q zIa2*mV4u-zJveQNnQX*NrawAJWNtA%@MsVP3;dxvz|KS7>|g_h6eGp0ywl;PARs3X z^N)+QlfC|E64rAZbBOdoVC;_8HvZD#gtmA6$W|J=vVVi|_MC~gJPU6l32DD0(mr2o zL)xj8+=FDKfhk53Baf^9m_GtyIcg2uQH_S?8axwA2gDl>G6G8wfmccdN}IEQu&7c` zazr3?p@I{E*va*_x~1E?Qr!~e-%;HX+Hi0gFnPX#$$z!MBqvlNEu4P2a9DT+c0YLm`N>tApPaE=f>^fv3FL_f?vrLQLe1^oMPhj@M;AI(Z!pD&tt9%Z~j4bPWMW;~a1 zc8RUzjDrStEF)GcO(e@zi#}yhdq3iNju*#=sJPnn)8Kb(CT57gI^-sr` zs4lmmI$%e2atEj`B~Ee~!%$X3d(&bdZc=SVa5t37ZJp!V0pEH_9PV0~76 z|NpE$D@otmV(%A{;Fnjzx5wF^p3A=ZS!VpJ#E5_4NXu-|QwrlK;eJ?4^!7rNYGi}( zLz7ymb8@vjIoG5dBhjOhdy6yG7{Q3e%pzv7c2!=9E$vBrh%KfLHNpFsmgY-Z@}@a# zp^VB(?$G_Q?6xZ-2vHs|5|?a@ALT1;NbFE2Nn$dzmq=wf29;gZj>=?Le8Of%2f%; zFVk$0{2|Q-$wF_o$i0U<9%h^R?9+E9=(Fk(z=v*h6MUF)HZq5~d;oLH0GLNx9h1et3VW{j+*nyQgiNMXwD>pP2IV`)SXpZInsfN?4is+1sZJlO8xBSc)VCU zmGqokJ%l}FIxI8I(r54GMU!Eo`kqu9sw1fhs4f-76B*T|=5hi3# zWLENOA>0GLOBmm{TB_x0Q4#s24*BBpBdw43{%gEW?VRp`EnZX%4vAzG)^|A!QpUoCY3xBX%6~W6iJKlY?_D>uIIT_(p*}R zmT?`ycyt#=pbYsACI;<%Buju43wl|_u$*o3eG<^TxMmy|MwVMZIld)MGLvPZTn-x$ z2A@jDi-O~%_}Gkhk`u%U&RS&O$>QW)OPsL$6w`0BO(?|>ER+&v3gv2EG*thL<}A@M?gmGT|1rXTI<)FSk?!}{r_YF`wi z5MaK!UPb}6LxE^I`3Rcxm8WU~zOC4yN&5!YPkv?%uAlFaZ>F}b>E8<4pVs%QVKLX* zgw^w%jc-~>5l>ecH1`;vkjZ3Pwxm7jczKs67|>?&M&nwS<$7V9NpF=Ty?Qxw1WHku zoYh2nr&^>(sRX|DxbKxzc8K~6*nc*0C%DmACE`B!%PDY`YKYl7Meg|J#;!_6Na(WHjEQ3_}jHf z8jRM2WQ~LALJc%SZ3ZK)NDwFbn+JGA6+Ur3j7B_jbL$Zvs5I0yJh4bsNZ0yrpxpDOJ z#QL%ki=D9gtbhx32^g;pb&FkpL+n#u1Z*TRXnd>O%dTRm*mF0+wZH1C1FlhB;11UW zj}6U0V9LTS?w`a8sn$<0k{`@8w#duFGS+8s zmKczPliAqy>IOKghsINVg?MmmYrJu-X)lTi`1?)scSrlcs6gVu8dpDRA?)0B>LM zdN#dTd~tniJS3c<{VDgAsIi_e{^a8Rz3N!kG}T!@Fy7FnD|=V$%5E0xh%d%jOX{9S zt>qYmW3Gt&{Ki&UO^rE_-9XOkjdC68XFh5@z5h{be*oUJoN9hgkvNyQrm<|SXA0?p;Hpo8OypQq} z_9_#LJmZy#UA%BKo4bBbe3k@Plq8~(&j0l8;Z@S3lo#52G*`C4n%T}<4hVO;-9xzqjK>$;9+dZIbgv;Q#db2;A?H(T=^QQfZJzF%uHH$=$vM|0by(qK1SYMy5l zuo{HgXFLSyEv>hH# z0X&kw1b8gH9`)0k+Q8#a@T-238@{#wXnbYRP5&joGT*=N7Jh&6QQyDi#tz?)sQ^tz z!i%5Eb8!dj4v?BaH0C6>Hn<$Km1qIGp1Qe6FzYi90)BLdh;M_uKzHdGnyeE$NUxjzJ6xwCYM}qgj5&sY2 z{lwU#!h6HWM0npan&AD>T7dWD>)ODZ^clwpOP>kvv-BCvS81|RvJhe!|1~2XJA#3C z?mott1R(u!F%(~uP~1PxsZwSC zIu4{z(0pvJXd2xrn~%#rR_9_$Gs;tXA8oUX>0Xfy7}!m6(uPI(8g&AIu_384(8(r4 z3ECBj;9n)u9+Sq{31@&Y#_f7}pY?Y8yw1bZ=02|$1r*|Bl9for81){-;}EyY<(Dgm z9r0tP^~+UqpG8=HRhK$~L)7Sbk8cAjmok2EQdHW^`+;q{rq#$C;a6aDJ$-l;7h2AT z59lL|I_UG(|D5V*Tr8Gdhh1frH2NBFT}Si< z{BIA?S3;YQGJo{agfEV?5f|;&o`)|m>54o(@}#w|gh?|-^FzDNo|!<9*e5QdgnGKu z-%R?OM}JF70`Tclq~tPj;BsQ)Ky`JPp{$Cn)H#1}l+^{wu(gT!_WAU~Mm~L5OcTba_ z!HgE$_9I84C}Gxr)o5n@l=Bj$VUSOB3$T9FL7%uvhI_CRN3K4R@^eKatUl@?CDKO_ z!@J`%V0dSiSmgTNl(zc%wWP1B+L}Kn6asdk5FN&JyLYD)3M3GocTkH$+SPG&pp8%{ zpl0%ssSe7G{!fCTtc_rh`E`~Y=zYUc<-i>!b~!Lt$`u@3PEDgm`TSK_2(nHz443_p zDB&C-@wH*Zwrgp4R3o3Lwlym)$$CcNRDBP|_vg z!+VYgoswRn;!D5)20$t4BRrJDq7`LY`@}@Ydgy)H1T65$!e$f&erw-}GgqU)!CXp{ zNP~XkD_zM*4xsl8Z0)7F4o=H|Nz&I5Y-vp6iS_&Z37F0C*LP%3pJ}yq(Kj~AI#+S| z71%3~-x{3W_=LFQYRiS4`=CXpSKE%qJ(!T64D<@pv{+kVY@o6Z(Zrs;4mNESe?f^9 z?c$}bUaJvePNzS5963rY;Wi1>s)7v6MZC}-eVE+6ow>HYj*1Hh+RnTNXUHLn3$>Cc ztcVd{=a|GpF{hjmbuu-Xl;}jsXd{d4Pw^ofgV=H9ePn`CY>usBlVBS)0I|pVowz0y zCy)tWLj6>nz80GW^z}-I?QZeoYRW|le{jD+SJD2(S`%8-v95DA;lj@rfuSZTSuK)- z!*O?AgL~@Lx>XM<{$Un?{L3G zgcI(!&OdX1hw_&AgV}@cE70iXQWKbgn9-I?%_98KK`M8!1R0T_=v7m)Jis6Q?3-N_ ztexfH|EWsher#-mJW9Na8f;7^ThPYIMo3&H-t6wlU9n32(MLc&rXfx*68DqhyHKW- zVRmZ&g|amK{No&yWffWp_QXON%^&J6j@|~w(!>mN^CsUf!6EupFwR-x+%|73x8C-y z((Y~4W;wJX)9?FL3DdTP(J9ms=wh*Tg#AS_UTwZB+~K<%)PFA$S0WXpyG^0iT`!1j zgJrGlEtne{``BB`{esK}B|X37@FboWWg`-&P-cBLcx=a0kdGq7sRgvJ4x`!f2lloM z2RLt!W`QvdYR`sCK1ISh!1Z9fHn^a5Bb-)!Xg#ACoYXnj!7Hr8*IJU_XF zN~-281Nk^~)XK1zYk}%`M@{@6)wbnV4;@vgEII>+*<{Ejqs3BD_-7^D+=EKu-Hx1{ zMLEs-UMdI0b&y|mXo>(-K&!tJ#{KW$=u9QD(}g{IhB##+F09TgqJ5f3mf!PWD{b>d zl%W~hdu4u8lI@Jp09uHVm`KAoc)=D`YSnz zO7>vKzB$W~K<03-B%lqW1rtI}{B1BJ*C9#3V|~U@m+cHt`hrQCB|Jh$h(eC zPW^^49aG}Jy1+%mn<*d3wm*_>Jdz|ISxoSRM}QK>IZ?j#DpJv)1lLtxkx(t7GXeRcxqtY&+Oz%haG6E}( z)M(!%%I`canc>|a>ESmt{uj$snL+Bga%))B^`^hn-Qi$a73;UMC+~=upWn*kAp$i9 z9KkY8B71FFjYzH8krHZfhPFCYCsk+WVIx}K4khy8WGO|4MO{r){6Z4omB|cl!sEoN z5*L1nweZKMx95b>`#|5Fn2A*rs&AJPewt3+1h&Gjs#!GU@BMG=hcsjPv1sxFfG<6U zr+=Y}g6NS9dX4il730usMVcO|`x2M(_34!YVk9uJnSslJWDpVZjN zS=0}oihGN-+p?77uyha$8leRH}5UbgTUbxHqN3 z41es5C;BaiAF8iK2|o$w66^9>!SQITbk7kq13J%1}oy4=tVI35hR zdycj7F*Do2D+{xnyaM&9hiv)>QF4uN;jTl#Oe1$XDTjw@Br!4jp)NMIAO#s_OXE)H z?Dk_Eh~=Od!NU zG#C!m0m|79$9-?*V?%iOaHq*V;lmT)!-JJYg||7YhdANDbz5~RIbi^2oxgO^q>y67 z=}Rqkfk0Q~O!Ay^{S5ds?6gafjh*7|1xmP@vaW2*R1|`WckdsvKU)nfUw`kc9(d&G zxp3DFo#{usVPA?e{aXBiTVJeKTefRWp_MKG`Y~|%#pzDC_auy>PdzYbu;RFHX{F;<(LZ7vN^M(V=Ckuc(aM zy@{@`xnxP|D+O#u05=20u@~xBIn7`|DCv1%Aq)kT@pv>*%2_;P6W#nrF#mC+8|)5J z?I}MknrrlQK79zA@#TiM@Leoji1piXJ2hVr8}hjf{OxWIE32_TXOC7`)e!K_GFF5h zDMOyt7Y29wQdE0>5J~~D&@$RP`>6$=9Mh(hr$u;n2HGBVg6veTi)9+iU%y^RLcur^IgN!1qEzB}MuY|$ai|2%4dH#pz5gE2)`=93{WzVD^`JtW8{7~!I$;B3bFVyxF zYEA0x8%~WhUIiaHbHbimdp8wo&H37x~zX*^=0Y#u@L}F<_V&)fBI}=HhEzzp9@Vedh?KBUQ3kMa=ip|nHH zPT~H*D01|BnoM~mN0GA{ov*vNOR!j3#F_7SMgOKbVSh7*V0bCozyK|T&)8;LAG0kv zHTvhjTk^0ngY48Fz6;Ei=;tr)b^+vh_q?smF;AF}e!FsuV>(;W? z8|l0+9@qb}-@*qCSdJb+eM}Lj10&(-^#`qSP#IDtyDDK&Vgu4s=wp@{N@M};woYQB zECaFZQ6e`bIV{5sNY2y7WV`Y*%Q)c*c)HGgisAYNbe#*wR9OQ6q{gItm&LUH-wcTp(U1Cdro*R0cY~i*{tm>Ki8M>_6_4@>~WTZ#w{T!_$ zef%K<9-y`OwS9WgAJUH-D^JDW{jA?<`1?%bx3QKJ>Ewk1rV|48Pyn_V{TyyV0|>OF zH#;nqMI%P#UwGZ9&{hC!-`hdQuX<#ZL(yp0Ex?Cgn^Xt0j~PL_uBfy5}cy_ML~{{C#{4}yu;Wn zG%|&@p4~2f1}34G+v}YfAL0h|ZD=d>yu;k*%CfMwFJud=HxBforUZs}8EE;!gBO0l zV4c-}1$CEbUL`v7f9DEp4P)2QWTXF|Hn_4a%;2&)i_DI#v6PSAcbWbqF?+eAjSmSk+Q%$O@FCqo-y|Q> zPwzR}`jATAbsX7;G&cLs_>czV{u!g^_}o9k@qfF-=0kEl^JjcWOE0l}NI5&jmvil2 zBz^Htj+Yaijucbl$wa3k^;Pnjz&%sk}Z!X)^d&9 zJbAv3i%qPlT=5s%RPy16-gB=Un!ahb9EAS)ZaF-C87{`yfFfDn};oi6KcpDts zC-~Sty6*|Rw#K-ohxzFzK7_Li`RR#x$9y;zUtMfPMB+DF$0yuk9dEH6H`|U!kF|~$ z*p8EJ_w=$IUt~KLw)-=0wY^`S*H_Au`YPK~C)@74*mhiE-2Ab5vQGcE?S^~C0Z-dz zJF)Q=W7}ex`>*@QSyxMK$7kD~DzuFyV2rfdcJsf+wI<}Nwd6FL0i>zFR-@veQNo8U zP*j;jU6t;?1g+wmJRHl>Mx&?myWqn}J@t}$I9_NGov}AUbKWShm>Wn_PZ_m56r%ZoCSlggsuqm&OWAFX^a;d*qibn(`RRPqRCD&d>J9=Obje(6ws zD`zrcl-C#VW=`q}@5xlpEy!Ts;woj@Az7Q7$AR+an4Q+KO=Aqc+;&`PJFb$)b*xPp zr)OEG=h{xcYx)&Hl*zBuI4#`Fb2@laPi!Zq*8Y-=L7(4pxa~0xhaAAPn|v*D@pAG% z+*uRQmIZ)`pJLYslc>#Rs20?j{Qau$o8Yh5%=Z)&NR=e3>Zq$(p!Z7gWscF<%Q@c^(`wXCb!tn!dPXYDQ&oyUZl{J03<|FEV`^wMM@k^n){{5@Fd_g?OAYvb zQ9^s2oOpoiB|!V1aLDHz8V-eg{9vet$Z;R8Jk7y9Fejd7}a zagjaevyxP1zgQ_RZag1pwE|4KbEyUX4VARv-V5kDhM|PnFAu;~;2)|I+1r_i znFZGxx;ZS*B{5ABpl5Y6`}TqDR z=*`I#F1R8?`!e*SGql}#L>$jZmN*`mY{#*-ezAdG(8`ZVRa-WwXB?Loj;l)HG75IB za3abhU8$-|x|tF-CQ&{pxY4ieLIRj^6PSsC{uSPAW!fca40yg5JMOxeG#5)g?-2$4 zWJvx(7e!!3&Ci25!z<=zDv<{Wmi#=7KjpZN0QB*R3?O=^0U+}m@@_J|VoU-#&Eokt z`5B}ibUX1v-fdw)KHfGKjh4`lF_(jpkC&Tl5^TTi*xV(jj=KOF#~y>7N1j3I|2-sk zU_AF+_T3wb_ei;_v!ocH{^{}^xwiM^S&~GhL)@*KZ2;qzk%pl;BNUGtv*leacZu%V z93tRMi0$#=*|gc(Zq5|hI1_spw}J>Bexk!+TM0b>DDOR_L}sMf7LP|Rp1J4pYW9e? zXSzCCu1nK8T&~Zj9c8%|oSU#*x#PHf$({~3=vN;y^2UD3Vfu<(a$~D)KSf%12E6l| zhh!uvkt%#s|CHO8q{56Q!4LhJKMX$x^;Y_nR`2sz&t@0gu_11Q-RtijVt<*TUdTjg zR4tV0PlFY&zx4<8LzT2BlB+>m|EuXQG0_v_DWXfQx)M0dPdnEok|jK6nV{wgDPSAVWKVZTXSl5sM_CY6zM%-33xE>@gQj*;*$bmhS^ z@%7{8J`r*^3~iY>^<0-3%Gs+-eA76PLG(o>*-Gy>NjxdN7qXhS4B$@~#D3aQRt9k& zDJATpQkaylK1ii5&;(Ce#!)Ca25^cJ^i9QP%0(aWdXSA=i#Yt4;Ejp~Z;&leSOK3S zj$rtx6}9$YGWFxyjoc)%fjOd5v`lr@E7LliBq0(*()`eB>rt;NLze|&eW;NN;^x+@ zR+Ae2&J}8`M7H6Av5`F1y5t6Y`zp=_4K1MUvGRALms{Y~J;?{F|YzxX<(N z0IylC1g{wHg9bYE^%z^kjZkqv#Xpj9Yj$4t#u|kabBNzz$^_wXW`g|DjN4gp3`j=? zux4uY2YMDeNBFWP9U~gwrS#^HD81(}Wv|AHM}Q`~a6lc@DQTFt)-{;DWTU2tcnlQ) zD{BL9!VKaJLsAw-c^P58h$$^|QZRI~e-P^r`LR4hUKVP#mcXnffw^8iqg!|b$je&X zs|AH&XeG$ZCT04b{n%4!A7&>6k|$*Nd7pdiJWrRuq5y#*tm;H~2C#$ETdb2S3%&P| z_$t-ck0O$?z3_5gX#mTDHq9~5 zefe-!wn0$mCC2}d5F%9*vP$eqB;dFDHc^(5ldMyP@E+18+yI`=&qFS}T z%d3QL0V3lz_n80v)mHV4<15b7SGg!SZS^bEYU!rXk6^B~rqX>qT9n8<)Y(EcK+ZMy zZq`qNcZsE$AVfyt(Ly}Rr4Y+Hk~e2OC>5IPPiKY3a6ZnJ^id`insS>$b5x6&6`EPL zTMUUO7CjI*l^0{1?!jkMRyAcemt@m!n%h~KafeZvISk6o0uNb6WvcH^RN05pZVOU7 zM^8za-Yv;eRJpmFM~IW#q*sg?(Hf^*uGN@@itT-s8RI_!c^w3PjVTRj`kE)t$jZN2m?l-yluaI!%hnnnedJFX5ki{@Q+PiKUiW@S$Fa&)plwPpGxcr6Uj7t!Tgsv$R4S&KtI`yd+t#I0va{|ckOPzEs(B=aQAdst(cNi+ZlY^oU>xoR>T12RaJ zJ&UE=O=>ZPMu%cpsH~sNl==x%rt14m3BQa!0WCkzSazV+m64WIO4^x!G3ijOtK>Hr zhNzN$)!gyQ?hSBGkbI|e_s@slO|)9+cm67J*!} z&v&Y^e#`%cVTP1<-gIHXL6aJL&g)6yfLaN8qPHOP%oEC18Sz1Pnby~fa{iAWlR2P- z4`|J5tmjmGn`{P&NeoM##x)6Rlfd#VdH;5cGoIoP)RP$LdFQJ#veqKBry0>rQouNH z2u7%ieqgAodoQ*Zvn4~%OJc5&Yu=h?+)Df%HxZV%mdaZ*&09;2TR)MvdgQH{^42W# z)=cBpCr@TAY$OMXXd{R~?92&&bqR^C9S8okV`{Xi)_g;;r-AG?c$2M$Oa?8O;9 z_#@Ql3>_XjS+(pnZ3mc}ZL)c^UD`(NTT0~tZP(?3KbKvG$vm;)@lc~Jm1kM3C_8kx z)8u2FJ7YzBZ*@NX+lM${H6s7#*Kp0@E)HA7V{;_)0E{enjT7w-&f#^Ik;IxAQ3jyt z$?i5#S;-3~eC|ETh9J0}+XgFFpJc-dT+eKSiVIJY@I?+-aV>`)sBeC`2gf>XOR^(p z!%_~8Y!Mfof&r_tG`0VmUhFNcOyhjqBY$B1iL!|Cgqs`7RDx5H!&9$I4 zT<8V5Mki(Zzkx$zzDH>PSY`U@@M8ccfOW+~&i~p^so*<^G}NVL=4b}zjZN;$Ib)M^ zIbUpYKO<#yo3=T$JRwD_-XB`w)ZaNAkHy|J^ZRsV z`g@dzH-|0khWX-<45t6s316Z>pST8%C zsh!(@--$Mi!DLpcy}ws`Z&rOP;592y$@Z;PW;}v#a{hqNW~R5H7$scX%2Lln>LEaz zen~4$w8tx)X;F5%q$GbR%^A>O(6VdiqLHIQKRHEp46deLU@6C4Q{rA^fY7i;mXQZ`_5* zDaPSq_bpdKR8|n{H{c}(ZC9AH-eB7cdo#h6_XQuB?JazJ?JJX16#Mp<6#ANt)&6MKU zOwO}Ft#qZ~`DN1j*vuWuwFzoBFVZ#X?3&{k2;ag+U+laqU<5`UGTPb;oaB#tmAOAs2;$%$eUV3d{i%rV-C|GoHdfD>e)Je~`h{tH8HQbqK}%&E@dPC$mNIA)=h5Jb z5dG0A0R@6_(kgWakC-$ZxkX$&IGIx#@Og1J{)l21rys;WhIdT9=X5B(Sz5`8$lQ8m2Tdn7w~K3Rw*7v z9q(t)r~?L#sEUke<%B9Cw(;dv?XhXe|?^B>t>(f z8-L}KOW;=F;u-$)J&@Q|(-d?A}Ne$l8Mk4Gu77d&&=pq2_-##W3kV%8$ z$4S67Zf=tMM)=T^Gs6ewlYZEiNEcqKO~zWCtsdFiP#p8_hWX{}G<^`xG&U0xCiwLfX}@y5b(Q{@c7+vvxL`1LdVKkIMkTT$MrRhRUCuRH$ViD@9= z-;xNXT$ZjB)l*<#U6KTJIUamyJwSm0HP!<(WY;WJ1_-dZgcL`jzXws($BW4tF;^Cu zB-kFOjCMr8l(j6%Hvt`Jo?)>Xi6@5n7I1D=wPAU9&wR2CJJW4!R}NG0JlmDSJamCQ zBEw4X0vngZGU>uBzAzFm)U?r8;Cv~Y!~eCJ*^y~!#3ta|*#XcO!`INaN27ZEi>j^N zo3Biq;=#YgxcWecEmP8qF;i)^l0F5S6l}(eBNqGXil|jNJ+LYtZknfRMR{;T16-o0 zZGK%55~%}zuadv(`5Ps& zqWciHYK-8f3~CDZ5+98{k>tKg)I;GI&S4R;ZYTq8#S{Xj_dv9-S6{Og*drz675xrJ zl%&@mk0^03=l8hjN3>{ttH^%>E}o2T9NB|xF{T$_)sc}Q#oPYjK=gZYzhrx?6H^8v z$NU6K;0fS$!mCjJUY6!^2;qj~Pxi}#RsD*?l;+n}bD2t%SOK^Aty?<(V(|AcZ3uzI zowv?FCNkjtkPDO0em@o`z8e;sS}OE`6qb-Z^VAH&MM?z3C#g$)JVf@{34P|~emu*^oBgL+Y}t%ly8~KYbYMb0BdJ5!_@W|1jrBV2Y4$gq zc#JIUOUD)j(TSe?<{4G+gvV&a^d?1{#gn(7xVp;?^U0~Hn&RLJniXid+~#VHf0%Z= zQipX9+DdVq6P1rZjQeU}m1GNTUAZZsEfe2Q zU(UksT!F8CDH)g8)&S38wgcy&woC#wGa0x{J>`c|q7d@L;^_I<8^g9GuM*~3hKI28 z>F%mQ=TvkR=XYa;^o1mIV~okzODhna1pk$}+l`@D!Wa}9gTJL&^})A;{qYs>(PB{Z zqZkjrm$m}#;@9dLAD4DB?j((Ex-pKczisuspz{C+0x-bl*0ff+*%vNIQXJVj<2)qn zo`SCRr%*SK4#kq52mX=@dP|cM?$n908ig|n^I`{~g*lyI-|O`!nWXC6bOoDrO+*D| zuH?L28e?sdF`q@5Lrf>7C7Zb0fQ7S(<1I_(+<0!1Vhoata>Q%5v*|$!_uSiEl)vr9 zQJXcjIQ%W7-t-zA;&EMIj!OwY+nFvU$szY}^D2nE=_FJk%^hjDGNl}6B|d+WiC7j> zaL8EpNQ-@J>)K-5cS>w8vyJ}SGOWP4iTie4NMA2<^Ir9^1OuJW|5b#}4PJ)^sY@Dv zQ?vO{Vm))7Ku|Q}y7)(x!zz40H}E#;B+e9nsWQ^=^(_wGmmKJsNCZa_`4l9Uz5(~r zoCTop$L@0<4SOam-aW>e7B(81n~ME`ZgZIn8Hvxnm@imsEavAQ=fynbk%Yzk_T%Pa z{t2$=9}R1@<=YUR5p>oxe=z6x)tY3CXMs2ui-*MHpE46%E{;iMhZvWDlw_zz6^pUo z8}1wzK%s=`Sr~*^LVKOgkvhK$moc?wj+gi3E%B()*%X_gy#nwEcQ_hlC@ADnXA4kz z)D7#eHihlodU+SBOy*tm1Mhk^g_i;G;J3$9wN?pti2*{(LZwKw9tH)c6#ZY~&TdHJ z_myB{0R}~e5U^0i`$h41ar8D7U_hbPlTx^TPLi4n)-|39xx7wPu&~ZUY;i~r(pf3U z2HvkB91a~x9C}%bqwNmzc)GtEkLMd3Z@H~4r0l%|-L!^6rSdM^C?SLUVrn7OMsYLJ zfc8GYs;(fDdbVA~qp9%N>Q7$b8l9A1&PL%BTm?Tj z!_TTOl5kHW>`n-0IILb!g8Pr>D~m`3s@iOP*nQE--p+50= z)fajA-dcGT2EaL{H9#tOQ_9)r!BT=d8sPYb7gI5n{4=-XY@Y@a6+~AO{*w)P4i|CMvtgh&ZoBxnjN1H^#v)dl`&D)v*uwr!P4`WUk9IEf30Fe_gc_0-N%|#XR^oSF5kz zQ7=MU-Ju3oXjO8j`sxTRqBjO3RIz*lUcI1!gkvc=|0+=)U*=_b{28u**dK?{=OmSz zm14^N!>HNF_^aCTzMwgMEi2N_Un_N+jY*bnvzok}H6E6*oc3E(ZfRKvAJKPIT<&8b zPMD4~l*MhUAI!sUBvKEGW(B}tYH0!5%Z1DJer*#jh~8M-9lgix$E5ZCgp5vzigY}Q zk2k?He&42wABYPIw-Z^)gSo(Z@skGQtHmwBwKBKa_)d1cgMTwgY&ERJ(BnuC*D?2p zQFzqe`_<6?G-bvinB=FJLm>W+%M^#-pEhmgpuGp<^7RgEOp7@5D|df}hy+Wp08HvT z>y-3lfAkSmj5%}|9aPTyx=Hu|{X9VqAnZl97+1S^=3ug;>NP~=VH1dv-;*tupRI^y z0OlD^{R}nY#QGUA_GVWbZTDYTs@!#-v5)C?k>_R|sgie}Rw9_3p}$kmn&7XKa!8+p z+m_DCX{P2`ud>c#D=wJ;hA1gA}LI`sO@?imcSW*p4YNhQE zvoOQr9ztn|B%#q2EyG{?b8^7hVC4qAFNuF)+bIE~h*e%&Fnp7?S% z3W(kZ)zCp$4<9=j3Yqzz;Hn73Kr8osrbI^JfVD4a>1`8N{DC8>4LEgGyWFiGu=*rf z`lqoQ+4c@K=Ro!UK>xh$MjO#w4?pw=jXYpWJ=-qcMWGtwEd|+b*exEaO|~m&-Kxzr zg29v7@t8^WOSAY#1&~$Gfvr}(btUQK8x9za`c}b__~-!OXXybeZ=1hQsv?Y`OD%Wz z1u@#QK0?}{*c^m6?;Oberei6+y>V*WL&j%>Rz-8G4a3?pT4yIyVJz^}SikFnT!MGO zrEyEQx_S)2VToBkuU+u}@%AR*O%>haaJmGdP!bfRucB71QWmw9#a2LCXeE(UtAOlU z0TEd%wo+v)w#CMf*Mfj3Ah;vApi*SFY%Qx41O!B=yet#SQb87hH2*p0%)K)=O-1>B z&&Tu7%$+-TwzJNenRD#9r51bJF&1my^Kh2QHOAuYspAoHt8Xq6x3cEHas|vo6YRVf zHPE~UNOt0~PlU6#=AD1^U()_peHGfD*O!#$na`s@(;-^W7Y>`fLubTM=9QhuE%)Uf z4)N5zDR*<#4I4=kMqv@}50_PbfKeBZBE$HFtzH`@XwQ>t6UOX#{nDb-{V{5|rMcS9 zB5Ib~Omv;rp6WUZkyRrhuw75EK*zuh|3p|Kr&To4&THJ_a5X~OjR)VT6ZqOW)jb^4 zOwJ!*g={o6Ts`MAm>N!2`e=#l@PzN8$j4a=IX^-iay@MKd{5ZCg7z+BN%^@VcvLJf zo7hxWyY&@sX#Ig6LhMsm4y8FUdb~TA&Ni$-UX%c5D`eEq%N&78b%o1IAa03eky`7< zFE9Q9wx_n}DC-^ST6f;5g@ozd(YoJKYle>XcC^540XDmTxQQYO;!$5dmo3T!Qv^2X zd56Yj^M)agmar?p-1uq&S0|h)agLDM^eFJ2yy?7%cfu8lI2()i{x!Ik=4$t82;2Z? z@6F&yvkOO8?c#3V1Zf!iR!N6EBGVr$`Ii!;&FhMiAFGsqTuBd9(%&fQO-lN{wzAF- z;$?c6lI|~mn8bGXiH-r>n(tqC+0)h{h#O(>|l=pVi($(xI03HO+LOw`Vi^Z@DOMrw)z zzawUHvd?r*Za58U@eG60szWY0 zJs7R?47y4f8=cOoP`YHoW)Xjk+sS-qeSU2v+qEribf-HL{F(L8QxC4lZ{QS$H<)0z zC7s<`>GNyrackat$R&G50^A@D3R{>1LJe-NaFqqx^)vfKl82EBuv!Rx!Im2zV*T%G zl&)i=txi`JbM*ms-Z$Dx7hvQ!#FUIcUoZ0F^M1|hqvTg${{6diYaveHC597vY=%S@ z-4diSz&gJPItMw!?&5Rdy|qfy3(qhcWLzM>B?t_kPZQ>+th_8oYxC!cfSd9(B@{R%wS0gm5a|R$i!b4=6!h*o!{?_lt9#l9@38n5-?Dn35G z7|VuFb_3~u@P|+AaSHD#-mkU9{)Z)HPgc_Jw~^`b4+oFeq|omb5k06{=pYnILW?gY zgSbg0YLEHGF?4JTR~yq_*Qq#NqvEvSRQw;L$o5Z9kW}2}q9${6D&GCDO2w(d6B@D_ zNyzu%AstR;NX>zStl#F;s96wknn7b=Uz{Lno>C(<-x`8*s74j|_qj!57_>CLl}{9kP&mR#f> zVXe5%hewX%SwBW&JJ-fY{jEbNz)~HDd={^6l-%HqiEO7f$qB01>t197gXC1X!KE=- zNiDjS^qh%EeZG;@y(W_unn_is6#v44-XpzrCB0|WOz+>-NbhLu^Og|3vwJv{y=6XW zH#yF1Cp3Ef(2#$LkI)MGryv{88md(%>QhEqlC;3+}B-?kED zd!dyi&o-@Px>ai_cBg2Chqb-<1z5s1YhGRC*tgm9 ze+8>-7qrT5aRnjWUNBFl^*~h;PQ=IpJ835O{rF8bK-AXzb z7>njNP)WIN-LND&1od7Uxewz&%1Q4v*X~M-Zo6|kO;%qy>X3e_^ zSct1tv;&WC48Wa$c!koR#p|K#?R-A@kR-8t-}CyUeFm=R?d0R;OEhuWlHl&aBU9q+ zZ|CI#=p343ejz0XATOZpiP-ZDgdaZx0LOE90R2I(G!F8H9}j~iUPe5d`o00=H!q{5 zQVdXhiC2X;c&Eat<&dcRc(@7okZ%`P+SDPA38Nt@oJ?^7;BM|{dXhtl?qrr(^9JK0 zG~lFQJIbW_tmZ-h9VXG4qEZ$GYJ&(KYY(}E`Hy>&o8TT&=@YJ2O;8Gf^a1(%_Fg<^ z$@~6QsL-#h^}}F?!keb^Q(^x;&TG~vbe-Kjojm%jipQg6=^VC2fOkZ5H4rYxxGst^$GVh_vVCqc0Qhbf0R!)8b*p`Mow2 z$Y9T7!k%~f>oAV>#ZJ$3^8Q{FLz3F#q2u%U`ssPVcQcNLi!-OYL3G8fV!eK|-Blm^ zVZlPb;vD<|ys(Ykon&!bJnQg;mpR>$srkQV@!vV*x9LB2mxoeZ<|UY{c`$f~t1bu2 zFBL`7J}?H9LKtsEEMI=T6rhKYyLjZl{}xd)xKZzApgEn z`yQ9xhaqAZOE^W`WRP_vSGAjR@qY&MJgj^E1rlT9yoeJMUMD#XB{}8-RoO%vb*Bo-WjXqfG!!aTF z3K7ZSYFuXbh>{`R5YTmRItVE9i}zK$4FWrCr@e9D4#AUeCt%tIFjHM?y*Bt{0w1Bm zVg;Ref5tQA?T4lFwMmO|Y3_@EP|DaoN_x+O`pGtL*#lB0&wD`6M|R_V;z9Yczg}=1 zy_~Dma*IfNGes%yWToC`9+dGbG7-OwL*s*wh^srsJR(qn?epL;-yGQcq^LvEfZj>2$nmc81Ex;&y&xUOW=wwT;3L%-XnS=%T- zUf1c=N!cT7BOU-LSkq4AFF3ZMG3*z%FCWS2ra`$X5i8^9{s0|(Gv7Tt9BqIY8;TIi z4{BPx)1&!|UW*%xxcE{=`s{F8#$kWuGOkIuvL17K?wy%Q8$oj)4+fD<2LK`)RES&v zS7+eCA@2YRuJ-v{>a((gVqPDN6`>|XH~}$?fx~D4J1-C*r9*{i!XtXh4kSVq4&w^| zm7;Y+gAY5DME4{J9H_Bc*ybdH)mnUMzAzLXFOzp$pwevULLlL=R>Nqf3XL;WgxBZV zH|@=bH_;I|JDfz=LQXFbt+9?EVBQ>Bbn>Ut%$)AU+jrl?TJp)!&n@f~M<)Sa$yj{g zLA;z46kjbz#Jpw+d`(wTHC;nhnv*SwbVZA;O*G=_U>Qr$GY}Tmt)ej=&+T;Wj!os| zOOm4-hj_1=q<l60L0iriM_Un)P z(dSKL6HSc=SKq+9S>qwXE~g-cRoLoOFB4$0$QjFPu;xF7M~?5$eTn-1c;{efley1` zHlPfkA+Ti#A!30M5z#cDP~c7z^+AmTxme`IDtR+RUc3w(I#%;pq-fTsL(ohBVld#G zOP~FG^0w>y)-a!To<-Du5Dt3!_)FFbiXB%>cK7TTCcFD!Vcrm49{?Sq4y0a(ybwsG zL+au_Auyo!6Yv1ChLuCAwid^EEC=-I_6neX=MUa@so5%CJ~q0EhOVq*$8dmf6Y2`ZOlSSn4yEx2@KN)nIC1f1bxwEx z9wA+c=$k=zF0#@=0HnszfPHCpw+#<#9AxP)n(+ACrFO)gE8B&Oc{s*XUS5^IP@C?> zd&P9;5T3J-d4@L$K(nl!(#J31%h|Bo^4$SpE6KS}s%8h3^q)%l;(glIrIKlDE_JL9 zO+AOmPiAa0L zHkG=fYqxizeDwZ!uS_=*<+&PYclm-v33j-Sz{Gz(%8z{4MuFWunS90d%I|10$n0i! zkBhW>Qqbc>gx$>(u*ny_->1jBB(i%!#KIJhgS5z+^!VcHM4>$a{?ksK8$IzuW1sEB zV}$JBC|VRNQF>cLlwQ2cA?)9KuwiH^YevQUY!*F3Q(4n1-eYe5Trrg`-tW`ru%u~# zek?52x$AQb6OtWQ454CqA7{K=eM0&J{kZ#Ba>m4)GVK8)aJmOXvhp21r0_^Fifn}e zl|vi3Cv!xU?R7H#$Urbm9E8!E%zl5;Qu3M*cfsz7X!j`*#^{4S^hTdf5o?f$C|u1r z>^k}}gIG0A@LyoJU{o5GQ+$ZGN?-4vvyM!(fq%gCihR~#pCGxh8shhfF<2bwJ`Tsmqp(JC zda`nox;s5zWk5*9Zk)v>xwdw7y0-8I3+>&)xwoQtWSrAu@FyP^?e3IFzJ!dT91G<{ z@uZ28T#wAZ#Ji|KH)SJV>ijqo{V>>`u7y{Zqwu12OS-(ypUa=3bNS`u*{hJ(5{ngf zvhXBuawS~xD(sdgWs!&;)f%;Ud?o5p&xz~awRX?+!C3QOxTT(9b9z2arx~Lj z;-`FeJszkyjW2(p<|O~qF?`t$pZ4sn$kUTJ@4UlpSHXag;un$s6C|`%G!1(BF9}XF z?QRc1hGJa1LfHD>037jX4JrGc1yei&l%V^7vX2JuW9Vh4O6u=zOnLb5(d2>sW+RoZK( zyL&OLi+MdWRop{dge3}xwvr8PJ91wK{bP86(=&Xg)0LIunlqMgy^!{L;Tj&N1isem z&iR`+&Ea`HmZRHP?EW#QC#8U68|Rqg78y`&3ZdFe3Eoj~xm4>*wjxYk({tmM3UHA_ zIE=B*+1o$+ki^~{7Kk4GtD}boe@>+pZ%X{=Qf4@R!`dRFR9eq^qv9>ao^~6Qe-%(A zX%|P8FSiI(DT&t1HzjbP^K`TlUqLzTkb5#QaKi;Sy$x(YQ2^RHLlM$_Q_&*LhCLy9 z7Cg5|B@iEurkz2sipCo6sv)xxG2SU7_qB8rtV}GKqwSv8(kMuw|Ea-rkW`=buCxcJ zKetSYI|W^e9~x-7&T7SlDPK}BO}W&LnfeO+*>lHxd9(uHRIew=k={Z9ArLgD*zRg6YVLRw*;+$L9kT_H1s zd@_iOj++hQB1VXdF|$Ejpae=Z$_o%7_5cxLLV8_=Q)l5MPW<-%UYqxWaAB2GiW)C^T9GGY>4^S+GVf@~~fs~#S zOb{fR*ehE(-c2uLL;0^aTc>XVJ6MBZ>R{T?y3I0y>^(HUE|kn{N2yOxG~SdRez(wZ zR^DAD?Sm3u9j;%Jo=J6;poWvEX*fbiaLffQhg<;`5iU$G52CWveG$pOp92xm=~@LE zKna4w#GZQ`(GoB+b0eThz5rb%hO7cNme_R#LlBYD462N$^>I5yUcsv2WcuFs!-%KD z{$FWI+@>L>l(^!!ovXoH3hRv*g%Mi4G~Fj%;PtaS%>ohorBVZ8U#cR+{^&iS2XSC3 z7+i_IXP3fajw?Cq@j%8ZBqJ_epjWA`a*k^_1kW5&3Rjg?2@S}z7Z2?&BHP|{XsfHg zkj|fs%Q1+b-l4jt!;F|`-xf^MxfRQ=N;BsEiRKKVc$xbXZsc5WYNNf#7TW<-MK(V@ zZh)&opklh?9j;P)!YY70*F2Hficpm-MYe)G^WjZoqJQv3tOQ@X5nODtc-7!VfOrS~ zF;I3X(l>tF=O-#~ZV)GXe5rl<@-}+2Z;wLPpKxN^WgjNo}@@}|8lai%& zSDCl$4qd|l_YAMyp~%o*yu0Pw8Y>)jN6_;i8vpWhMFrGThif7vkS|TBjt} z!mZ5iGyaYR{gMJaVvNnyo2#WcTrDjChSMwARWuf5)(qObMV-x)nv?2cu=Vem4sZQk zBa!MQ`XyoMaRo}&Enw{&w-nd;ykGo92b4zw5g$TC95)|YWpDkLQ1EU+EH}0S6yz@v z|D1_5uN?1+wHs}dksbAf5HFc`+TCL!*{cuYMN+%Fj}1K&vJF5m`M+lU?r@J!S335o z3AB~R(x4ah9G>xs%P|zTtIA)-vtzaFC_}alvLi4H@4jF3m#v;D*1UZ;lI3OK^MSO{ z3uvRdzTBJ-6Q;(R*A!6bK$RB%f#-XRBt3-jXVFt@{?kyj5Av&GuJ<+Cq#0+{0o=XT z+ojR!Oyer~Zg_#WqOk!-1N1?(9B>5n0fLfb1so-v&?VSiKdul@Cmn5^&E>I>o@gRH zF_@j;-RA&t0Ff^Mf|aBc03sI>BBQl}P4k%q5(i4tuV|^B6$#P5F9YA{*%h_INWalw zVDkqE%EV%Wpo|n6;A>OiT4@UCqxU7%qOAcGlkCBF{*Fq%?TAzoYwFF*XXsjAFIRpG ze6XPEM-~pUoRH4PpGYA=h0~o9rRj%gSFt07H4fF5eS)uz;fs~OKrd`TN(2|tWVa!^ z2x(GI4>p&k>5cps6{GHH{;u(yZKL**yI!^LPY%(gG|ZPGr9aw$Qd(p6qP#OCn3TQ) z>AAp^nFeYAyy(o%Zqe!a5fUuf)M140;%8?u*OU7N()H-zblr6g&~<@E*Sj>jZp-t%3nX1Phvy;k`L;C* zox>i7w+jJ0O@g1zSjnmWt_`~WEF>Qwv3J0KB=O%N(@GIzW!~G=LWVcTBmcqYM=*l%m;vZCAkb5gj;0jJ9xxcK&%ABwGI zLU%;&l=P&Mf2&wR+3<LqE7YEX8}4q`hh= zyJFQ(LF?E0o`yX|)E_I8aydXk@MIj&k;6SY67_)m3z@gs-Q#Ue_hXUvw$Dac-`#>L(5fG`l-N9i1(%q$9KE<}^i3$d^nexKcadvj=~t!O6+ZZq(;Y!dj+!tHUAA>$M)T)*Im%KxlJ zw=eIn_Thn?k9q`J<*fN1BCxdsrLf;!2}AQn$#qR1jw2Pb75gWDMiwsbA40gcG#l@9 zW)gI`uIO-GJO=V4tSq!bk={f<;X4Mrs!X!3f!M}(`}OmJN`Bpn)ok1n4tvXw680b7 zta4}CFS4VvEioOn)(QW@CoOF5a=2`>-m=cf&#Uir+VQq@Ech&n1PXl04j!x^HYVqk zY4PL~pHFcu42F3dY>~khWCN0z&0q_1opwGz9K*S1IbCbERP4a<0#a9$uy!08f%uAw z5+8%R4f|~+^%#y1KFUIvcci;<1A^tkNoe!DfDEj+J?7D&%#NQHI z4QO55r&936w_J81uKWK-d|C7Zb&7SX313!!|NkSt)L;9z`114*MtnKE9Py=XGmbA8 zih|*bZLyrJ@NNcp;rqlN<@+OD|0O=%vyvunVb{OIr}oQ(DBYMfhVyvB!$ASR@qN z{FL8cO`cGFH$d5D%Vd=wK-s^e->-^f;g_GPKE727i+L)@d!I5taRKjHy3&&dPX!tO zO;E-AU&&IZ)2`9q_TR|DYtpLq$GT)ETmGdo{%h6O?^|X30rjy$rDxsJg2FFV`97|C zf7ovdeCr3+KTnywKc5U*zwdilWXF@n?-Q*{F0jUnWPwFb8t|7@u9okn1+RaOBEnt{ zsGr3u^xCcZ_iUar_*OyQ7pm|N);_^s>E-Dsu7$4(NY*?N(BEQ(zkc#Wwf1nnu6`}c zSC^9prtb@;*Gs`&fmU97zq{|Heuk#i`Z!l}G9R zhp9pE#i_KBlUjZJeHDC45AuGtLS2c*_bg2TPIUGDv$6%s_-h#7$?s&Da%YhCzfi^> zXv^{zL7%JvuZ5Q(iL3F>>~SXxNplBAGbQ)uUp`1iv<=aHfO>WYhE&3iaHgy zDJAYg#m!)6gu9ZirVzK+k#Hz8+wKl?xK5@v{nP1=u_tWHIwV63gKJYQ%5Vp~vKJ6# zxD96VCI4v?Y`kp(FR;Z=x>`ha@i?O_e4#qwdGy|nhYNYTsjjW5t`gpCmaEX&^bh-* zzrs*4>Pv|Ks3hv?I$50{$?vaUNUjPn)FDbZD7KI;pyK8m{Xj8Gq6sE)-p#fg-;$D7D z7a~FQGwU~^1HA3WJuNJF;G655qd@0;XhGoRF!-ioGnq8kd+V9OKVBfe_)OvdjzD}? zXsMgyAL*mDS@W-GmhXWNmLx~l4z8*##%cZs^$lG@_5E!B+xiAptFKMaDxJK_nWBq1 ze_v&NUL~z(_(J9nhifN#b>GZuk8-#Z9PWpzFK3+5bz#|VW4phhCT_R;#Ug<(B)~S2 zq&X2)mL@my_Big8C-q&&W`A9MeSRJ77ky3b&*}Ux+izS$`<2i**GmIeaipg+^I~f6 ze%_u#DD6K+f#%)^1ctj|H`r^5<44E&%sKp$6YN)jVO@B*1D&6Z@X z4_baQ!bRz)P4qMKD+zDWza9Gg@Lt9R?db39z#{5s-eZ9s{ca(4bYU~-XzLc((O5XJ zTCJlc|E8l`zVrq1-+$BJ*$b$@yHW!C8%q7X(iHnU_O}0{{@xb6zY_nFU2CYz4{LOB`-rpzqj!P#G}WOSHdTz` z^$Az2rjL7uHANTwZ!b5YcMEGiT=h+bvvS+91JHO@u0H?qEQ#^?@B5`s+zTb|=3dw{ zZ~B+eD{Q1|#IxxFEce5nwsr;V7Cz4jCTl(%@8ZdL6R9k)T~)_PuH?pj?yI(GgTAzk zn>LESV&QwX3Kb4^|JQVJJqohTWN_f5up z$RD53?d#-%A0b9@3#%z8GNuz73EBkjh$lM6;bPuA(oS~W#kG@ZyjmGkr(y^D{&R_B zUtE>xcdl0BKdOGNEI-yHR@cwl&sNqy+WUVXzgo{D_zh1CRlAgA6-|$@;z_bOajuoVWj1OKd;Oy$ zv@Fok=1J}p+N9}Ddo<7D9!(0Ecwvv`Nkm41zrG)nQn%t~4M(P51{~4ttJVAoAU@Qx zyGKBf7G?5Y;FFgU<4ic383$L#hB@8s+wsmP<#aG*K9idQb9W^3roc_F%oiQ5;V^p+ z_oLt@BbNR0AQH$!ES+5v3 zvj*0%Z>3Mv*uQ+i?J3u?w`R?x5%z2E_n)=es-dz_kxzeIL;bWrF;}tww6AUtSoFSP z|7!Oy?cHH>sfR1t{hhr#_2U40H{r?!ZFd5Fx#pg_8 z{+>_2NgXX*zeL&Hb()^ATzu5< zSiO5?hPU!UHT!xSpKSvow~ECW3E%~Ihx=7XT{JTg~;Y0P<YngTga;wgl6FvohW0Yydo4 zDKDL8{{=iBX?iVq&YODO@N7La7(D+WeP&y(3D55F5}v)f7~r}8rJCV+@WbndXYLFg zp4l@D@O){83eP(41%+qk`w~pQRLb;%$^dxIS6&uZ{tI{xX>u)imK9z%JYRSz2s{(7 znEnwwXV$tVJYQ%d;puW1;CbWVn&DY*-gU$C`)N8n=T9@h^Sx;*Jlnn-6rP{Vlwj&| zMyBsL69CV~%F9+~{slbWzxP`3G`)S@@SOgl2G4K_&v_37&}VpXcrFdUHauHNcrNQ~ zfafEFYKG^dA6z#)|2I{K=iaFXcy5}i!gJ{Kpzz!^LxSnYQ!+jEQ~*2+m6!8R{R?<* zzUNx-jP+bMJU6@$M87T0{%88_-q35pGwUGuN$5>PtoCNnPPzF zUxg|>XT1{?p5fCam|C8c=~^cP;2EyGY<%)x!1MB5*Meud`?}$Iet-tgFbU6$*8|`g z795_;bWM2X$4YqqC)ohc_xslj&kx?eZg_TjTZd=-+Xi?xeOra+mMKBu+3Fn$rq55v z^y?=A;F+zwEIjcq;MpwdTJS7%T{k@M?H`0b%d-9%eOmnHYr%8Q0}`H&js|#cPOllB zKh3;uc)sG%;W^l2fM+j{3ePKV2ZiUrDH2S#d1d-E3xMZInc)rh{tI~azT;Z(EPnI4 z;n_Vs2z{oF{b%$U@ztiJlA_9n8J?9^q!*u@Z2RcyeE(T3wSQS z^;+chqj6t+I9CKRpGuj>Y}0INBJbeZj2# z;-%fK?XCICX`jowrn9ADiM+>!+k;!x^FTYC?#C>!=j(o&ccY%&bK;~lS+G>8VDhMqM3{f^9+92|+j zrJ(~u%|oCG7sStY*)%VP;NpQ0rjagrxvyQaGt}(B#;%jyPdt@SH zC-dxgfnJtdd9F8E+pibT`aMV-zaFcQ4;yxr*`yq}gXe`#I9$bmBkLyzAK8P|My3o3 z2PsBIH}(qE{_W7>Es;kKZ2uQH?80j00Hv`WOu)UVK`l_ zqU|9i45D4hkvN+3QjUZTqnnbOGW1tE4$cg8wFuuwGx=j5J}Yd?N|*&Za~*|?%EEQk9QxL;C8$+JqWH6Iunjr^S%iscVLailb`$AAr!7O zXKZx1$(n~(iQOITc6aYY^zySeGuiHb2*~kf5hT3?d8)%z!UljN1lM{9>^w6J4zXN- z_%IKISo7l0*TOmr)C`AkSMoYD?Cwa2%I-|~UuLURPnNC1D()vj!Gr^ujqqqKZz!)g zvm$9F-cC&9_yd)|JY0AKF5_CP`55>Rru7V~4DtK(inDgu+upzJrdp;X>$LUBcMpsX zx8^?)h6jrRPttrv@k)9o2e8XznwD5r<%jm1?hHOveI(IUOj9ZYdtbW{b!A zP*PankTVC*C$B*qUHipXCniyD!c>5hnsLQvkk8jO7cOL;3UK0 z<#unkBiu>XU=iI*$I6gcY-htjMQ)Z%t3hpX>U;!a&; zIh@+OYbjn8pVPczXbEO4ZYK&Q@=If^e1EkY?cRNU#lIQGR&k18p6el2mOIgcI)FRd zQf(GTj4{sQv)L4l_@X@lqOIrWVg+#EnkFyA?uPR|qcBvsYYm@8YyJ{&5^-Gjckaf? zO5l?khfYa4rrX_*_e{1f$&3ycC*jsu^IpFSbGIE7lc?c|7lbIPP;42J3`ju5NCkA^C=EH9M7rP3i3LG%i=2AtDV`JiNMsuyzz$k zHLi6)6AD8(XwAC-PR88#*x|@%yJ%MM8b+Tz;Qfu)%^u02YW{$SITIyIwEim2Upw8M zZSEeC&bFPSGCSv9?vUBc(=o)=InuR*?hX80&Si_Ej>&#;r{f-5V%`P2HF>SWWrF+U z4o{+`v@-?nadoy;Y!Ty2v@ZF{=_>I~{w>T@?LMUYF&muev6prhk$WcBSG&7&2AuySrw{9m& z?X!;c;e_62-JrnptyfVv#M<3$0)1u!PH};D9L1R3)i_#Su-)zqL(dn5`$5`@Q{i}> zlC%ES^q^mK#XyGIX>rBj^=;q*M{^ThILJDZ7Zw6{3;S(?@R&Jbzm#WAu79~lTR^02 z#5ikQX6+R0Df1bpJJf-Zx;ACro$BU-r8u(zWXKR}J2MZX+aRZ_c%8$I2wTgZ@VF)O zM#$0PDVr{g~5zJz`?$Dnz{I62Uq?j{fivFY!4(iK=21~JOt zBm&Q!x4ZV(tto3b*K{#ZMM5@Or>;$HS`MNo|7(*tj=Ps9mcf|feocW8v9-Fm!fV4p zycm=cmm%g-(_`qb$O59&p1YRw*vBu~6Ij+shsXIa*CftoHgZ6;r|tO3S+?UfI&B*A z2RNrGg*DqI5eqDnGwdvI;}3zMc*U&YSP>h_;pzvokz%4#(TY+a_}2<~gXRz(X?DV~ z>*-_QoGnauCfvR$wTEym&Thc~$3c$7HpU~d6)U->x90-a^k%%O-UnCWiFen|f#Z3_ z!62c=nMg!!!1&9VKL2vi1~A%N30w6tig|4*dl3}VUqrmxq?9nc&fXYg5PA*aL*$cb zQ|}x72*}M!UPlELLikz0HK*wVNwoot*BLjy1=_ zx_H7LwIe-R^X?2G8gz9>5$NI#cZv+b9(Yo0)&0*#%6K9LM#W+?fpi7Mana_pK zm{)vwa9+%62^*3@GXFeZ9PZIV1s0*Gyv5;a#7EtMcwU2`mq zt$K~lZzFRX%GHRLP?77Gfa%Hq9l1eQPU%B!I7gg^CBor&j4z0QPZE%Kglqt1_vcXp z@}!5ebt9>pe{%iK8FImqU1Rv`SVwLLQ)VQcRtLQ> z)zz-VyHmzz;aT2wy1y)^=apDKov{Q2j_ls427ft@HK*EVn%Pkf3zqh><}ho$x8-gb zcdXrR<9!UxA0Bv`9li*C;{e*jz;aIxw!5Z-*Rnt#ci-FmemsHA5pf^$fVr$~^)UfiR@{-RV()(QO=>%bq$= zI?K#DXM#^Ny8NN#wd8r|WgJ#E_l<_B!df^Ds4;39W8-t^O(%ojo`amNfChq2U) zu!FBcJ>)CW%%V#t$FLvBQHd;Uj~+}X8fJ^8$AyFZLdJE9GKdc;QeWy<~J@(sUja@@Oj5T z{9RVT>CU?!6jt63Hf{v?{~7J0r;#IczmVSCor;ghtU_-;`RsKlY4GPC_nCo5|g@OS3_N!d7kFl^^;54wPX#t*X-4HyZ6PgLv;+(8x5Alvy6jA+9 z`2?u^3MiPJBBfr+6)|4l1Pi-)xTx=rfcmy$eVf=}PTdu2ue-nB4-@s9YUuA9FU$UF zrXsW%wqc)Vh;M7{6fRhDA?9j1X`I#Ldkf>S&_;4k+NxhmI|_*V;H(h5r}7JOW4QMe z&)RBj9oRvFfd;0j;CBWbe7{B9;n3_&Yz6ZY6&NyNd0A%DYn>%(mnA|RaV+5q4p zy*}rFj41SCTUv|s|4~pMS6Z_Q#Ai-dRw0;j!Xf@^Y5p)jc~%C`^V8P?tDIgOwYFe7 zI8LJ&F}x$+)~AkuUs^YQGQ{Knzw2Jz2`Le^a7TY3&F*gBfPds~c-jU&2TOYP=Izde zJKTBC)&d^!Yvx@!tDq6i1Db`yAk59{gQs1%UyZREfiaQyaQ8UQTvZpGP5B3_akvm~ zV3aSAH+_T7fXFo_XHi&G@+t4-m(}>Ae0)1>G``=_#M8Pr^cgvDLRDrXwq=OVFD8n& z3$qT9vaydz6o_WrJg4i0dAa3RTv&`P+ZhJczE0KVZ!*YJyT|;zO>>%x!MHZ#>e!n1 zA$eftys&j;RhTFN-bC5V7m)wXgr@kEFD2D2Wj?R#0G3Qy#QwYjW#Z#5u-8FYKos4a znK0Kbb1y3gMOp5pTGmP75CVp;u6a#M`J0he80wqxsFj&t!l%xhaLt40#bA07MNb>) z>1le33B@NTJ>}5T2zn}{Clfu*r>D#H@TroXcF~h544;0YCkQ;rpJM5|M0&cBp6vA0 zn4U7s__T+m(fo9U^Hp0@HQcIS>T%^NqnnbHV?(xDIn&4=+_J?1s_4De}| zP4LOSs>3JEXMoS80X4#>oln50+hq+t`$q-9XIX>}pQxV<@Yx=r!e>*24xf?;6+Sr; zDtx|+P~r1sgbJUZB2@UKN2u`W9ihUfRfGzk2O?DX6h^4<84{twXKaKDpEeOHd}1S1 z_%w@9;nRXYu?gEHd|ufu;nRfEp!d>cfDc{`1=q3-7t{qXtt0x&n#VeBhgiu6z3Zl)zu_^Slp5Q?gsd4;dvUL7ig#5D9tFF;Emed!8;RD(2o?UDvs ze?|ak57yB^+hCgkw13u7L3_H64%&)3Dri5gqk{IAIx1*))KNiuu8s=Y{5mRVC)80v z+pCTW+OBm}&=%EELHl+c6|^7LQ9;|MjtbiDbyUzkSw{u!Q~Zg2vJJF&u2aq0CgJ+z zU=AA>hbx;tT!_=To24qXP^7*8*&-Jnaa&}8B%by@sl2p6MlYpArdGu(S{$6&T7qrS z?q|zoxeczV{Rh^a=t4R3KYaPenvO(d$h;3!0o{&+SGLEGB0S0e7MwSOo7akM-s4+* z#`qfsE6ZSmuF?}cK|0-082Tj^wJN8M0# zZex(l?GzIAw@VzQ?|F$XHruov^o?6U%SD3+NL>~>njE0UfQgC>I@1Ho`2Im!T6tpV80%El|KYo z-En4z_x;5Cw(!tIU%xAxNi1!MG^T&DNv6M(X@sr!d*x-5%`!bPu;8>!!R~u1_&o6> zFe*bB@1D5g1Y=AjuDwSA7x3gogEm=0KLYi6;WAQ*=AH|~Z;MS-b`h2-g;1V*O(tQh znvFqJA#bCe$MKQ4SUq;7qGQdd;ll&?O&u)v7%I> zS}IX1)#yca$J*YUX3?H|skU`eF98l{bh!oRyY<>Xg+2pb1ol4*W;YS`*A2msJ(Kmo zsx!5j_e`URuAEPnQiN#&h=}NH2*K`z0HfU~3|Jz&wLi(*TTxBdViQjm(b8cnFZU8( z(5Z)TL?HVNgD>> zku^UZHKj8wr;iX$@P{G_@4~n!60N$26X0TK8yZ60yT(Rmq`IoW3kY9z*ACIp^8u8P ztjlnL(tMo^W$HP=Tw(zsnwAbRt2uv(D*;GlLMZk9;DC&a;cN{zH9Tbx4JX8Wutb=s z;qXDWj-nb6U3;A&mojfa);uH}CYoYjJ97OT8}ARp6vxKgP*@y?6^L0ywS*T1Sa4*F zR}X|_Ky-RM+)AgO&e%N>wg#Z=)ghK&L)6)O%v4uSk@B0GK!bqV)fAnElH^DCa=4VxV zfq?T-I9WrKcq)V#6<{sYZll^wTJ3K=PbVzmfwjbfl9;FkzGwCN;m-I?8XxqUu4s08 z!9jt9?4`x<^_7!!%pG$g}drhY1eF>|d-0*whWe8wqI$4DNoil7N=Puquf9d_+*cVhX0CxV(6mgvu zL6KT$R^%Eq9-d-5UZg1|GQ%k@5L1jS)^!==N1c8oHwwfQLuv0vSC#n1X*O%qX^6ev zb63^zB|@VgX!CcG#lmy$RS~ZpgD!_C_#2Yh8wdtPznioSjtE=&h^1|@k`H*>)0(#x z!|WjIAVm!kc+<0jp?JTnlIKcX+u5Q%;HsZ5Fkafr7Z@M)0%C9957ILzeJ5X%v;CYl zS;%wAU>vXL) zJ(M6%5>MA^6L>aj@vKjJT!-(5!FL!V9#>aqB(!X;EFsxIcq~UXu5M5x@l>*}rL^UGmXi9sn@_vK+|dikv-d~be+ znN1G&olbYGb&2^YzPyF~jOQN#>}#LEA33Fd3Nco!d6OT(1^0P7t#&9pF`ID~oWz-} z7eO)3F9%B~8NetY6vWd!$0inpnM7#KL2u~EE@Sm%?5ldRVTPW({F0tLGyr;C1-P)Q zK^Sl=PwT$N!Vo#aib8qEZts2!m!cvj-9dJ8aVS2A(vqPqf6exayfDhUAM)0VJh8Nh zg1ki{&rEqYL*8tWSBvsaav-q+kyo4Y4nkh0$P+8%9gvqU@@}BK^^oTjc{ftt63B}e zd37l7bI6Mpd37moCgj<8p3~EM%266}1XAq<_Sg4Rv>p}BhN4G#9?Vj|r3ClsE$O#? zYx+IfMxWUsb;QgjAElYy_g$!*QcvqRsBgn}hG~s~yl=mwHvam7>P(8)>#P+f>I@Tg z7JX}|^V|{s{fFNwb#l;6`Bs7i#x?OnTb^A7^)RQUQO1C{|n-ovxyd za6a_@-Egqg{O%F>l4z=BKh&<()kH^ z6$#&im^;WP2EL|cmAH=-3sMdwktd?$V3=u;E2A%$WVweu=%FxT$2zkqck@kf2Z=5v znf~NZ8B+`TzY5!-`=8OY>b_|SuoDqzu~h5Z)goj_Fh5`d=Jd&1V#oT@WE?EQLE}jt zK5m*1-qOPdA)jX9#JxBR#I@8};7!2cVlhMfX%RHog^G)LXrCD(c7b6wpn!1(meCbT zdtY95?QdZw>n&KPJ_o<84VZ(*=Xm!=n*-+H!aXzx^=Py>^3cGM_viJqlYMC9M-~Sk zdHdhw$Sb4&)ySWU)<;f577Q)o5}yPbwl)dMrL-6BR{t$VB{Uo^6rld4;cB? zyJ_SN7R!-a76*)cFt4AT?M)*;@qO@-cRClY_YZn?=>;}mDdZ9sb9e}*|7ZO{1 z<2z!D%_J)yc}ldi>=3o{!2&}&pTqZWEikk*6Y^eNU})zp$m_p=+PPJZ!+%87|FWom z#21G8`@?s=zo739e@p#)<~eaJ_vR3eTr$>APA(;WQ%(uRG1> zezhq8h1=grDBOLKL}ACG02KZLmf$UW5DGV!Z{s%cd23z)61CwcC#w8-C5u9mmSTuq zTm-#83qB}XijXM9Z+g%-g(u)VBuR`?4A+Yv^qPbEOA1^?Fi)<4dQb3?y#b}U7V?{} z(5fNi8_ZQc`T;*W%s(1VA8jqik3{_U8117&A4qcS`b_}(f~5okRFoxktxG7zM0smgFzxKe^>sb2o+-V48nrMb?{NOa(S71vXLvKXS_a zGAgiFDFA8K`HpL{>l}q$&%YgtdkELfU3e!oM$P*?Ya$m`%Si2BRfb=MSeGPmg#Srn z)UKyubISq69er~*auiP*TmbJ^4aS!qgx0Pt0Ieco7ORS}o>g{uvmD-da5l68ILB#t ze~7;%KT{mGX`Dfb5FMU|A6IZ981Jhx(vR;pqMwFeN&2~g((`osd1*KQ^jue>pNcOH z;?HBAx?crZ``s$Y%4x!xGU1FMGJHQmq>>6;QVKwt6*Ufk$h>OYCr9Bvw=`BD(y-@b zOmdg=VB`w}cUW#bE@Kt4+EE*VnhChRIYpQWlQzrcyUqAtFa3|gCA3bAgE zI*#%JEer!6(us`C{Gw*f0cZN*Qe@piMBb2&nnT3I3SD!sT}yL_`9)>n$4gWe-ZYgy zE~Dk7z6jJ9cCU7lVYlBHTyuDAhtwSYimaKN&aAHd_Ks9Fhhf7(bGV2X4s`VwA*^T( zd&_9GCs0IP^o?B-p!Q!m&Y?KzQUDT|8xi1dpD(Gc{`>&F;bqp-IZ~2_1 z90Lh`EQm}V^N^IuKYp&OmeUGD;V|tByK!y4i@z<+hrjqu?VH0eV6NS@15}KiLrhLj z#CP|F(5l$U*XumZCay*Fv5}O827B{P4&bEKi(-nynPyy1Polvkeok6!{O1AqW%C>H z>zDO9e(~=B4fss?_FFkI8E{LYoWw7-OU18m)~opSnv?LW+**H5PdxZDY`unC zz66O|zi+_emycG_`tMAT+4Vg{Lwn&feP)}zDU|ltcF}OXvkgl7P$1*ovki-aK9IL& zw!uW!1@gX{Z7_3nguM44&y#Ioi@MQf7yA^Aj5qu&#H&u!9a`zV${R0wRa^8bcb1`7 zRlvnZ&NB4s0_63XWmu|ILSE-thMt~=yq2>J#=J9-cjqjFG4D9!)tO~5<{g5(i|-po zc@Xmc_rAfHhvl}vFGjwkD~tPV$U;%Q&X9%jc#$CsMd_Sk(0^+S{kK@rfA2XA z`tR95{nsY+-;Wgim(nw({<|6UU%SwMKSU+qk22D>1hyDdUEW%o&A`#aLO2vS3Z(%& z6y^81?SH5I@&cb>?Wz)|Dr#T=F4I?$Li-5NHFC=I6(cVYqZFWBm;Cf9ZQ+0*Pp*#` zl1_`?#G23}gLK{u{o@??_+1;~3*;uPsAZz96puM>6RnKsqlSN2fWv5uA;1u+d~;pA zS6_qrohVfVjy_NYD_)K87JWz*rf0!mK2}~5AdQuW7q5KyhfAzaq zSI$)(xFy)BS{w(`H3P^p>^Q;P)3ELD@Oi|a+rjrzh^BYKpR;0O#2|Chd(8z40NH%( zrHQE9^H05B;(UE~i0c(#=Ha?pEIV>KV7dlGAWrqf3a1nIX}y9D%Of`Qcj@yf`p%yG zK!1%xJwro%hkwO>o}o`0Yx03Er?-v>HEA;W)omoIE$UmM3D?R4~2;V}VPQExV7nh^(%@D#j`w$A>q#oB6zPVdf;k$i>Dttp8 z6RSKae4$}Y_?C)0C5rI1#!BIPZbiUK0>0%iySiSkB)+Lzv)D=`zJyY;D~bQEhm}M&tRxbV-(S>M5>GD(SV`Ok`ycI221t+?d|43G&4(lps4X5E&NYBbzTlHqS?P1squk&$x7j*V$=nC_rneUapka z@qibX1dHg4Mfw+&@I~C_P*LpRaehBo7EALt%4Uk`Bl?N>bF5$NxSK3+y&*py2J-ck zFi}PHc&&=PvWhaQf_@@?AE#9@baR-Fdt=_#s(4O2s3ncp4S& zsTE%`U2%T5F1XFslkP8ZG7%blij1+pBd z;p~SsSgs(jzL~NdXyfdgA47Hf8FsTLb@L*1a~^ebzShn1X*jAhy?6&I&ZzhzD!x=J z{w)=^>&5S-;`^w0F%{pa6@P<@57vvtkF?V`OrSpmMT` zl}P2Vcfr>~ROXT_100hbUmmJ#VeoP#R8HHn5UM6Sv{ck}8u$~HiIinf5dFGDlsPQh z)0XFaP|uX=dM8UqEI|&(7SGVdWkL@Wms2;OxIBR3@_;5TuZ9E>myHUgxU6`ai_2HP z)x{;*KCo}k^7el#;heTd7V+c`Go-LQmrTO4`HY&4Z6E^J=I>EhCX%u3z*A&wYq0m9 z8`~BYt3tDXkt#IDCjPy#?PQD;k!6b&ZPfhGzM6KZ2Gd~}F@O?;Wi7?b!!Cv^g0bNZ z`*Z_~C!%iM5bV!>Wq%GeH(|LQ_OTgVz~X^IPs3E$krApK>TNBK2FSyOVW_t)rHLV- z3hK=B8)3h90s=;x>bcWU;R@2@6hn{Sq#pC`L_~y`l2#bh&cjNlt;5#Cae$A^*ov-l zHe9kBbM*jQ&=9CFqY&hk5+LDg*rq!W(_Bv_ZCgM<={R zU4%{Q*L9@mKKZ7>CbfxYv$k&Yn`nYA&aGjS{NPToSNA!G2vs#tQ<*B>_p+X^KGl=_XXf%x7OC8ROGs!#DILyk`xx;zc$p{y!=n zDfK#d!A{N>O6>F~D)%dui;>liqH5tiTUDs_B9@A8rQ-3jcy}re@7c3cwO%Ar@l{mZ zE_(ql*rd-yFN)HrToIM)Da*kNHvSW>x9L>wD=IfwHvc`MPd>b7-@KzWpFzbxpyJue z`VGo;{z$7fhsw>Mas{&5cS+{Jd)5=Zq8Fv>Z0JoYK2uf;%O+-FOo7iCD* zlZyAJ;$>7^TXnF7R6JHMe*Q-+-j#}%Q*mvD!5Wj+6{8pLvkZ$TQ}IeFuB|56DV)0L zQF`(DRC{YGZX(I6tsK~DDjum9pH9W^qvAFyuB{T-+f>}97oSMQ8&UBnDz2>vSTRoc zbY0hPJ(!AHsCXpbR%7!ksrf=`URwaL{QqKa zrRG0^iho7L12z9=sJPVppQPd+Q1L*`zX=tWntv=6f18R2YW^E(>ZIm>3l-0y;(?m~ zUv4a}YyPc$tB8K6csUi<)&Xp@8_UrGU^vZQ-~!;YX*hejV$ymYRrxGc8F+%}hpD1U zt^}EfP?fqK;%WWfD&6?ZCTm4uIWUrnD$9ZKtK@Rv`Kf+Qy<(rfrE3zGACeZh{;Not zxMiF{3F*qSS?D-nfoqyTn*8&!j?bFmlHy(1{&%m6$0Y^u8C+6)aXz&Al45V>UAjD&1s)~mX3Fzfd@6H$y-D~igOR>9rsV%E`!JuQHCgvQFZ+7 z@q^J=1`GU&bg@+N#eiQDG8_F-7?$>ZeTg6Vb387|ja(MbzE43)xKO6VW8=WN^cmD( zDd{k3W$lU$dn~&VWvvUclc`lH#*O|^40(GELSGZl9UuSd;e5*c*N=I`{ZW)U*P8ig+pk{gF%AYtw+1ba&D!rL_0F+aKY9MdlgbH6dZ3G<1}Y9;21|3knPI<8o=KQ4 z>T~_C?+a5?K57PrB&8@&e5BtGgTsC;y`e4bCASh9VDse{_~RS2)k|v~l-&_byS|TVtWpwQsUcOhb5y-~HevETVPW?)pO7RMrtkeYkS1 z^uW_TzZ*Ltd}SR*tt4o+;o{j=s^2?P94JDj);3XQ-M6Pdv~d22tVd2&RBNjaSP@^6 z`p0swcU!fQ!XqMOa^q(r*(F2k#}JnvY_yYCArb^&+eLKJ6!cIo}JTkKz!!%8|MD z>i0Y^Ybxe+WIvoZFfpj>{ZVnSeh~#*Jj$K-tc?kt?9`&MXI5Y|vK>J?ZF8OD$#!2$WWxLTghc{8x1J>P5g;o?n1)x%GuW-N~P#}W zG`?*97@c{Qa#qEfl3&Ov>*;U(>VxEs=;|^R>iAQvNKR`stoRW`E%9}q^{+dlbD_U0 zV5@M-!(klzRYhZ@5J8O_%SzOa9pJ5nKeKhf5&lSf$2-)g5M{ec>s9V$eA8IP?rJ}L zG%Sx}%Qdf{8}ZemA&sIiFX_T%f6A{Ku|Khvy3`HHda*_@xPxOT)uIG&6%X^v95c{G zO<(5^RK4~4dOysNhI#tQD2f-#xOvj~k0LjIQsf&u;#+kdyTlupRgg>ACr?%CnP`88*2?rfLh-2(=&%I($f+bvxf1(F z)h0n8Gwa{=)OOB-wLGj_=9+)#)`#Yh`ZntNcxj)C?&)XmUiD<11!|GfH<@?0{L*|H zj~gcneQnjjdj;3!yOlG1i#Fi2lTr3bZ^N=Ev~=4}^YGnS4V7e_iS8ezVBM#bveJ#i z@jLm`<4j2`?w7x%e*&~lZdX*H1FFz;)t9}SKh!!&iT+|dO$#4XR&-6?72nm>34bwG zo%G|e3w_hf{+oQerp@fr4#UAec2y_(%)1^{r$?@nRkIe1#$kd()=P2d$U9k;N1cT? z8FxpjopLHgx(gug6Is0){Ws)+syUaLch+pu;>zWHS~rxO zCsC?MQXE}I`vzFcBrX4-{8#@WFoU~LyYspD#`rg_u(bAgQCaCI&xv2gojO}ca*ZtI6}@2k-VzJAn& z=R}=T54I36>n7lnU&RqT$E;rC6th?kk3gPfFV0w2zR2?LPoPVT5%s?{ZQ1}<2|GD} z=lJ|i2XE^p{;~?z`99c2y_+U| z!_Ko>-E|?2|JMGgQ?J)-Ha#?B`4FsF*HRTAp?Oo}1B(gSG!h3)P!2UpNTZdw?CZeA z^SVe_Wu9huGWpli=Q)ZlhPeJod=>t3u{wX53O$&8Zv!cK+cWO}O{8#EXEZMu-o6{x zk_#lcS~2lScem4P)JjgS@;SjuYTQ~1*hciFVrJ@GZb?Nv6MjiPy^N7+;Dm?v=c$_s z1;XZg%CJ{}(tnu9JEZZ%((`JXT#!sFxNC#lAdyebHu)jZt>TX7 zXcKKlb@+A(X}9%hkM25F^80R0>eaynA{r{F2no~ZALpG-?0hN|2MDnNZ%qS(&!42I zjp(>zMe1O=excd|rio;pCsx2Eue$-mD1 zIhNV3dB20r-)2+4cw%I&L}-lxBQCd`S4$I=%ZPvYTUw=omK4H1K|r)|*C)!Nk5{ zMPBTN?w0=lPN*(xUP;(1dCc_tk5*%5J*F-6*|lcycv!T#F+{1?yF#Af|(C%J>?gcbPsL8Re|YT01* zRt0Xi@B`QNsnLndo(R}j!(SwzLC;^`kMrv3b3I9YG~t#*cTG_isRcAzXGdR9JKPZL z%x2LM{hVN!UxL&dGj8K!tKr?{da>iJI_?%ogavsuhyZ8~(HwR#R# zd;V8-MUzq9@3gAHWwz^mF)tqZ-@g=YQ(uGXDVH-*T8Vu&Va)uV>nca@{wLUl#WNGoOf;EIDVd zN~5uvd+k;nYB1)hwC=?>yd8Cc631ULmZSS3mU^1pf4V6w14Nf! zUlX6S4uono~pfnbuObpZp?ei zPFMw9hp_@+i*wiq;Q(YO$=UJ#l{KbP@?t6h3zaD=Sj>DjmLWbGExrofbCV@!%{{lb zJ?Ghf5y&YJ+Sin;TuU7&gVPFk(}KAKz;6LXBcAs3F6~t9dPxp;~#>$@8U1Na%f`bkLmg{zl%i6 zh}XAj1N9J`gMs=#)yU=czn=RmNown`oBb`?zU8CrXiLkDi3rOe0fYlA7p^IBQ>~FC z+PBr|;|3Ex%#60YxJJ5i@V_{~cHJ^q`Y_vDpd!!ZnxW9qnnJ&x9FdWDV^ZRDp>L0E zBa*9chrvBF3}nXE&3mD&pjBb$k6gax-m$cga&XonNb=lB9|O3EDC3RsZpFJbmP6w` z+FPhM0s|prrJIX@H23PN1`}9zVFhD$PBH*4Oo$#YvYi6hAW|4 zyJSts67$u#1jjaMK&=7wjZy_zu1rE0dpqdQ>PZ-?Ip zLH!js&|jl5eR}9JuV=9#W_@PN!J=EdY%&D`n`koGGg;Cyre1OVqX8!U4zq00$+m6^ zoQ${?L{Zv6G8C?iqJSYnc9Rsdu+bd8Q;H&|S=x?fQZw$i zI2xc(CYg%=l&Ll~aJgilbq<`sLAKClM%dT)q?qUUv4GOy3#@N_DkEq45)E!wvAU5s zme*)r!4R~U(XJC8w{GBa^>tk(<+bQKe@YwEp*_m2AtajQaF$1~{#S;z41#X>?~+Ay z02{nsfot|h2tkUj6G7L(yf@v&yx@9FFz6tDKj>1IZST2V4!>7SCOo)}s7tQ-=;a!*b@uw` zvjC$ZKkSD$oWRfN9cVP{*6)kPA?!t^;sDkonIHigxGfbXDbFC~ zKqv^vxqx`C+eQ?e4k3aHpFq#w5q_!_%&}>pLiFbgP1{V2%!))&?>oo>~PA;XS?vyWmp(vOREH4=eno-Dtdz zawZEOkrfI9-(1#il^qu0k2al-$By+SZ}A(#KT7dk3oKwEU~PUT*N@JYa1X7a-rzH= z&;M#49u-Y zux>XYpxhrRf#Z+mt+oB~(zviAy=|20zap)r6G=V+Q{ z2bN#T%pcjckj5TwD-$fha}g&4vKA_sPyPMi8D0t24ffMZY`1sRoAhix9EG~E;lo-P z)@9W@7d1S~n6VD&%_lkM)3sMl`od_U`{n&K1jKrUu^JpyIy#rgTa~2pM;n!k;PWAn zgXr;N$@n{L)SX{V*?Cnh#7w4s1J?-vG&kmBm;H_OijDPIljR{Si+9*>jY6YtDqk`L zFl9*mzb0}2=V2=fSWv2u*xOO~9k3A}F1bOhhU>h$n_ucfMcrtkL|Hs16pvmw?jZu%?-fzP$SG7wuxBW?>g z;(-ehq{fZ_MZgez69|hA*k$TC!WD`2^8mq?0~5RDLjGCfKyB)9zgXk}`4D_|H;8;UD41m&WZig!$X10T zGY)Ev1l1Xb1J}Vq=o1xVet)u<=vsURW?a7loJY`c^R}=@lK1;Hk9dA#c27vi&*Awf z5e0oKdX65oQEL+=BTh_Xk_>0&Zu&RI!zjlZG+Y>;@AVbMViCjA#`xpjFY#)Wzc@44 zGi#D@UP%c%sLFYRl*hFqRxQOALl?Xhav`XiGZl^kE3BHB{#U+=z8itK+fU!z*^d|z z_~EMskC>6)9pztW>BUDoR+AdfLm=h51mQV!*@eW}1btb>yR2;l&Swy1;dFTwU=V&w6{hQDBCvWlqx z%zFAC27?2cWz<)xoUA{uSPZGlDmvS>V=ta%(~_^^eJMo{=uZqF%XAsdZp@HAj}PDt zk`Q^$kj<~6%mv?xkMQ6a-*5&TGrUE$7F4(m&S`2U*JhtY%7U;EvzEW^IhdXZ=ei}L z=3uEbi>yS8;pG}cCmiV34UxqpY^2>(a@}k5ztJg~>?`@v-tt0;H89!}xlpQ+TpTup zeE9VDv46pzU@dH1bc1fB;I#M~E#>xizr)P8IW62iU!n#+#mz@^XFB#`->}0+-5fO` z=J!|8@hJNkOj-i@eWC4y^y zaS3OeeV7X`Y{QJuNqV(>ue>ZxL|B8?Y429tv^CXRx;^HkHnO+dNho`ddFiYa4;vo49jN zGxqEq6NKj9+o;GodpsP6Ep%>}1w0$)mN_W)k;7R6Z-a{*ZCR)P$4e)B3; zMd~OO^Z^;(DPzv{KH2Rb2vg`N0Qmo;iX1wzh)y(Z9;UmllOp~C7@mr>^@^&p<#5o^ zqbbMUO()iQ&|aW>&KoP4hB1f%=U#0lp?;xpNSKWKJ95$_K8nA0L_3w}>i*!O^lT_u zUQ#4HqJ?2h8ArP^FCs;p1Z zCTfc(-=rr&_~z{q{rcZV1!nP47WQ5rc1}8o4toW7KQ!;W&77})P|@umq26WN%%?4k z@xa!z&LC+yTv8^KC6(piNz+zzB4tk$Ur&VWi#{>E&3QpUjlRucva4 zBz&R}e#yna$+O_J_$}8@81|m{8TDh%H}8YJv&yWyOKuoE!Ds%?^a*+}I37_JN^#)G zHUA(4n^4R^rkkE2$q&=L&QA{enOA@B4cc+W&UijszKzpNKvRj~ha}VH%M|EuDHbnW zG6V)E{2qFTJ2xJqo`gOaRT0OMKluA%n>O}IX;t|U&aN_2<2j;pf^29FhyEmTg0-@ zt}3yl;HQCcmWL}abMA=tCQKsS2>wu&c@Q#Nnkx{t@}`+d?s7lc(Vm-D?f}?m55E(} zLj*TGqrPHAs<0eVD&coeT;g)yg^`n@M&lDFm?dCQ>KzmNZCfo6_Dar&G$Qpv(j1y= zzea7pH-ug?s?kE9eb7!<2opa}ub39G*H*7tIMV;ONLQ|=Dby#47{L^-lo7Z9aI^qJ z^Hv>K>WGrF`EOaY7S4{~!^7k8wmWzwlMvoi|YoTN$Iqfchx(yIIj{QnR;Uyu(96 z;mwF}#>@&;{ii=o@Xs>IhPTFFNk2gI_#d#Y$vyTXbg4RuQN?%>^5JGZi6= z5MZ&O{yMlX1;(z)@R709mHn}=tZ(C0y1|u`SYu{5gesQ9?e~XkOaxl!AKVm(mFCqz zGCU^%z+uUs3y45Cg{u3j#RI07g+kw2R3J-we7+*=!vUGcmsxI1z+6{bm|zd@b;-^o zcW8!L5jkW#c3zaWe!;&&FV+kNji4j#C#lG|W4VeHOj$CAJ$C94vCdqT$|fLmxNV>s z$@I;wGmGZ&#;IjYM2j!0xOCI0o2Vvc>GO@)GSRl2ZE*KlM6%G&mVV0MV+{8RPO=vmon6Y@NWw@C2?Dnz$}V^F;cynZ z#0_7aHi1FCd;)GAPXBqryBlesHpY{Xrpl-|;J>oh7g2l{w4l)G+kqnE08jY)56&_I z=DV7k6f%(8Ep9h>ccOpzQUlgZy6({oG- z60a{HP5@fP3+f-x7ot;Gs z=ukvWtX)65K^cfY?ChF(3dxYqs1t+*?YLhGbsNSMy}vk+`T|jdxxa>C>t3@UTc6f_ z2u*>iig$00FDstHqG67=F^=?Irpjt`{*opGg@S&Ij(}%*1ZikO*Ionw?UECsZC4cs z$SeuD+8fW5fm~myx`gi6acYLDTQcp%cwx{Z3{_y*$!U=54;gWQr5_iRz`CQS?a_Ip zvUq2$_VwP#You47WyqBctbQGK{OlBPGjvs*00`(~1giI`!TgUgVgU%5`rXmZqf1I) z;NP18!dpHJpzllZ?r8B*$W_U%fDFXLrb84+z$>5`%F360ZPV|$KiZs00)&6!Z)(r2 zhIssdzU+<`=E*=B56iy*?n4c)ZOpv@Fsj}uEMOr2;^AoZMHl^a!JAUe4}W)2zK_F%=h98;V_C{ z4#%}N{ZWrY%(zZpQzUHudi@z)F-Ux*@MK}|Vq?`6d|sHsajPOYA&9gyLdCmHtb9og zVhs|Nesz7*gB3hy(EDiV%@R14P(l~6x$MjUxuABpze z2$!zPUhX*_hTL6wZ8P(;ra7gqbzd0bG+VNSsK0a~&ZO^y_8bnUe~~!WHS1|UpW3vT zbjRvV2DH27ssj!(=7TlCU#^Cj4%MD;?&sHr(_{j(u7}JOIh&cHf=mEa7wLe$=R;+S zNJ=1$+08(5ikfh!mfQ080}?e)mf!~4BUHDn8WYB;kU!ZDk61?} z4?;Lz2j>xCV3RLjCS>t9B`6nmJY6&g(1$^#P2z#sAWjh6*;a27KQmNaoaL^S2y<_&-Y~AJ z&=|tb@!9-g*wAa%JV$qxb;{l95T@MoXZN}Zg&M=<^Vj;VmniGK4xo&6zp|>IAUh*a%^iUy}UflTe4Oc)h=Un%8)^lxDTp% zo2Szb6~F%D(c}2U^CupC;5WZ8?6`Mi=u>{v*63{GIZ5V!9J%#ksc(wV_b{c;#qQ<@S$cL` z7cKQW&CvaJ+K`G~X=Lxc3hd$RK3!Vrce0eW^nckzzwUf7Z}c~|=e#z#PQ|Uaba>O< zNfx!T-oZ9n{kas=Lf)Rm!V)xlb$U)(AHx#g|Ibqzzx#0MKFW|VZY9|(ez$yjKt!VFK={D!x zVnE0tFd;_f;aX=@-cpEmPl7j;Y)?W;uf=Sy7$Kt^cXYL*d*ki65V~q{AHPA!vq6w& zYASz5xiWa(kPKZXcuu{cYkH7bEyZ#NoE7CanKUAMMTpo9nxI53Wkx?+s({BkM%=zl zKlJ%BPB%%@)Y#PCin1PGz(<>Dd^rEeCro)e>(Viuefs;IN1J?CajAwqf&1o{{7-y@ z$iTN_=CeMOK&Tsb4;JL7XP^U_=H+P1+hcqIp&mn;?8q%nRkvj0o0mUse|W-SFp8hD zPv3Lx7=ha4FWE()!kerj4a*W(LLOIhJp@LDp~-a%zSlqK%fnfJ4n6FvHnf0V+C#)O zb)_sk5;LHePn9>|>GoOw>3UP-N-rO|MeSfF^fIve1{@Mpbp30Ys~+ZOnT=a7km-Ma zUN!-Gybl@Ssq+m$LRxTlcZ%p8_Wi4x0ExZG?CJLyAg7-#Pwuiw>T#v@pqFsJqH9UR zrfwvdK6m%ew^~T!IwZzY$cU3&;tg_SS9RqnT|5PP?BzymWm57_UgTK<0cUQQ6hV182o|}S|Z8$rMjU0QK{my zYaBoFOaznu6hg4f778fOMDq3=U?gvUW|y37<*5w?jKo6$W^9v)s{pQR>YZ+^VD60x z1^|5V#|8 zx5IFiBh`9#eF7}5gc*)o!>eQ2CP4?>wJ$b(l+O<8605+6A7@4LKVIGx+djAdIN9GsyYozO*}RcR5}7i+cOF zI}`?zhsJMY23pwhDj$PK?wsP@{R#ek6u^<-CG!DJH8oR+i7l4q>N?LTfiBmEV!e-L zNgisO4ydX?<@LyKIwA3VC-an6rYvRdlsv#H;iD2BAtxfNcxy{6RqmBC>^_)0ch&LV z>K)PU&v`8K+L4%{!vm&b7%8daRd2RRVm|e5z{Hr@Dn##-PD-p-E6bZjq&y0b9}B;; zK5H%Bwf`?6Bk~&X?Kgbl!$kDT455Zlfvd(7bDIWNdj4mD)V978E>CG;X<@M=(7wcI z%b(l8!To$qX!7Xft=gt(_XxLslz*ZGOI+d#mCU?h_edcZTignSAOHt(sK8`V!lg)2 z-*`yHWwA$Ff;4<0)9)Thr}VMz9(kxh4IXxq5;aIzflSK}4?A^>no!pZ*(OL@*jU9i zJDhb)*ii{A+L2uWQuKJ?ug`j~fYRj(kV`g!9mKEKkXc%mAq1#|4d$1CG!avZ0{t)@ z1c7miAbzwgyOv$59uPw6vfn_LW*UT$4I8XY$|xFQ81NwjVKqG2H(dea$X18DFd{^? z;pqoAKvi}Givm>BU=o3m0|PwrAxhYw^?Eh1WUX-orvOwl>k9N3j^sBde!xuQvk})w zf>dvOA0#z~@YjYbj~{>{)!{MmNE|-Ik=+gGPa7;bZ67gUeE?KpL$Gx3gPJ4zx~!4( zCYv@qhC%?b7d(L&OB+Xsryx9jz>+g2L1{>U*c+e@AHo3%@EZw`vJcj(ff4GCgVM5* zCRYIcV%`G;iZ&d`WdR?3%0QGN0gO@*jXz)<$Oyiv!}*Y-ssluVI-Kx13&E-a=b_+1 z7-Yf-Z_^OOByjyNlR9wxA27oI`$IAiJ8bC)Ax4Ca0u;cX4XQ>45O)JK<3m)~-hjs1 zV1()Wz;+`j;Aa}BIvYlqeN{wnlyze%n}HY@o+bc0KSPneSU;Il9g@b zL>RQdiZaF#DBv5n%za>`5fq`ohLG>s2YrhK*}g|+pn($+f#jIAWe&hP!cYJPGHDs0 zcr!Q<$q*B1!}&f#f&A%+m)-+VB`X4DcmhF`2?Oo2At-PVA3Kqk;Jr2+3z6jdS_aCP|OQx*c*mvV)Y2N9uv1N^N9k5PsAUF|<7WFws4 z&tKoP?alu^OmsuvQHFQd?9@W`b*zkkPr4=4U%#;kt>dVy^oL@txzGpzA6SqsOyR39 z7FNAv0+zALt#yu4@cxp<>3?Pw{=D6#;mWi5!UH858aP+tmPV`$IZ>Wc%vekem&8%$ zh5FxPmH$~^K_sxQE4>qUuU(Qb8Qd&+Nu{{Wc~Ka!aN~aOJ)%l5B;zYcS#b1>g-BXP zW1Tqw&I!0(fzN)FA|_ko=9R0n@XE(f-+$?mjLPhm7^8hZ9_rsbdHLRg@w3j`+&o*1 zp6*QUYIig5mTo@+m@Zk674pRiQnCUncP9sTE&K6h^)aRqy;@stFtZ-mX#J?NC+M6s zJ3;|eYiKVazDhQN(}8{Uo;zCE#d`1z4Mr%%X3pN2apy{@bzBm4^~CovUpN8&Bm9pg zU%bZxy(H?ppe~#(wtP}5>uuAXuP3jvs))X(6ulOFHxu-0Y6dd~f&W9BrG~V)s1=TE z16JyN@JyICbJ+S~$j$Lfcvt&;L3;ao2?Jkhy-)x%nHo(sQhCu4alPtm7(Uwcf*~b| zpCL%)W5}&(bznra-Jj1`P)hZVi)e?Fd8d+epf>rx|FA>tXNZjCd|<1$G!0T90;^Aa zc2Cn%CD}pMH`^J4xF~DF*By2^oQ1QT(X!+F?rY7yE()I?!VB1eW zDsY1D80>X7?Q-^&D-fDGvVzjM#e10jwG0d82>Tz3N9sxH$uCH6>dAgw5MD#iXbefU z*jCmA)(7Jnfrprpi=bmd(wWbKy-PXPPec1~wo}6U!pA;f&5ZtC%;+=rOZfA9)S&{N z%v)oL2lEEdSGjBsPintUok`I{)Tr;rD6w41RlZ7nCqoM-A(>7w4=*O4cv7YcuW6HX zh|l?>8m3mTTBbk`2vK^K9=)#3oa=>4Oi$aTajc4CeVC>Y`L+{~Xh3e2)XS*pePaY< zip$gj;?)DgRz9)V_7Dm?d+mwZRMW4JRq$`j4XeIlLCM9f7b-`@+@DzxR)Z6tBZ`d2mj-u*=n#3^hIv0Kqac3%|?HTlI3i z!mheH^mxWGjc@Iyav-i_a5RYt1p@TG1hZHcE>KNQbG)A91%Fr-DL_D#eE;Gt zOo)+ek=`aY|Kn-H`dPrE5_DAb$I)$t)NW<#Yc-qWWh!m%dE#mK`T4xEqdXLnUYXnS-C@J6A?~ zwNB#uV}C81!$vtNOhn<_<5Yt(4W=D61V{C}Q(LQ{pe`QG@LLRlnNwH#g&}^%S&l`T zwRDgR`$n#7%^j2g~lj;f8W&C#uF8KK4^1Qzzap=3vQ12 z{S>$|wp8iFD>;zj^ra8;7HUOM+VX;<$Vgzn0z=n9FAhVo|I2pgBatW8^OGpCy;eb( z?$+izeU7jA*JQl5p(aUBwp8<-S^97yrHCw#x^@Xh9%>ISZv`U;N$NbSZ*Y@V!BO@V>ttf42zuiMcYSqFHLaV{2*aJJ^Dw`@3jk=?%am{4TW8C7Pcya?tsT+u?^46!YY!%hUj8h-%mC^)bVb;D~R=(QIooftz zXVJ#7vzR%hLD`EnXxIK-v%@|*1*||aJ+n{N)I|G<+TF37;B(7+C2E(%@8wNHbb}Jz z7Z71|%M~lhBQqTRl<}E;?>A?^t%48ouZ2p>Q2-6)9#5N%&TdK9A03{5Dtw?&{?2cT z$DJ52qi_pMbK7>SpzqshkY}5kKWJC8piH7OkJabQv>yd4!0~=44BGss8Tcz!@O_WQ zW%wHAE)mmLc~vnoa%c5BoNh|yzyCJz7rfs}XQPGS3vaHOV=&-VL!N?44a>+HLR3Q% zBZx6=N(6R#E;JZulE0X_sijETI`4nk5bxNw47C)D7wxFFq>%hv-o?e!m7V(Kd1z9n zHZz-iy{q0<(^-;xjTC3SNm(W6IbXI_vb6X+LL$tZPbmmqrWlUmTc@K=TRNOSBPI#xz>bqc5qN)Ei;aka0}S1=!(s zKSfgp++*)$#&j>46Z^BNiX$AD%}H`5HP7+Xn6ndGNuBFbZ}{0hi#-=CTNu%tY#6t^ zwU(|ZIa;))H-+ef(2rfI67@HLDDdhPrUf_{MMi_@;L}7YI<3DXBjE7A2QZef`< z_b%rSwJocj%aVn{Hz=V>@61%62==ahzpK-cP*t+P-hc%ShEryj^vu=mphc1Z)?uKh zAH{=a;kJ$Bhi8s!bIjf_M{!h`Lps7!*BT8#mx?qTbsA`45PDgA#D`@y6TDWcQA({5!>cfB{kiQ znQi7NvmjGCPJ1{la~=ZMY}?^t{#f+9>Fv>g(z9p5N?$10^obxX`B<@zW{|OWKHw^K<8K`}~t5+})nCjix%iHWh0z*t7U~^vjwp5t*R2i)^_lQoGF3kRQnX){~aq=1>>!DCYeLL;4y0m&Y${ z*jrM4Sb>h0O}#UIq-Q!!u^}ur3O@w?u`3-!l+VzN6T3X@6m+z!>1AFg-KvI8t;fRr zH4ncF@zB4`UT=gntG3<4eYXr!3T_0n3z^)HsHt;5y$>eH?@eA^8~#@Z+1)SUW+k90{E=-K_nvPUCTAdTN(Ls zr+yU+iiup<1(g%ZtX%~t37wuK&=MnG0wmQl(Kg+5&MDFGYH;5kFF#ewKe6OdK z&BlYW5P=P_&GSO_qi?-r+XWl&e7>W%IGcHiy7(kiqCKfJy_qKhgp50yGJIG>1L6L+ zCaKf)n!=Vy*^_u#887%8j3(&NbS~r`B~B{mX&MtreZF*MsLr|dGYp7qdqXy*I`dyM zctKYJA1xJch|XDourIiCF5(0wx4wJG0@rE?#qVp|oCNaYfZ_^=AnW(yQ}Z1ZAGW9Z zohmDV7C!6>>PdNer^L_xZR{*tbeqzinoG`ZGTm@G$?Useq@_>( zkCv@|yc30i9~T0m@!&hmQl-8@3E|k^R}XuVW_4=+O?m5sgJ{2LmEGXjx*IMMeV`82 z-oB7p54Yv*f2Kulth)+{p5m%sxTuo#KaOAyQ<${fCZvFr%sf&^0(r~u_nr$f*{ZA_ zb=IUgf@Hsv1)3&&fivj&pVnxhpbpIPyaG}Lgul@0?7p^Ok>ss?A7_{wDEVr`v3X-m z|1nAG&;j)g$7}Asq=AnEJd$S3xh9-6Yc!+&~qwUQ>`Iv)TLCE#&E9;K%*R zQpp#YbFUJGr?)F-6*4|AVNY*u?4iet*`Ye8ixr?J#M50B2HR4_3*mFIW=t>lO2sr` z_5I15X{BZ!9hYT@Jiaky6PyQZ$m` zbCcIk-hZpp81nr-a9gHj3QSHE$CTaZuic;g^4ux@ad(1~Df7|r-=h8x8I0L??b~Uu zTA5@rxH1;eho3%oMqBe4#l+pCsWfV&@hr1;SVt8|Fgq27cUEV_J?Q>)T-1-t+NGgV zyV@!oxpVPsZR_>X4RLD5x5orM(dr%fJ}z|7QwrU$ek}F^r~Ui8tantn#&Z03?3JaY z-yd1oBmP+{foa78H3`<^%@CRUi*GBHTSz&Yo}l>CO|C^D*Jqk?B8;BbeXd_7KB8vk zvHvvNuGcwWHY8D~`@XRGAkA%FZmD$%e+YSHJ|1a=K3Ce=&VWs4d*Tp1*12n-5Err< zlU2z2>EuCO4UEoLfE@jl^m5wAs{M9jC0o5X+(Bt` zM4z+mm?JIHnB2V>d3Wg z&9#gp3Psnu!s8gSE3UHTdb_epEwIWf{89iCw@<476y3ePPQjzVE~Q}b;AO>*7p*mG z&-1Otr^Ng3dP)PHm)V;zIfFuW9=DXtmCNZz&K1HOcwL`GPsMc#@@s~r`GSPo1}81o zUp*o}m{;a@`>nN}gBJia@uagg(G53?bE01=S^xRS{a~0;;bW)P1Uh6_@U)A!quca* zY57%q;@i?v;CNIuMQhWI@fYsPBU2H4F10X>!CD^L)I5GcO|Ywh|u&Z?$BJ_voU#wG+HD zFYNN)yxTK=l`GzBs!-bQ1GOp$(9ZjO_qsI5d^~+PCJ03nkLGXNjY^T4>3h5~HiiDpJQaHQ$I2f3W%G^qCtRF8 z*kxH-abXKxTA#7Y2pfu;i>1^1XJcOUu|I`={j}uwk!V8O$i&s`h;cQ*CK2WNlS)0R zAkfKuMpmV?-dGc5@-me4yBA<=v{tEoliA7#<49Y4TgL+u|bG z2%Km=WSgj!uTl=LT4S5lh%884l;GyF2PWI%O9Q1!%-@W0{&VMww8VL<1>x5cCmH$X zDxhkkKEwa%YkRMn$VeoNZ8KQGAXMET)Xbo_i3iLAskaI`-(PCD8*aU>#k1Rh9K(Ay zoDqee_bk{q$uQJIeWC=d{~D|z`o6yaxDrU>dOSJMe4Ok5sz5CyK5b5k03&lxwz($JY*%ts{BDHZJV z3Q8oei^Objinh>x^_Mag-LI3xwcCFPLOmV7e_Q>>`DRf>xX0L*Q zThhk-F=|>S(y=0{CysD<@do9JPN$;cUqW)B%?(cT#4^KoT=mPvhNKyWZpF=Y2N{Nb zi`7CC)xh~`?TPBDyU*xds(iT&jBfcnuH=*ju|evFJD*pbrnO&{tC0_lm&hNNC~af~ z4@|wJXFyD5N$lFCiVm7;6haTqQqS_bKimX$PQhdKo&ZVxm#qtz^jFU+Cyzx}rUFf% zoySzUapYeYPZ?MP@{$92$A423^a_drQVu(Xw|10zC%Llt?6s*;K21)T+Z{Cix}i1> z*!BICzNfTyKl4AT-ZChzu89^VxJz(%ceeq8yOZGV?ry=I!JWaK;2zxFgF6H#Fu3#a zKDWMGb^lDAIdx`F@6~Hh?djd!tKqtK-5-RudMl6d;(5F$AmUN6^wdi2w$DTYg@c19 z`bX=^$x?a41e#z7@@3$}qi9Lu8K&N`IhSM;p@`N7{>EIs7(7$Kd#gn_#(Tr&FY{e{ znoAP2pwKPJ*|NM)xB2q=ie4zr<8akX*u%Sy@?{Ft1nnter>o>C6023XI~k=_9sRIu z*z#h`Y!f>r?w5lLuIQa%>VHG)LDa=Tr|s3;=urZPF=t(V>1Ma9f~*hzGL8ocjw;I> zk6)Jp#|5DS5%g|Fi+)u_=pbcOvwjH@#ve<%Yd1_={XkyEf4#HBPeM^uXzhbIp$t48 zx?=$ReL~b_Uh4gq7Z%pw7Zx;_08E<4K_%aIk@9W`%L&7Cn-{b+UQ-{>3DbV^$)&K| z{a1<%3~XtjH0yT3A&I}rZM^cW?MXfb59{N2P#^^5+HQIB}cX~7Zm zeqszP&{YH9S;;itYkM;zx#R9zW#~oCqqfIQs~om`PJ>*lD!F^Nz)3wI-=hRoJ4h3m zP8qz*ZCzy-xSJ<}U7M%$T3wLW{akVIZ0%qwDZ-w`;vJ=1zbV4v~g_Eh&&=H}f zAAFfPbW4_Dnn)j`H&$m%%@Occ9Ate4Q+^?wL_~B&bZ%OKSSl0%x$}g@%rN#Qx>XMu zbwGf99g;QQm^q7_`S(TSbO`=WRTrLJ&jjE#t?qBwhbC}cR~YQ>CbPkG{{+;4ofKm> zcJ)xB)3KSFO}W3qB~<4PL73z~Hcp0IGJv`cq+@4`yO%pgtu2S^I3mL@8e?jN(SM+l z;@0&H^PGw;YF$30QP3B$S5dz+D2ZAx63T&ygk>}GeMESuqo zM>FT?@$UhV7)jr;L7SFB3}@z0meYJGrs>a_xFL;H z@+c}{dUv$#273%mOE!nU9_c{Y2uX+;hJS8NL%^{K@p64KyR$UGgy=)3U8YGa=P{XY z^qYp#^<*x<*#2u1js6IZpeMoEhIQOGHbM=3Y6oh;-cGUsf?z|+8F4g>jJ_@?F=9z# zTO!on1KRWSc;k02J`5=AdL`uLW0s@Gn7|GUrSwsla)Q5G(`nlMj~02bY_T8;>WANt zyFH_~OA10%7V30?*ZBD}$Ox|A#AMT*%`%K3k$E4&)hWa_eT{u~S4i5Y-UJm`S^CSL zBIpq2v~$4R^Nz}}E=~BEeq~NxtTo@+$6U;Y&pyK$J4Cwf5yJdG%0Z3UuFY9D{y*cD z{enAJ|5VO9s5yLu9;|rT7is)sWo)%H_l!O_!PWLosLqic{|Dr)GnqB%hwu4FH;lSQ_}wbm!>L1m~& z=1cSJ?>p5JzoXZ(g{xZ&s|P6Jjcm@$y5idrZIe&a_V@>9A?}_1+WyOwMi$|{Exlv) z_%3TL$KxMz4xLrq8K1J*%JZ00`}a+yQ>f`hFs|OWKPPEjJWqsWRs4n*%NWgnbI0JE z+){k^^&Ix(wYtjtFKPM5FG^4@Od#xCm%dRD<3V?;_>M0oaM&TQ=u3aws#Px+w!dS9 z8P2Hct@Fcf=H-Qd$+o>Yg;3dFA$!{={O3&^wI1EV`OCC}zn=1-0So)3?>U^__-$)G z?t6KDv8$Wn6@N5cVLIAz5?V^G)@x6&QEL{CcN(DOP+3r`gEeIur>2Enw0KCq8fcl7^;kVqP8&}OpQd=p?;ZlMpRL`1C^9zdTqAKop*BchO!&OKPl zj8&mgP7+uryXKU9exhkgb{#4~Vgb4Nykq`Wye4uxMyAIi67cVn><>$Hm`$ybM$DtZ z()mc|6pnLLa5!X2dF6NS|NJX*|NAT8wLtn`U{u8WL#XVjCF`&Bani&F-=_2auiEX; zK8kPrJ1-`;mKkjdlq{|2VQXe-;bKCZ;_ga{{}lL&lld{Rj_k#{B(KZH=#qL==dj}i zyw^u!>eej9GD3OOGN3E*DIzV7i+8Q-gvY7wPX$<&2PU>a;l@A&Q25B|lfEkHrkAXdMo<9m_Qnd1 z^cc-&%PEun)GhX>XwB#IBI?gqXne^hX+n6g+(!j%jHXXPJXUR!lKrmAKafwQc>4s6 zELcv_^fOOKUr#w!^Le*0-()Yfbos|HqmZ^VaqVrbG9{SE(Kae4o_wJ1~!kjz5xF z`;+%A!}VvLjwX(y%U0{qyzkp*nwk@Y~Hn!QD7Fm!Xu^MZTY(4S;|q@91O$M!Apqry=AV_D0$#oGE4~6$B%M! zQS5X(qU^^XlN~5PIMQ>B!|k9b;q&c+=@t7_XiK|JR!h5Hml94M88k`=^WfC@ z@y&GS#PzKp?1oPvJ2~|nn6`yWp$%MkiXicW4;n0=jk0_$Ith~vLV0uJS@XaDOS6A0saxfx6kFQSe_~crR@S;?FoHEzvG(xdvw+U2G z>77A}v~RVWCXzUdesZFE<; z(6IT3A{lGtuRGRz*A0xns!4{I>fNP)9uSt@#2^?@FqLCCfoFP*o&98ro4|HW z^dg+FE>&tza%zv^!Tu}dRoSW;eqeRg7KX13?cKN>5evnQIIWFaN*N8^b*LL3Rr0bV zvQs!uOo+sl!J#YK#`h|8dc?&k4(bNP@0@TCEb5_035Odl=%GhMOpv>@3GW9mm#SW> zRT$3ugun<+fb+s;J%pDiwQT+clKH(-!h(pu2`K-IdVbg7b7+Xv1)qvU(N@lmc-~;( z+efu9{!Kn~;AS57O#yMd$$mCQwrjfLz+qDfgZ;D;LpCO4>>cDm_A-bh)I-f&D~qzW zcGC5Q9%;o#7rVPv>7hVL56JmDIPeVRsK71-Kbp*_xpS|W-S-YD;1ufG%gl24Wubeb zg5V;XcWe9p4V>EH|Moz??s?ofQe?M{c7Pbj&}??0>wvyK@=6{;D`$Uug48F}rs^aH z&AaC_ydbl3pt>%x5gXI`h_uhGlEEFnwy%IIx@h-eo?*CXdyjpFxt`{Msf3E41pAkW$V=M;w8Dica$-nzr%&pdF%6Sdk6Pmep5J#N)a*X8s`_e8jfyo za}<&J+Y>N~dyILc9;bc)K1}u-;zSc|g^n==^^U}2w)0NLXQMbXqHW7`bPp=EM=w{! zFYKHy)sOzUr_kizIo*d|dMIPBfjvvr!@jL{!EZ*5ab!_yH)(n>{Sd%@I$ONQmy9nV z=t8@ORgw~5+>HYMc|6f#*Y%+|@f{GwapF|yI?LnK#zSnF9UM3m?EahoTMZR7pA=HL z<>G{N{mV&-vLgSXP1<*`t*{f7f@xu!!#iWhH+U2!07M1feArmBI(*}QNJy#Z@IImO zC%|G1>grkxCH|0R-{Qftf)v*xuJy1F$0FbY%HZZr8h)gI5Mb&C5EyU$Oq#N#Kk?Sf z29=m{9iN;@&e+H)s|TK}5|vBN99d`GE-n`y^;&TYjirDxVMR`?OJ!$B*KYmNszqn2 z3~mS2d@3L1XF4I6eL(Z2Bb+)Rplzv&6NJ)RD$UzukTloH(;=s#53KFn!fXu4ew$J# zo$*l?s8}mG6SDT0gI6Ltqe=PawvA3`^R?uNG#5vRvKj0x9|J-(xNR!BJx9Dx25n{C z>g5#5&O9d)+#(Yel$HsUl%@9Ep5Gi;&)&4iCxa}Tj`{P7OOLMfiY=WJ@@|{viY@oe z6n!j!v(X=aC1(`ml!XFMv{gT6)VOww*L*yNS5sOBS2BWqOe;jA=yPbZz{w*U0y^qC zw*HKA!e1qWtYLI+0o95#e|B}itPvF_&VdoP$k*v0GDV+ZLGF{28cAgoRv#R7pZJrv z;*q3UNo8ew9}l#Je9PePiZdfApqBS}S03dU(7AH+1u3(S3$u++mb%Z<>1%T!*NOEn z#Tm7#w6}f9nN9x1cW^pr>ci%AKz8jOh$93SUU{M|f08M?m=kypq!A*dwOJRzVSX8j z0Xe6D7~bbOFKbGL7eB$)n0SAZP*f%=p>3a4_92-oF1v@ef02#>g&W)gzMZt|?a&If z20I{DeAhUFY`IcA+J(>9}G?az;aLrp8a?GSKo^|L`$ zq}Hnd( zx$cN_3&C)fza^j5zJ1Q4$F%t(?~Beo5LbwhPU!ny zK=*x3B3sZ(Z@9l+SwPyeM zgfld;O(@bRMA@t2x*1+APp?3TTM; zKc+lcq_n&(C{MhD3(H7K%0NB0)osQ9TRZ>J__(O_XpwjEz5U;RlNe20yH_^+&lyiC zpkUii%lNbIhvDsC%7AHa3%ZIka48_s_j$}>*_r*dTes!HvT$4>MS7u;nrGV&y~QTp zq^66!EuDL1R&Jr?p3-|@&uxyo?ekgFiMMtPXai*9^Fz&N$pF0k{<*?loj%7+|BuA* z|K}%T@)K9njFz_|$(bM$8y_829}9!~|2=yoY40O^R$>0(HCI%2tMSQqZ``Kir`5_& z9A+B=zy7(k%T&6^lm8Fj#idfqIvgRCu488vpU)tN?MF`fTu5TV9`Vz+CB@4di5$)q zn}TdDgrgPWO~) zsyUzAU2tX4L?|U#KxZJ!4wVRtAbTE1DST`O8~?)8z0IFYDgZUibvUeL8U+!p?mgBQ zVlQ`QU^?~9?7-;%=`MCr^^@-b!)F5dQMif^V=b?PJ5po1UiRf}Pi)FZcnEzb0Gp@W zT`2u1)lta^guASVP7GJ*uT4gR+EADTZe4*N8Lo~AwU=7QQ{(@?EvM|oj@d?k*gk04 z?1YmA+~MCFeRh_bOq+5^{_+83Dz4s-D9fHQVFulp0D$HDHIxnPAFx+}Sb3`Mri0iE zC;*D@N)B%aGsu{&NY8eS9Z0ffl9zUKk}sZ*?x=mB;2QwZXrXr#1F!w$fRO;thNism z8UCCjXEf+?ikU;%mj{{$hWpO`#Ru-z`^H%){^6d*lQluvb+_zlX!{2da7h!&_^vO_iNtgG}Msa?v2nYU^ zey!gl3DS>#3wdxTXI^?7fdP`;>qXMnTHWi0N|3x}PY%4uDc75@txKbx*?|noYi}HJ z{{s4M`1GSw^E~vqwM>rp9=y{JpbRLAy|+p>=|uy!uuVsvMhj8OYOFavqc7&j@EFkI zz~0#K=$y`74H*Z` zvVEl>#JVNpKAW_HZdyar9{mGeayqHY8tS1Ls$jOL(R@OWFtEIm%)Fa(WmB!qjD}>l zE&qpr_nrTEkJAAuLl9k%_PBWADD@5j5fU^i1nNBuf>2f&a}{r(m0!w#Te*M|Ab9*re_i`r=opX*r_A$-FzWFTm|D|@b@(o4l7(u z7@sd|j`mX;-GjH)YBn5T*3^YIcc8)jXB*l(ec-c28!lpK|01slvTL*R0!$Msx z7hCI>y2dM3Sl;TL5_dWGx3X2=!*($kBOq6@vL7?L@Q%RW%l`|9ICJe!ME<)w$7Cby z?Tj*FKb~$W*{yP|_e%*F66q-}_OysueteM-vi{bo#d;k{7Ma|^@E0ALGu-foH&vX{ zA*0Mi#)jbO2vvnW*0Z}5XTT;HucM0bz-(1?ew_36A&*YA;&qxbX%;z$0%?dxvfdEE z^9r+ubG+XRb9RV{Mu9F7!*mwRN*q4Nm^@R5mbQGJyy@jmHp$&WK|#DXpJOmax|bVJ-5g98fH` zr|uHQUBuhI5te}5G*E7yuZY3iXF($@64P>=SD+SCbEn>X?Cw+d5vr%!x-^G`k8qt4b-$!QN3x$KkG2P{sBVq%S?&+ec}g7S2Yv&=YA z9~84?W!+nb*&XtpnyoDLB2npaoWjYxq`E3`lmj;uV7~%1d}g;a-R64N2jhGxHwuL4R_yqH9Pzk^zwMvj_<-3_s-=QC2W5pSBD#I-_Kqilw= zAWmtLd>Memwnmr*|H zDF==p0=sUM{PJ<17K(L3E~1lQ3rpx-)YA_Y4dJwVge%HET@jko(>LoFaL`{sXpJPH zRN*hp>^J%cHE_0Jq-HhuI$*%Yr-W)@dPd-vmIp|;X`ku~29I-eyhY=FMQ-00MMgR2 zp62En=9usZHK*s>(uFM-V=b#Luvb-~l?*SkF8elb--JP?YjFf1&`Z^mxHey{PzlN= zzF{L_dCe^hkuw-2-+!h9mPOtEOVL;8hKWCVUKh zHgY=nz}sqxLv5{MICYzDV{>J#8ar2rZz#(q=8ZLMVMmGCb`nP4r|UuGi&RzGFXD{F z46GTQH(7)qW={_U`K1F!=hTrhEu?qPYVzUZ>6nBg++6*B@56S^pSQuPqZTUumUQ~; zgp$-}&DWA@#W8C|mNE`>M;BhAj0~_N^q5j8&xXz;ckH$%QeTC(Tte#?q2(WOj`iPI z=5xwRi1?#5U>D7{M3#ld3Dq}tCm}P9H86sSm`f~k^J^D?@(; z5mX=H&erhZ-tk*TT?i-slQ&hX*L)z;m&2Mf zS`FRPV*3M`agA`e?f+{Qioy5`XQY8I%5+-;+gv^dNANL#Sx?3@jD3vg%i`aq{$zCW z*>8qgUkvM{a;GlDAYd33z4?h*;Y6;4sXVK`{tR@RU5zqA=SH~kxZfc*(+k1e40hx1 za@rufu@^9tH$H?ZksFZi1V*XcMPGr@zAOyehZjh21C=qmDHL^a5Wq)?tkNG8EqlxJ zetFxX0*=$Ig#gQ0a+%9+?`82D!&vsrNW{BuS22v&2W!IwP@ZGC}Om(He$ghrlIA4+dV zDyB=W{E?-M|<|e>{kd+Z5Z060nb$3V%zuVGBJX+!~ zJ5q*2!1Rl6#ADtr5~=BkqIK5|T{Zh8=iTuD7n!u;gnSU9@jV2GtzxnS#B!h~s;zfR z;`Ej|7J>?O>}8dqGt+nrCuz9X&X~TBZ2IB6{!{P0?im0c z4Rw~h)?TgCy~FA=L)eu-Nc%iWw?cK}s7uuU`rb%NxF4WkH`2VKk&%qJIr|oxuwBkR z3*NUkF+&u!rJ4ywcjmD~-A3m3OXYdJM=kYx{DE(0x6yB@Nq5!u031O-kpBwl_Da%r zWW$X+u50zgNLM>Oiiu{vOu$M!%hT8(*vq2t=M}8+F7>0`ouR5zo6awnVUw7;s$Ipy zAI;Zwdb>Y0tKoNGV%Y}k8a4)Mxj+3xSsy=Q@1%tPziP{i>hZF~;yRuI=$NFqCNX|h zB1ZdCbfGuGzIU2q$z?(flysrtvGF%*Q)~`4@imy^R`8s3T+1%DHjM?{Pcz{>MdsFh z>>>E+j41xt?I$MrEWbvIwIU|W8~#^pOhj5I(@;Q0t^k5JTOO~9rvl!DFGDmEHW?fB z(}*(|6Cj|b)H26qYZ9u^PHIf`zLpHyGpFY?U-F{ioWnQB?|wR5m+*v>s$K8@SPKbY z5qs6^VHS{-M!=p$}(?MD6dS=X5ga7ttD7{a+H{ z2?kWGun2Bj{wf)ehPx{7Zz!7`#85`jZpQt&Co^h{~{rs^BQ3B6J#)QTE}!-dA_K?16&H zJCWg+MXHZGE`#ht05nN~f}7V@Dq{{C+$_2885j&nxdM}=@3T42#diyMZYJ~UgdtFS zZh|KYA>GmVCoa0B7#5DkQGyq%X(MAumb0=5^(X3Xx2j&2$e-zE#e&))a;OW8jWuwW zIYr5Map8p_sMhw*y3%A*u^~9z?Z!r|WO>=KZ&)sc$5*xZ)m3AUhy}I%m$m}*#)wm2 ze)kV@;tSHhxTRH7cNI7>TGO4>O({(0vfJTY6B_ z_7-EPzw#M5#{z#!b6h|!SSf@tG}neuR2W>?1dV8MJHQ{=IJ802^F{5|E6*bk&k9R3 zATFSreixKd>TZmBwlh_6u!cI(kRAU3A=T?Q3})t>Ph)dV0p*`)dMN+K^5&a3hfdH? zHzAeMcaUu;8Mm^}{6l{dN=w(wWyH_<%-EuA_(HbjjKv`Qg%lz)>GTlKl{e{W_~F9` z(C-%_*CeA7YM_`PW*+LbsidZ)s06dKXqL3RC=vT1YF(UXiIHzi;Z+H)3^7@yk z$b;63+H;V24wbq~bIbd{W(he_2j=ZmIOr*y{e3{dL=0r~v<2)1o_H3&reFP@Cbr%z zLa`B5VC(D_CpR*`iR4Q)ZF+p1zXPvpai z2+hr*ONLxmO)fB__a_iFM5OO>QG!?)jXXi4Yk)>CLo^Oz9{8euA$*v^)<#G;W#nT_ zg4u5`%=Jp5^7{!0bG)JcFY>0Linuy1aS-B3JIQBaKC>uIXe8nXWeQ2e zyaHWw@I0=H&zWJ?Gg80?su_-^G!huQSWzw=~FW-4d!x3hefz+qolg+2TNzs*rk3y3TxovhNyG4 zNgy38g;gPQ#n-!>$fwa$sPpqM;sDDNHW)mq)%m%gV=yV8!kYBUa2@Gnz_>c-)hQiD z4QEVi4n5I3)K;w|X4S$DEkYE9kKYVNz%CG7F9=qa!n&Hp>5~n>N@uUxnyZIY)lGAC zLA~ik+S$y`OQg3B0^jf@M3&l2zdUzXB|Y^m{+8?L7ZlGtUeK`kKHWKC*PpC#23lm# zD^ud(9-XiF91pLepYF&sB^cn2?wlpE3Ywd;&?birE_hA9gG^XEKwTV1+f`pp4Lc`? z@*O1;N*S$m`n>$G5YmdJyeaU>!h}lr-0yNFAv46g6_hx$$(Bb=PmPH^^Kre+yQjRc zr_i&JQ-Skfv@P2BD;soSnN*YI-$ zeAQwmyYV@zl93U$ghSdQ9446jG^-0C_go3v;Je9SmrKVthw~@n$sy{W zZ?t&s%CDkv9bYagRN+{^#tJyDSIRUZFN5cN7C8FY@Ig+LV)of%zGRJ4Agm=hvleac z4Q0rC?M=vb|1_4e&(NR!&mf&Nd?Wc=tDh$+uq!myA~Yo-tA^sxFG%hsFB?5c#1fL4 z=$eL{gb(X5(XYhxj8U>GaYL*mh$8VZm;1hbmpI-of=K(ZLAiG|_~+GAgVUAs>DISB z?!mc#hM~ik2#7Z+i<+9Y48@s$X`k0j>d7pOO~O=SR`uv%PQi2|s>iuq54h1-iIi$A z*D2ATj<9>~GLK5npLqSqj-KQprD%j(Tv=2xVd+efe#@}W?NXFwt2372YxmE52Vx^t zV*M4euTl1WII*g#>|f$AzRHr}!N6?(q5djAh*NLsm`_(omtVd7y_%f?R@cOY^?);N z<@c~dtdwc&@NbitddW2r%fqeQ%-^18BM%}Uc}G!`_KmKGx!0gf&LgapgFMUgqWbO4 zW`_6scPrVnj{frPQzq=5KNrKv%at;0?kdD52;U=Ov=_;evF9yYlRfxMx3j-@p5Ov6rLx%*&wsshpjc)fZU;xi4tEI zUMAq!t*2|*T^i#Eu@Xc`P0H$35Md+pu0z2YBawsU&tB^dei-e9-UwTIG+szP?f9$m zhulAX?^Bmpob{q_^y^RVAaAGiI+FQoGXKy7kdSB_wTogOP@E9qfH|zQZJVStSt!Zx zvlJ}nx~aVu1C&-%#yVg2SyI4FMaY{Jmu=2tD8tILynuVM;87bi>dU4QSD7kmYN)#- z#g}vQ?h7inhL%yp(HV_5n-?R&9mmL{lu0pFN&K4@%Phe0SBm*<+JCY*4!pvI{)l<< z6~4hF8WU^E2$`9tSrhHgn-~BB?@E`djpAJivkkq6iK--V1BWJg!^GMFgRixdFK695 zM^_1Q+TNIHD(+`ih#@TSy4)92Esa0g0vp)4mK^)O)7*;Pn;*s^sFKF6Z6>qch-ZYI zv?PTvz+2R3RliktMZ;w#ZoyNH3r(;;mdEu9ccj!kxh|T&t`Nwo$--$9W*bR_#5-Y* zIw)oXxqOh-$YwyUol~NHp3I10ULT~v6u@MT5!$!{;h1>wD~~tZ9f2t68-*VvOV<<# zre|KpmMcj(1@79orZnWS*OEsa-2|9%nC0#QJ4rrt|9i@Cb}jX zzQCN^zyX0H;CK)YtUWA89h=-B?gpocWcCpCV>Cq5*<jN}Z$(6au+5pV+d!GfMs7p}rVMu|bp< zWoYiF)(&%^QKq1UWJCnKit}M(#c4pQ193UfaD6U*enGekvTGMiFh&fg>mrAf3-f_V zS>o#WxxekOO=(kap6|xetDF3US1-T7s5SBXhS=^7YPA!#0wF;Fu3{)b0I}jOVGXL{ z+Uq;(Lbk%+tsygypQ~1;p$dzEwsJr23D}QbI#LyH=MbXm0-nWQcQ_2ze8p+HWtYJ| z->4;g0s~alOf1e&W7S}jc)}HAhhFssGnsUVf&vW+WnbxH`_u**k?XsKn%`jm5qK7GP>f!zOW#B98gh=1qHRkVx+m`@UNHrEeuSU0D zDoVasj`>#R>4%S2s99$=JJcP^JK8C`@Ea1!|N1gu;S#4Rm0cV7Y`O#FjYGB}s^_BM zTxJ88@b@fCLecD zNjFyV@PHy*gV|XK<0-@k{h{UCgp8xg#u|c8ljtrlTREC^gIF(yW%B1ovu8m3e55chGU@o#P+DpQ&H%(%>}}D0Q2u@JD8A(8f=E_?8O-;xbNL#VhLkM+SjPtvzPT@1%?kbV^QyhsF=zAk)yyk78{XjI*A#BlVc^0DJIxG8Jt`D}&# zGTzb?*t9nNAX3}%^hV>mJ=5~`CgZyuX!G(?df#1Z_wpiLbw1WJ-tv0iooD~@l6bl; z?eT{ozFvuN{&M6bjG&y9EB(2TD$w2u;T(_N=Pez{wi(FIz)2+`#%)>lBRTc_%D>~U&1)A1R{Wr;U~Zu-HV6tx zI3T8c6bsvXYU8+oZ3`kyXbD*p(YTyZyR#`dP1WJoU=dbj`2K6_stVNPz1Um8;LI&` zl(b^gwnko^RX=g4m0Edjt?en@9o}Ke_(A?S?gu(L!jH{UFB zSWhRn=p}2WN(Yqg-=TNWd47#;rf#c>C!{Y>8`X8wu&4ItrosKC0i1Dtwh}!;_0;3D z(7z7_od8ZU88zCXPOFXb-;WKs!P$BQoQw&FQ6FM9b(<)nqtx>vosh7S&tJf&7B3(+ zxAxekq1sL;mi#}s{hllDuDHpjSgiqK7vPjf*hz_)y@Y#CtOLDRVJ8OQzrFD zCk{M2K&M5E&fod#G8#1mZ2<)tA)+XfGC7ziC2r62Zq3#b#zHeq5{$MyGhB8`9=uLx z=)8ySLg7U8Bj4DSuwt-?EkIR_E}wulpnGydplH?)vg#&cj2@P&#E7W~=UZa$z9?4? z7uqb6qC(u{m*Tf#p*rQBBoE}FL81=0A3WW4gT+zl_bqF~3vU&XUd?axUmatJ9)GAh zLzYK}_?VhQ%*MMM#6HW#wbJ;m16{##Fk3F^%eb*lkY}!hAvC6MxON!Kxidq2 z#3yfqoECqO-lx;-Gi=Fr+qRJ%PSgJW<^0H{j@ue*%f{S1f7(}zw1@FrmmX!`)c*p% zI-0~AJTt)d#fQ?S%Zd-@rI(>{_fxIJe_vE7DP_5&C*pV*$up^d<*htjxVk-Xe)?!3 zpUWUCcKfQ1eVEb3vAr*=$mH~o7#Ej9I7DaP%e)THJkv^C3z7D~W^dGFr|ipEZ=W<%hLnUOP&38Le7B zUvQT6DxX4bkDQGs28LPZTPf=2l<;zD_BX4oI4-?+NHTuXmPz*=S@UweeQ#dztAQnn z_O!uD?2z>lC*3l;6g1zY;6_4~e10n(^ZO2)1Kjv=C@eoN6kVjYI=&1mQ_C(QTp)#k z$%=xEvqSYD&h8Q=m_|+~cO&kna&RMV&V}DK3zYu%+mMFit;N@E*j`a;$UrT{jZ6Qz z;NQV#?T(&or1)+5x8ynO+I)Ta>QZb#Yd0nF%0dBPUN<>80<u=CeSCkd|y^S2^q@RKIuYA!xraDmzF4a-{*lpO$09mIS zNe}^Qf|CYf#uvtC&I=T#tc|02sPx&rcOJ2rg>I|xa7#7`YC%&cqSz)*Pd=qzlJ@wn zb8l;PxLFEcWbL27#IT@Pix5dn38B8kSYrdH=5}6nr98-)hAFBKZj5(4e@OuJOUj{} zY!ya!k1ZxuoGNzxal1v3Y)H%{d^&(7xuc zT(mk5XT~8V>pi44xiqnws|bMNrkv|62A}N7uk8q?M#~=v%$Ze7@W=d_WnS15Yt*y) z8N?HM`Zoc$X(*%acUJ6HT`C=!U)Y1Y7!tcL&pq%e+gj-S^E^_*vW~T5fxpVo)BM$b zRX7bjEE&$ zh*Y`WWPVVCpxnQ8uJ__KohUR5W7Yoo52oaUfklF-OH>%g&d|jint`AH64q>~P%{j_ zt(C3*(t}VBOR~(HCFw%#m#f(QT1R2nV1XWAR-n|YZ@9#G7H(BlfNW6%iflhT`a&8f z?w79>1pFP)tGt>-+ zi_(qIfF&`1uG+YkzXO)0xc3ux%Hl`rR{Qt$0H8|R4UioD?`!-yjn0`%}(xtt@zc4_cK+4 zMuyU{&@L!HC8ehdyxcA|D!p(!5-mN^Y6h^eTpK7F z0dNCw5Nr0>HQ4qsu1W<+w6y^wwkQllyLo{_1Bxm;V7i!*qJPNvKzUAVFS|e~ zFnn8Wxb&MgGHD$1cK}Zp{l2k6YsAQy9X9}A*%3X$9fA)W>OS@A%$eTb%`OQZVbEZ4 z_U4Hhp$qv`ALsM}9GZ=LNf2UrJ&KfgF?E-Pk91~Y0kiYc7)ac0$Gxx=n=mfp@#8#0|HB$~}y)p}er3Dz$`Bq2w zFa-63q$iL`fQbuc00Je=9o2isXz6QxZou$lzSo9>VWn4!Mmo4nc$I{t*`i{1j(Xuc zfWR^fj9yi_UywVzpO{b+JR)M%fKH0YMDoIcDa|0aA z)c~+9jNbuLKa#?wHYg|hwTpVC{qpvl&(jD<;7U~i;Z2|3 z6nGr<;>xr_0~T;DUTGyCGLX<)TYM&&oO&^R8nMEz4Fg;%*E5O1XCm3MMg+*Ic>}VP zlE7`8l0hQ{zF5Fnk;N;oK!&0H0Jn<0dQHmx#AG`(plzm&mqy+m1IbhJlvnky;`oSV zDiYvN4QxP5flCx2r#4`ohVcrIKWb$EEQ*6f|MQKNMWI*qn?4FKRh^E6vQQ2iSej}M zV2EptkbXht25_kB4;4vQ;-hUxpL$U#AAq%T3WKGgKDnU;8#A&miwFqF8{IF+B-?lP zN`eIb@uVl|=3Trh=g8dqRGg{U1G`J&0~e`$z0R{r!$%siG{JnCCc~6fodP5#9KeWW z32eY&kxNu#S-sa_8Qs+}ThNHd&nBZ`16sI6OAvPV_{1)dne;X^b%KItq&E5I%J)Hg;n!k70n%`am?h= z7ae69-O}};tbK6WiS=mV*J%o8kz&e`aotkZVQ1aa?u|6--;DEV*4?VFT$xL(cp3|CiCTE;Fr>EX5#%@zMX)u$BG3g=WP#CubGvFaNdcfUbD#iOSg8jf_3~sU##l*wdkvJ zk#P6}=~aGO8{zzb)L2|c0L5~7l}nt!Nzz7n;~;`;(|G0nn(E+7qcQ9MN7Gk_MbZ6l z%d_I5#L^85(%m7kfOI3EfOH8+hf+%{-5?!Of`HP}-3`(q-QBrM{^t8$*Za@eYiDL> zXXgIgaVE~>4)|M(=Z9dC@cxy4E@YriG&G2}c#PfklLGvcf+s@8>FyU!%#|;Y*zHjc2?aJhGMEuS==Ro!;Nc-wFC1u-iT2{1NnHfL;jtrHlX*LgO29 zfVD8gU8~nTJ8FcF71y_V)bPnVC9_^(avI3XUh{E-{BOsuC3?q$5cuFi&#N!yBYAD}-LA z-B1x9B=&6nRjp}clU$1z>p8(cwOS{R8yTD9C;haAzFY8eaWRaKcKX=@6g$rs%@^`B z@kFj@5cO~3*NGAJ*MqyH(4$)zPj}Dl<5_OzEe`pQl9Oml%A%7Bsr*(t@zmREDE;hB zM&Rac45`w=-B7Hx57%v2x#Q0#-WF#`PK}=yEo1zziiQj^g+3h*Bk-aj_^K}ScnKWK zu6XHjdn8>34f}4r}$ge8hXLKvz!WxbI+*A`?_=0ot>+8pYdWD7K{&(QnqDa)TBnG zY&ra=62@UBPlg*TW5t|Ov}5p{p$7ftQ;8$rAGJ8%X90K<7f#Y^^VY3`)4-n&-!Yi6 zi`^Q1_m^W$*ppV&`G}&W%Kyl{cdPPoD4&FHF9$+&SwGcR2WAMqzFkleT6ieEqtWTB zgNr`cuSi<@*}l`XZfm1Dk-uGd6Q_FYxNtY{aCGPE;wLH^c3oFSTixbEeJ(+6klYxf$T$glf%pCl}O{NC+K`edvt`aBu-Pu=u*G_R}WvVfSWfbg!Z36qO@Haf$wFozElRI+Dh5#E?fZ zh4wgI>(NNpA&~3&(}Cq@i0NH}N`@Khx5Q6JXv@#N<5AWeGR5Hz5sU(<<5)Xmi^Oo> zu#9D08&?a(0riTa5+ScoIo0(yFZhGSW#R%S4zGPEIeT!Lv4rLgUwxQ6_6cK&G{k>R zRQaayzw!$*Xfv0pbF|Zcs-Lg^S0@;F5(h8I28`19$>=}y_V3!YID=Yk7WrObUTAC( ztVL#ZZ+?bXqj|{5Oqwv`Cwqup5QAauMudtVwm)Tru#sE6E$mV^{Umb!s)h}tK4ngk zZF7}X=DUGQY*?U+C;iv8!Yn#)%&lu#&6Qy^wtlK3*3D5*`h9HUa=UN z9kfo&q&$ijwGkSZ<_Z`Y3HwJj=zpJ~x|$d3@nTPd)N{XD&4@2K)R`rZX!p!p%U0-; ztu$A6X@yM1q9Zvz36M8G*%nd<3=8+MI3UHIBKZ%c{*8O)G?D z1NT@F&rWj&R4H^z1jNRV{oa3|`1*lwL&CDZT>#w5zfU`_k=W+$b_&v!jhwo78qB#V z4`b!BNmM+M@_2)!9qsjqG6)f0lK%|ffBT>_`D(h4t{~%@C!J|I@Mkv$mwRSqsnBO) zBfC>ob)Lt!?_N?+T=8_bDoPxdK`6FQqAyvR`R5j)ET2dHf(!mlN~k%KZbKn=hA%yw zvK}zmqhTVkI+lL3NcMJFyr$ksjq#d{xStgmy3vzn>bTpdXmMlfQ8R~M`&d?s8;0-@}k!~Ghlf-r`@m5n0>(I12&>;OGaL(EzoCRLGwO677 zJTl(4@E=PmNf9FluDcoRVR&aBW&`l&D9;3_I2aEG+-14kMmOKFmy-Ml*zh8}y<i_~4f`O~$$RUnKYL=m1v)T_aIqzr9hY8~l=& z|DnQ|pUfk3dJbncQt;GxN`CO|^LN@+k6Sq+8r~t#r~1Nq7%L?PS$bPAeiTthJNu#e zMP7zzJ07;+TP&&0h$=e5^9K%t%2Wd}N6vV{4QM9)%|5jx9vbErkfpz$F)pbhsErxC zRsN=P=EZizL$sFEYZa-qN`z!Yb6C9l22XgRyBykG#h=srRErkPhIE2rp@Pliw&WL|Hnwn9yosxb(2UpYj1uJXD5iwy zGhs?e(5-7p2ETetgNY9XWoNJaYKxOoM}S=2JG1Ev*||M9EXr~Yff)t*=){Ddi|L)3 zKalO;CmzJrs6v@f^1`EI^-6|-HE^MMhlY)32UuwRJF_bJSrbXNl43-Cfm(`m+7isBZRFG zkJqNp{#m|u_|$Lqn80aKH$@Vq_c{QYLmPz~y1Tk#K_L+l)n_zc5vW+BY8>F@-y0}t ze=FrUMKo@Y;gD5{pE4nBRSzXVYI8le?ov@~O9#KlPTzM<{79-!piorR-$Dvgy;RP; zdpASSLtED+ zUw}M$eDyt=e2O>ihkal~=&r6NBkOpyd=hX5W<30eo(Ruav<@bnXAvMqy-&;fz;T#p zd;+E4zM>J$l2UjcFAz1B%TrN_Gdk|5++E z?P6Y_BN!Swv{Ud=a_W;?M~k6wpqKX@PSNu9U}cu7euVO{D2qGA1nfytDdW0J7$m3pTwcjo z{CA&hn0$ZH13W@2m-cXRC6lRkjbVi}yw+p1K?&R8;zTymK)u?z!wOL};{;%4bOi#F z1y<_XX~k2@Ghc9c2#yT~8`|p2xR*ZB121%&pG!7x+re+pO7L?IP0mjPbN)o0**AJr ze0*gQFm-l>EyV@)Mufx%wBj?xF+LF~9bN^!zAs{;(o){7pmgEDR+p)4#l4Yd!+u$$ zs?^v(xPq~_vuz?8^1)ZrPR8-<^vfUV!k-Zy_~F0Jk%m^xOd*n2`4D$k_3u%(G88d# z6lV7Puk6LMUTcTbxgl$uWqh)++T@t)ONyBCq#VH1n?^2y->Z#U4zy%8_;jD_ zE0=K}Y!X>;1xD=bMk9@Q7?Kq3X1Tgc)Yt09%Na+4Vg%8Q0}3RtPuvF_BmYsQ5l~|1 zUoUoNJ5U|V3#mR1N7p?5KGQ9dkS?*=#ger+Qj8oS+-IgFr%s?E-!75G$ITA;I8c~N z3zp~LzE+Y}JY-#p3$($j-nUaZxLEyo?w;C<6z#@YwzLZ6dCPs~YhCFZyz3{#)s>|s z?E9j`{hK`m|GwS7hDNN03?^U_9W_XRnnvqC`^Qm+4;!ei@zkW;z5X1&ny7rXZZlG^Z^ zw-x-@roL{;RrE%NYNL^^$EK>DBVF&h5mCy0ElE2b^zf^#x}WHX>r46eDz_Kt4bxwv=nHKrB@<4J5@TzHgYu?P14XH$M+Y<(Sg^>%TQ?DuNUFJqn>d0dLR zP~n8_E1of-b&b%&q>xg4C%8J7$Ch~KQ;i%xSBFC9hyLOc`FGQ03_Lu;c)>C2H5&JH ze&U^_FI&3@2@?yZd7K8m(3=};+G_`%R@S@cHDwU$$S#IV%zdG{#ik^b?WZ$j*E?Z5%7Zm3#rd%Y-xK-{gaw+c8WU0&1eEy7w=wsA%kG#l_EJ!mfOg!&`4VIPN~*GC zSOgt3ar(aDlXQ@VIIA0ANc_i0QBSppWw`iSDyd)uq0x$ec1Yzp?OO4HJDz%*Q-{*% z%1zyl<)+4ovJZ4OrqV@s%D?Ne7{ALz|17@0B;s}R-!NwpCmxOsU!AG_e2%ws5?Jyi z1$Z=O*`)iOC|^S*^h@91?YqgeOTRJE#pvi^`RN12jU)2QfM1s4x0{%ZOYeMbSzTSz zp{gfxH>!QB*)NBethbNGhhvwEtYjs=CBq%d2jAC$}Wu$7^`(IOcIQ}=ukZ}22jOZp%F6q2*vFJzt z@S@BbZoMe!bN%j@g3-U2jtC_1LGwSZoxj(T7}qZ=^XN`WhlC$Jh4Pw*=%Y`ia zy1mUD&gaXowqfB9hi(>&`zqa0{adjI&gdUa-SzcbU@C)3x)#hOdCcMF?eDikzggD` zY<5qud>9H6()8#UQSKEAoFhYGihz_kohOGmoi9$^I7ZsU%J~n;LahFhZ0`|=a^Yn> zq8(XwF^i23HfyBI{`AHEYS2~HGi+@`%&;~;Ftxvg=17A5h_0UARlz*oAd;UxRaD07 zsOA&;H^Xol?a>uuG(*?JrpdZAbs~UNI&M*r?vtdM1XbuSY6|JY_R<>{@l?@qe%Hn< z{tW8hOajY>!p;T>l{Hyn86O;jttK%%>BZj9G`q`m8h-QOGBLMq$I~xypzTrOvkAoX zV0{0)f8KwMF2nAvLg5cOndBVs6f zco0R;jPvqOkne+TrI=sH`Py%d^N2*kHoUWZ#u3?<+lr-s8{*?jhGQd4Up`aMiwjM? z=g8_?wRI*{%7C?pS#UB+z!UB(tRL5&op*7IHYGrlyQITaPaz3hXgk);+f7W=CzESo zpU%WgR_e{$`?aI<_;Rz)d0n4NoNIBLC(}wWfa~!gi$4>@%(Qvm&dUb$3q(`@=J1S34K!aL>E$1 zFqH|VUc4Zf=lO3=3T3;=Ly)j)9^Cbbl7t~qGY&&Ws`|b)y}lH^TXukWwqvDbD$AZF z@j2Vwg}?1^K%btO5nI(~^u)ahS9`&E2D6l3io9XLQtn4i$ z5tks5^}9ljnRt-9{3>zQa5$Tb^P2lHeax|=hgdvaUVna zn{U&sfA)vv^9d<&;U`QwV&3q9~$XB~`a=tQrxGJ!PJ%O};N3o7dmipp_St zz-Vm=G$8^)+~mAIhbCVPuh8&{PKi@6yR5g=`}4Dbo2|x;fRYwX<{qXM=B0*vOjN)A zZCH2^r$TvD66~CqX|wYXu0t997%#0yWaCY!LD;vRo~ z4Q4(q>vm?q+Q}(s^ob7nXWL_8XuyW09`bWwn3`d;$WU19=vAPTg>vKf+FZsX6AA7; z=ihBqaQg+DA8j$*OZ{_V!n{V2VZA9bWWiFH*MS%!KW{#k}DafyyWtps$g z`_5A+4`bOlS?_G8zTGeTYb zScIw$Dfr!+|9ko&frDKGtTOxCo%ynP-`&P)GR|cmdcDLWe;wiN#@Wx>+l|JbE`E#n zYIhrp7~Uc>@hDD=AYvM)a==0^U;Jm#iva;M~`Z+wSp zH#RtT@KN~ed*H+PPP42ZR{Mjqe2eB`EXu0%x^J;uVRlf2>89`BL?-m_$D>&3pE@F$ z33ujo<-o=r7C*`QHO~xt`H8t|9>)S_eNSTBLKaS9(}>Vsj+${rb^FaiyNDO|V{3?q z9x9EPk75B+H%Hs6UNpv-{Bh=!P(FQ|Qn{@>)k%!(Yf!#R zBjw^J#GnlhH8mES#{3&p7?!fq6;~6X(pGxeYn5C82zrz!_`qF=X+R9b!VQkFq}#uj&qT$qpMsWAmOe>bqiBv-w@L&Fyy9i0%AIELv=IIcVk;raFRG@}`VVs#rk1n%mYaM&8xgZ?b({U@mhvOGy1_b=G5e#>f|Z4s@H?ZI5IXYcq-;W@wgcx^eBW+ZkeZ!mAXz?R1hiuE~6D%|dwB$$!p-xiVrQ zUAckx{E3|}0C!*{_C`<4cNo?>ARdEvxvhH`(&UwAm9Zw9W9?3=tNr@xQ)(*R`;Mm7 z{K_%XL*Y99o>(bR(i>T^yBvX+Wt`|_U88hJNZ>H!o|N}Bzrl4vD1~!pyz^o-PFF#A zdA=te_L}br-U^ZGn79_xeY>i`O>T9SGv*SmRt7QdMe?Weu%REOsu<>%DP<`F$+cbG zF{+sP=q&WKmv^fdW?RDi_0%B~s0g;IrKQ)rV=B$KzQ@;XE^o%hOnth;UeENC+D@YW z;|&gNjbv7$(~W^cG|O{xG6Q!ljA~wIX8W*C1UI&-G&(S(Q;+5|@!Ehc9vQbE+lC)9 zljf#b*B$$n#P&DzD<0a0&PuTLsqU%8`#+Tr{GC)TwTn-*-?6{ zcl~O=oCIThilUKn64Q|eXN`&@2QODgGK$bZWLlkzf9z#VP;z75m<*oZuU`U}A4>$T z=Z6nZ_pn?3TnPhr<8U74Gj6>6Tdh-O6c%WeF>cHl+d#2*>5c?@ulB`=8BalVDWicw{VzgCXQjIsm zI3jWMS;nm5zlZx#_84K>EZXPiXL(wyUSQWplCnvTbyEky_dzeR=4F;QB+SSSr6S}} z1W{K7H}Cdd@XlTl&Aum`wIrNXC%>4jBb?nKngtWi>J!bf5zSr@&Q21}niJ0Y?A1UG z+_eqre$1lj`u2SB>ACy=KRTPO8=kdmma)V*e?}g#NXd?e)1IIaF8o-Z@-e?#TEdKA zyPL%zzgwZF(_36Nzk5w(SM66zP#eg(nJOZsK0Z99zW=q+4;J*Fe;Pc((|f1L)(kjYv16kFK$|+Dm@m;2$63yf%>kY5( zvx?x%YqAT>RutItPOtnP8&sv|J%67)vNg15S`4@Ud45&HoDHqjkqLS=7`{MDc%9<^ zU)2hM=+@FuqI<@l<~JV(Z7)^kgZf+&@UDC@{fu@!NiLf)VUOBsl$01+hk4olhgZD8 zU_xD_d{;7Wk!8$ftsB}kjkD&*-y|}Kt)*O*5UtPKcHhS8mRZ^#&HD%HT-Onjs5td? zS|`2W%u%wz+eieY-)r#kMbIu>Xmn|0F{TawgoBH9AiK~P#yNoLOEh0q|7z%)ecOFH zR?Z6=X*1CVw{K>89~(az>;>KByJ6=~yf;z4X)Kce*>ynO`Aou^I#&d3aYG~X_p?A8 zWgg}OrBp4`z>_8hE5BDRbe>N;_T1FutC(c`lLKkWWClHCPHUZ}#U!(QaItUjLu_bW zaBN_|O=;ifFcZrh@Gb2?^mfMH8T%WH8Aldk!@zi|w_%z@ zV^=u|PMLPKp}6iHSNfM;R0gxb4f~Y0cz>ap?+seS#VyRpiA9A~D zC#qXhl&@5GYhwx*MEI6w&fFHr-$pV7{kiAB{Tw_H%D>tE%yEAwb_0)f**n4cW1?KI zpEh=Zu-((F#&5@BEAz<$WFtc_jZx^0S6=s<=uN}*?$~;f$3+?9FVr&(U*MjCT75)4 zLdnGA7wpmx+#$uMvRnn}bXKgt-!vh#M=-WLg2frq)14Ud$oP}AY<>i8F8D6m7X+M% z-G%K+s2J9y=s%bl6f#0DFaAZiFVh-Pu(T0f9GF!N>%!@s=qYytdO4G?xN;t|l`P?z zAF`pwbu+cRt<-JG>8jZx&#^MUQ5akKZjB-58xesU#d%v28tU&Jdp`_5HEKT+)hME{ ztLo3Qbq_UAlLr|?E9go%45hK)=s_1Uu^tNSxnJXuBwVRt7wY!;o1umi$J<|i5LU>s ziP4;A0Z;y7`7!XR?sj@qog9tN7Cw4^7GR^-Qkh*xsMHFEL8EsrPUDO+5ZSxYfz1RI zO4Pzu|Dj8!X@^q6d+de5&6@eMGAlA*Ax0ptOVU=cEvNC9q;nskNrP7M?Q?;Y!JW|*$M*lcuXGfK+%-B1 zpR${%txX*{IC^)ZyQ&7p8F>q5rZG*uy6A$N1aqU*a;(mY@ol%sKWSA3n+#j`Q@)rq zN;pF1<1L!1lLy1k#Vc)kGps=_J=WtR6FHvRrj`G4-n{zR(_%~wnADHS}Sv?+>I3uny3QNk@hGW^ZwyX@;u*E`hkJzL1E zyg7~XS21gWU*6?Ka{rxYWJt>(c)`N<{%dy!MJ-(tW%HNy6mM9K?lBsy`7-)H_;72LfY3a73Pf6v$PsJQD$hO~{sJ8je+qT~h*M9Gp4xfHA zmad4D?`kR@>7YNc`3bzq04}5V>D$e^dm57*EIinXUWxBxTlz>WZc6&77s>5=?|&+2 zz0*lJXz~_lXq|-H_!#>t9^6d~J?Z%f{j;cRRk!-ha(qxc`H0V~OSsD1>Mgyl|=4QV3CBSDe873fWBHe8J@j zm16Evi}YSs%BO3<^f`O}67&4s3;0Lw!8kqa&m|t>Fz!wLPAKWCU(K)7Q;3{#C`<&2 ziXU@%&1*g6tcF>cImoLpmA!0(n}y+17^1(swqm0a#9G3M;Q4wySos8RKbCh(Y^fGZ z@jEZ6CapBw2R;71VdfB&X`93rh~Uzw-F&X>mQ0Fipxs`3zbp90pE8ppq~#^%2iyLC z|G}Hl)kK{yO(Cdt((vn{-T!IJLcO)_d)+OkxP3Fv!+WFobL_uH^KVtU!#C2S za+xdZ#44iXiOyJlX`t8@p)Sc11WnU?+rjaHno_!-`8E_1)DG``K)^^hOr8`8%Y98|ml1d;xW`-(C3` zynAvgjkF2Z<`dHz^V)$8cs==|JbOCo)*pu!9UrxMsCgd~@X$w+gcJ!b-)gn21qWYj z@(#bi|C&Xa86wW)Xr#~Vtpz-xXig|0C)!FVWc0pjgKC z;;`KwPS99I;`gpeEd6_RIv%*AV?uWKN~CKXk!}ihy1VXuVTIQ!A{I%BCqSeaMdi>OK|4{^1 zKI@%^YBo~ueyO7OlzL;8^YA>!Y`Z*hv%|t%-ARORXx0RuH!4bpuZbhlza`#SF)`&8VijO~N<^?4rHrJ2;3KmqI3xsV-zB zxugFCB|3*mVJq}7k%?@Vwv2ivWbv_d?S(_+o)G5R>KDn|N?twOZ?s)nuY7_jW_PmU zRmmycUrl3nRW7AL(duR3J1-Jx$0Z1b%&Z$j5nLnkL>hDvl#CJn=CWZ5)h{RR#|joC zyEe0m81h&6KVyHHhB#deVCSc^!H$Kn5CmTii!?81_e_mq|g5#?SjVo&?7sqZ*Cx#`rhpC;!>u zWKXlggCI(|dwOqX%(p~y#kA=M4B3`R0ox$jplD0;EBAM(iS;HQ@iFe-OxL|6B=p&oDaM%Wl|x(0^b2B4|E51mhRLi=B&PfCzn*Zzo!cEH%EQuZ1O(r%5o^#jLM0BJRasNQ+goq(O&cAS(n3J$v9C9awG`4 zTac8_$Qt9j3aN80zkiMPMR_-V_D}vsSiW&*xU2ckHpY%3&6Z74eCHjtGOduOeK`|J zRZ$G=r1cE@0oU3Fm^&S#Bljlnb1=oVn)vg;stLagBold@%9c{m(jD1ZN#SrFZ@70> z(>k?>E!TmcCDmesX18x$s&9&xOzTik-JgzqF?3D+S9>UudclbUYA2sY{}FZNiwvf; z_TYB?wB5IMSih*thgW6;Y=u6x)lvelNO^{HKDn|oRJ}{IWv~2iCwJ46ZE&`mFlk3X zDo)=>TN&6J-gol8MW_XXxPL)MjR`JwwCekk&E-f%zm9yStw}eg z?MDj!Xe&Ie=NA>&3CE)rxIK%G-q7pcyS_bsUEoeW9%%nuG^+h7-wYi(R~vVV@RcWo z=R8)M?I7ECb%KBPgMa%5p!}cy8wUSn?||nVQ_7fI?F>=%;ysL;=!}5JTvUf7i{$$@NPTc& z8`R{h9oDin4Qzwt4rU_cu;^DW*unBmADHpA?`!fpXEPnx2|twCV~G`7U(qE6<9-y| z0&|O07iIqo5^uv^e78b5|#Y36U>=|rq}_#&eeYft3X>` zN49I$cU>UZZ!?F%@v9ue;E=ZwYasE}^fl1YTKWzsmtP_c!XNa%HKVhHK(zgBLm(w= zo|Xv^j6+u<7`OJ;n{Cj}(r(b)7LIDHJZ(IOVo1oM+UXn%k&@TiS668KJ`v#u4UHaOJ8fW2#ZmUK3WvzIheYb% zS3&QXSfHm@?>0beliv-XlF*1>2f~$+v7#g3^QF>rL|_KhCMZx%16ILCaDf=y!#PJ7 z4zR*aZn^)!PNDSc@h1&xdI$9ZkmlP8ZDf0g6*1i8f)5iGccC%>4g$W>pwiHxq^By| zAn~j-c+LY~79>cbY88Yw%8r!0vf_mnoks0|(A7O*6_>&Gu(0Mh3`wHFC2TAbcU>BDp4rJ^BX7_M26;d>(lf!8f1m>Hi{ECzP4?VhRqiai20-{l6;P4}Zvb)%{Tl!l z)sO7baW<1!1gKMgUH)?pAJBstDcIQ6vn^0GlPOF?Y+MR}Ht`h{*({+4oePrr-VlF~ zNOaAHY;UN` zJ`Dcst^-utuOJA#;#~)L8`~`&BD2g2fKw;|Xo?m$6zN624T|l3GYI~1EZhM;J_c&` zxGQrTbb@V;Y;RaTynU9wDKp~Yusg7P#1B>Q*=L3pH3Ia#Waa#H2=3)R0IlesBaoC1 zyh!sDfHsK*z2Gvi=qQ-6Oj{*tz)z$D{Q0N=o|9iR$8v`l-#-9Z{~#9S+Sp&8Py-Yc z!BmU#uXsy)J*BU9iuPof2w`V5V><%Ny=j1*a^N@3YN{PlNQjJp)gJAWASHue;vm&a zfSgY7q(a0fH)x=`={cR?d}x*F-v6oM#u1N`zvJ_Zc|(a)?fRo9xye1@fl(g@%I8jt4v5>-YEcRWbjcU zq#C|Y2o=uAX@&V0aPWLa(;*411CDm>a@7b}vsE)p{Ps5iJZD~T5L|Yj2e=Kki;K^y zE&bPFxY}WMo$v_wYg)-U0u>498PxtKJg3uS80>)M90$QTV}-AJGW~&7P=*UzP|vbl z2myQv7Xi#>+qiBOWHZZwY(G6vf{kr7qC>}wfJwBomas9T2rzKJ;SNeN_<$I`y5$Ho ze;s)NW>Sm-HPCy3$ZZ3h@W`)j1WaBm)(1CPRviQ%Qx+!V^}S60YBjNw>K+lJ1=INE zS(u#UHbU?jzS{46wdj+|Rm{Cdvz_SzylS+M2)@d4vChVLG?Z`5yBDzWvk!dkZ3Y{2 zMXQ8b7jJ^lyIulSZqNZFrrN3nBnBfNpJ%H2W>tCyhMPWsMCp1C z0fHf-L;E%RLhTgsNBeM}OBPtcpSpwtP-NU(Hpcfk37;Kc!JE+cu(+%v8mPyP_aIo) z%N%yNmOm+5u%8PrUrp&gdj`RBr;SHX0g_yS9$0@phTDBQp z^mCK!0K?NlzofdkG6vSA-hF{2Q$y=#$Sx4wdfmP`!SN7-U3H{?-)nRz^W?H3Y}NG# zfJ|*fDui-B6xH5O3*>-@&UY}4qiO>9s)X$nQnE!Cz`&U)1R_&wISU@M=6cK;z!pC% z1|*KB4UoM@gVJ8G;C;s_;@wgzFpLgBn#2Ls7y2Lmt%C{)0KpAO2T1dkkBA`eI-dYe z<43dwQgsyR0h?`1&4RaScJKM>8+EHD01lq~wug;%<>1~`5dg>(vavwJnt+6M`}K>9 zkBK0DlcEUFsUcFkK>Td`xCJt=qTB>ImgK`}+Az045?|coAXw4w*FcdU6Ryzc@GF=2 z%e1nF-=88**iv3WNxkO-Al=!ut011jBsk4|H98bS*uDe&I~&lf;7KtkvvAJ^;$91& z%ODFT%n6w|4E~zAj%shz5(9wj1^fjLG(|c8GZ`K$7O3@~47He}wCWheS&NDBCKl-K z@ZyOT!;)1UIkd=*;l@FwQ8YnqSr&-gckkWVkdm(3rfiHio3GGGe`*)0`-M`b7=WK5w+fWw;t4(~k& zI6O4~N(U|A@K6AO8fgHOIa(mSxhrgg4sgzZECS36g4O^`p#qv$F}iCYpV)V>Q$6ZT zN_pPBHI-WeMseZTbA-kABNp4S0;WRt^ zwh}p(GvETqB2i{cC6=LWdZecGtsvPira#g_h(VBn|%(SBg8#}Ldy$0D&&w?50 zb_W?%% z04@eJG*3)y@SF|UAUIxdV!#=ekTsNl$8EDAHwaGVxtY~d^s9RZ^kN(sZirO87!sE2 z5Cv@($2W6Dt01!Bt~3=djsz)USK~QdFSv;z5FkD=;NKx`iLXINFQN5uzgI!#=^_K* z$2rY(IL!mgHVE?+KxHTfuzs5jW@vrO(n}b3S5YtcwS-eT0nX&Iu|eE^hB>G#NX+vtJYc3t}Q z4C#8c27cuZFe8kM2sS1%8UT5|rFV*$_f&#u&;s9@7af4SX_o@Par8epGIu~twfd)s zngxJQokzS-+}aB`jU`Y~pC9@Jq0Gn*L1=x!|K#g~C^HnfuP^fEF;sDfXHSK(WlMh$ ze9dz+s;ij&_#qmiQXPM1%IFiMxbhfM;LdYX2nWKLZCWTr+9eBP;MMSh>I&d1y7O2U zh_Ws~z*HHaq!fUFbem;H!)7Kp5*??t{~SPA)wK?E!O1BnEsu@*r~+w7~~( zB|;LADs?C~++=^@@U~rn{}L|lPvLdX3v||#z5<<{50*t|hSe^mU>K+%srRFby3uopzqD4Z)O5*g}?d`LDk@X{A14FD-AXx0y zPXfv61pP5Qe4%ZJ)ktb%+gfNtYYYyg>xZ@>ja<0~w{D(s;*Z!e#=UcdGtj!sX>-i-QWJ8GZf@6ZDrU70x%NaR?1*x) zk0v}jI5%iL`6DLX)^*N4C%LtEUnB0x8M+W}WzWl{{^5;Umcv$s*p+?%Mf$-xyJ6(k zo9e8=Dw76r&;7Z?ok0(ytj$?rPq%VYPmxx0!z{>!Ve-yqoo-g)L6I1V>)}Avy}ACm z2g|17>E`Cki|%t3?}bJ&&`ATkdWVoMwVzzp=dW9D@INVMjX6g=Zgkv0A?%|wAv0Tg z7A|^Oy)s#USNI;R(!@RG;1JujyMqqnpt$fj=HWf&=y8Br0#9qlXYL|heq!=0^5D84! zxgyie`hZcRkDqebkuJCzH`eJPE$ysK0JNi0U zOqkQ4|FQMh;g5C2VS$jC{Lh*FV8+;hxLfAx&F7fd%b|ISo#g4+bCv^aEFWqa>Bcgf z@p6b*cZ2>s<{aF1I_?D%a+Ovx6;ZSi$gXIN8#fDuSf5K-{8K5~Ef)w0VrS#*xNRn3 z@6L=_L>nzSFT)Q0`9^oe8^mS0i#>o>F@=sv_OYvohLZT=#rVMj_KT4>7;Zk5E=;|| zf3>~`isE#$MbVjJZd`@vDA;ymY+$HwK zX(4(=%y051?6td5@XYBne@&?4gjvcNhs4L;+069?%iRBrfz0=zF4D12ig4uywsE?| zKOxe((E0^?g&Md+XPR^46kES{-P^u_we=nF;7bLHuruO<2$^g}J(KN-xuVoky0%aq zE)FN*bRk=jqqBq$X~6S#SWY?2(N}(dJv)N=uix2#aaQ=cN;kYsqivA|?3L|8rY}Ps z>loNu(TB9pj@rJjXYi%G75-fI-wNJv*Pg5W+TA-XTjW=3JIG2iI_JiMLw%H^UW8+O zPJ3Mm5!~l+V`+ zSRQfW>DA@t7{dYI(eH7$$PtHgjkuDn&`Q7W-9b}=mqw40Hz9HMq|Y*!vbuEmxokU(l*tk!%M82M?xtT8ubh@^pBc6Ut&5rn26MaB{_$Is}%Dt{m! z7n8f{x+D*fUq2p`x1wLLpZ*_BUme%v_l6B3j2?nY36n3abazV_l$4SJlF|)>fkTi6 zK`H45N$D8fDIp=S(G8QJe=!3@o38pgO!Tw75|v?4ADTV z15GD>s3fi!;&S}@x+=_q4EfLW*GfQPh%8h&nr|(@GI;1VK%%SqszKN3=F|lodDLx3 ziA=aFUz7Gub;SHla}24RHK9bpe!qhz1Lu1;>?=hcnGk%bY6nD`Z?34od7*bw#xt^U;X;jZC})t!&Bfeg6Y9KE8HorMUB{ zj=HY=ql*(gxvoqABo*(qdvGI`3f~eVGb%b79-LOqcp77xje8sHnOd%o$N@>dkDy1P z5@APP;2VthhchIx8Sbtt4BImB;?A&Y8b)Y_dn9o;TU>z~%fTEDW>eYNa-lOuWo|jT zA0u3i#v-vdKTPk$r@Zbi$waY>i-OqOjwLxn2Hjl)VI}UOHV4P&s)G|X0@OCciMn1N z@p#?MsvnmpYqCJ4-h>>{_;>5cEj>oS%U7jO-n3l1H1A!zXgz|qaNSfsf=0a@n(Mt3 zR>e7u?X}B(+HOBJorc*0!i0VAe29v0V|ueN!bq1*J<<=0dW489VX)fkx-Um--UjF9 z=D+~;fB_ypyS>t^g5NgH@gX2S7H7yfzq^KX0IQ0qYnQ?TZ2I6{d+JYYdO-z_feTT{ z{jh84?L*9=HFIW`B^gpxs_fc@0?;cB(Cg>WZTh=zMeHZoU8xRv9xm+U+QpFZ`sfMm zInwjhop{D9Y)bOZr@0v`4PRvxPF30#e;4Ayw7C+V=(NI!{9B4O9<9Ji&jVn2Jc801 zcu|?(>>0)^2wxwCw_^YGhG88&b#NIrB{vk2pys=_jolHcLOaOyh9 zW{i<5b=Jp~TN>gr)|;{a-Vq|h4w$;d`?1$E&v^l@(AO@UjMtT@Tx>cZst|Auh9aEi z+gX?}7h>iuSLmt=Ap-V?v2Zn_9^JOEP>S8N0z_>CqBIDR-LgzQmtHLxGpFwDb_n|V z$X4vk93FLjWc}*SN2n5~xp{^Z@w)S2==JHkPnr|-`5w}Hq#Hr=Y31fszW$vdAnJ{` zDGoT11coz6P`*)<)H*mR3($$iGLG!EGXXXfHkpQTzT0h21Z+qRC}sel#)X*ee@iSL zXNYP^@bk59<%NnqFgQoP^gc(rRRDjcca|r8&NLM#CQ}{f@*MDop}lrNAaq^>$VdFR zK2kBmWen`Jm+S(1M!Kv^kbbChBZZa;fEi+xVl*j&0Q#icXWZ%r zqbhLiLSu%T`8NUkpH_*y=SXWa+#6ts4}l?4_xil~?~8z>PRNnO=E}Z`F(%dp06uUD zuwkFqT|*lwBHU3r>}V(a;@GKkyFCVQtPH@h(g3t10JNS|;$qj%kR*UoDv?KfR@|ZO zF*>M#@5=(R^j2iZBf0ug+>hU7xL5$MnG)1Lv{OI2EeI)x7^S}xkDrCr0Rt2P2B-qK zR>CZd25_y`W^DTFV_!-%T@oOb2G4TKt0t#3_|sI^vnQtllBN| zKHuvz7#@w?8{ca$2YiDDh;Oy`yX~xgcRp-IxLCk9`gNi045}hRIjXvP|=qZpbi0RF>!psS|(dT;o4HsF!n;9#Y zhr0%1hztmH3BVdTfV*j%;h2VY+F7U1kqHlxa&Yd@QKIMQz%0Kx=PF*PxN!rFuQvRs z03UE&UOkM5O)&hXUHsq(OeA>i!UQDP9zeTwK)X3WJ0L3HfOaW>c0X4Ge5t;;9zP{S zM5Tl?>ZHNN@2!_U< z_E+bxvw@P;?U;eIIfFaJI*9!YL=P@Sc@3AkPMDTw$(+9Z4-FDstxmVXW;@*JTkL>$ zE0IGMFl{>rYV)>Euc0Jx0gEaCS)f-52JSF1-UUhWktc&`zX<$}%|5w#kAXK&&H`Yr zAZp}`O#}U4+I!b0*z6w{b!gD{7pIALitOj#?-VHxhHv1zf08dj`V+`{`C><)77WvF z(aW@gX`h3Hu(@j`9{E?V{rM6PyMc4@ryvmJ4beD=d@1$pGTQZ|Gzv+w*8>F z$f+wwqS>%@eD5M~7Q?u_^$<{sRu$`kq7=;mdio9wlz14pFdemQoL3z`Du5}w6faP! z$01X|c4l3~x-FN1YF8Z>hrpz{0s_p&;Z(zN%Js3H~9cn>w~dV{G1j-i25PW zP2`yj6tfNiDi{gyELXDso3a0|@%rDiz4(N1jlY>O;|Z=i|1`yuBXM~$<|HI=})+*ltTVj_)^^KIk58Y3@Fv#?O6b; zAJV^pp{{Z0kgGaC-8BKgIFXcqao&%2L$iOAoQ8Z;O96W}9&ka7mi~~)P1IL2K;~dT z5nbTMFbsS@3*hOgmZ;SXP+^(c zKs|U?8Q@0;X91qRjG?#3m;En^Ts$?fmLlm8uuQEyz|-%tY}dDs&AdcQDTtw2TG z{2j{$gb83N>G6}m7;nI|r_NGMa;Wg0<+a|3rBxzS-CieKne3EBpQXG=k~hZfVQz`@eOT?7-$e4f zP-#-mFIBKRg&{qhB`-BKh&t~OIG0nJ@zek<*z&2|k>q6(3r+1s$3-ssq9gpp%0UPx zyp{7p;q1+b-ya%1_r)r1Wu^H%PXHrHXpTERvb`nhYmYjX zmshj}Lpxg+J1}5m@%60WxVWYS?4o0uqKCQa^67=b$B<>_ZJcZ!XqSIxo?!NeiXB)+ zU=zAPzzzHFLgCP-hgqUFZjOKzyJQf6TB&|8M*zRM{~P4bA2r9^=pkzbRw@6m1**6| zW93++Bij<7_6ZD?EGWNF7_95u4>1!Ly6A{@NxcerPH+$+o-1k{KP~-JHvZB|Aix4; zmGF-s%WL|<8iU-u01Hi*8D^vRR0*>2ZM>MjvF z8jyW8{Pd#ZP1fX9$Z&RK)|FOuhWp}-Q~>H)K8rR0wN;IO(Q){{CfVv7K-`L8KP0)2 zXpSH*`uhR?xKX)mmBp{W%qhZR9n2{*KvMO%@?)`7?k&aXCDZe@P~&}^{i9u4)hrW}JUx@xU5E18Ra zgD8dZ;{>sX(Ky;Z7fXAbw4Dn!t<_!u1(f;reT`d)g=EnC{{CKg1IrE=c*QT zy{Nn&GVRpEJXaSrMtUGqy-6>srdcy9Kl|4%_%{0@a~{|fsX86 z#n(Az45qJsf#8cF(Qi;;+=p54CyPx`GvNZkc&1G6H^rmpUGoIp-oP*Ymmp2D=#Z=_ z>a}nC#QPy7hQ$}aEs)lGpueCjhycDPxWR5ASM5{Jg@UId0K)0-W(R)82O+ZB3}p)h zQEDJl`6|AyD|}YHn;n`o2lGI`o{NqbS1bo1N8_$ezkxJuQU@|CVY}$a_HwEP)mM^R zf=w>oZ;GC{r;af<)~vM75g_Bpt3ZEafJTnFqyc0`Pj3O{VFntJ*rbCH zLe{#Aj-Ltd11xq2@B)G%K!1J=(2xCJzyAN4@TUI-l1KMH0$ir44D zk1uU*)!g-A{6G)6MHe~A%6Ki{=RqSBnf}x?!l}o!0w1mg$J0H$E);w=H@-*9!!@4% z@C*9BaXU|-{DUUIA}F9a$RcH#naf1$Amp33RDgw_V(3LjIeffG)^!K4&{t!v{gBEA zssL0P|AGTp;`e`5Aij2y6j}6=p-28^GEC!n&ELMe~1Du>eQ!6)q64 zhP|i{vUr2O!Y5Febyb;qeDDT%j#__c14vG$uqj*aPH;Jz$jz+FgF zw(6;{JKgYkf(+$+xrAe64<@5q4=1tN8gNJ@>gWyjOPU zzPq*8(bf{EcfPn|rDSHR=;}lDN8;NP%5*eoHu90@la_9q2gt@w`tDgZwd2eO2vcrO z_(6MTR5WR|!?`>?n};p8dy_B0H$;)z?TyMxdEkSca-JVP@DDn8&&v8KLeU+-we+QT z(*j=Cq|eyO0zqM)d)be01F39{ae0I6jqRNWvgTBUDeAvXoH*wFIWkbPrz&N(nToAo zwz`}_(dk_c`mF=^$6l-s`?4w;uJ>xCWNMmOR*UDjMvPiOs=MJx^f{+uor#J!LHSy8 zgmO$wa?Z1J$~#pty*}H_&_h3J@xot^7t4BP@%76Z1QISKUf=HOaFE;LKS0L{hmcEv z(1)zSv78x~>vWj4cGBgP(bjOTX1~@g_vIyuq43cu-;uhAOY#tFa-~fTTh5^QxPzW$ zHsJ>=8Sv`z!-t)1UrIi#&14h#j&so%{MqFbOjvsL^eOFW<|l$4=|h3_rZutCWu2!V zbP!Xvf|z?~+ml1Lrw}59T%w0J{nla4OV`R%lK3ahl0o{ocWtpM#q=v-GM73?_H|RS zetu~I^4jw$d(OLD7LDbb-wWd2Uu|BE|95?>qWM6qxU6A@(I-r7Q%>+1!CphSNTCsF z?hj$7-v5H;b3TlX#oRlW zDXVF0j*97p6s4@|K%lBCZitb0@7v!Koju=6oi+IIFA}w@zPi3XwUV7rpP(S8s{@Uo z&M86u70#;n{G5y}2FkPu=uh!NNZnxP6LAiUM3=@gbF;PO7S&Ds_*u9pl1U2XDe!^A z-2qg1Q~vAF%CZlg7(^b#edLBu+Fwr6-=nwvb@aJy=Z$QSP0TyJmOf1^_!eb-Vxwl-|N{M5KPGqEMzM#9_t z81+8Cef*WZ()N3%^329=Z3Qa=d$3k5YKJ6}%zDkRlj5EXq$^q|{>~`1KjV}(wsCo0 z!vsGNqH^^3iF~l5@`Diq5nnBOuHG*`uCt5~=hZxe+&3r76r6VuPag3KjqIf)e%J${cDM6KGRcbneZE1qZi#)3P0RbS?kPB-aY~>Ee*f))=)(FATZz=o$`bx( zmK6xEJ)fV%-A=159}!2Q$@SD)A@biF4N2{M{cRKk-5*oNbh{JlWLd`53~mHNfZe&e zlswaJwA-jrU(rQxy?fd@Wjg1QzYmkZE4%xqK7b8ghRJ4J8sWal9D zrBm<=zf9f9|Ca6DY}}V1Azys7qJ&H4UNZ!zol_Z&e{KkfZcL$T2=Dm}a(lMwT}yr0 z{nafPLWhZ1B|JqAYZ&c5|gGXw+>5%_OsG}r;nlaIyiYBXR zcrISqmzd~m-Bhyml;X46>K%6}$(41QgVs!;{g$Df*681(S@y4SQMoorTbcyRS=&vQ z?8IoHTVZ-e1jia3XFUf!hC%x{!f|lA`% zDkts*hPR3)+f-C9gVWtwUSTLB|HIhOxaqBf9c=`}D%c)OBHoi$1s86!dADa4=Yh4C ztQEx7LOMyApN@K|zdi_8WioiUr(T$nokrY#q$_2Vy~8Pb^<>|LN1x}S@_EVCl)|Jm zS0SZR@G+A}Y|BK|vrYH+vd9;7f${l(|tDfIOn0q7Uf8tAjTpm2sdmi3n zvMqpc3-Wq$A}bRN+*A7Mfb7ZVw4*&P+D>su0u#Q|P796h!~((n@A_!vjC_JN{`q0% zS0OW2S#fQ@WUZq;ET%eN7mFARr<*2=fRl#bwRiDYAG%)rM?_6$2Xf2bq3-3(;n(0d zMJhW83lwB3FjQN*~38)n2jgO5BZ>dcAolwqE^hEdUjpo^J- zv(E~7OdiAfu7bMqYuw(49ef3Rc&@Wb!X5K~4>Tl=o<*7C-De1L)93;|*Xso3u zgg0c@@?RgH;c-|!#XmmZD`pT?e}^jom+T%-OUx&^#C^fAo&{Gg^TDfC(L7;3Mb3&Z zdTpJFux3mzg2RXr@&c!fd9OzU*aL{Kdsgx(Y80){qeEbKp=;r%tps8lekZ~AXBij| zStHo+ggysYW!?xcCxQ%y495`GVnplouXCVu`ou=lqEyBMi`&g4{*3qCowfv1N6yDI zEY$MeB*0lT=s2Sln_apts^ z+)Hih`(d=T&;6@jU5Yo(F$lO2YNoBik8`0{0A;^S-y=6NQ1BKyaSs+Ab{37lVIkZk z{*4dsVkOzCZK;9ukOPZpfNuJ)wSf~@KKLk`tr2xfe+adBD25)^^rmVpVJvq(i&bz4 z_h(*7J{YIY6}? zzkZ6LlOo1fV5lz10cUPD1CSxq@37Oq_VVe6cER@6j~v!2f1bI8Bzp}qe0=Oy2Q@fh=wEsfK3 zsG>Vfc2Dj^i52A!6|_-EW^Hc>%aH$j2Frehy|jE)xT@H)&kJd(I#MT6-0#MO2fOm4 z6AOcXv9nLLZ@+Q)r6g6wj+@xD`fyC+2Y0iyxlIev9n&ABM#jl$-?QmT7aYF2E_@)M zdeFiDZz(p2JXmLEl%&HXb6Vonq#%B!`C~sieH&)azLUd^^$|+WY2xdHUn=|kXx9EV zub)2KC>a$e+9YXv<;(#3vy`9U2acE5Hi>NG=fk<7{^m&@u9=+kyUUTk@q<4c=|hV) zZjZg7S+Nr-0_vt?Ez*A*qFu{rc9Xm90_#}pc~w2iy(&BZvyeWSi*>|&EtiTW5-jeTpQq6Q^VR`NxQ7S$h0w=NI9AMxRp> zHInJpe!EY0S}#Z=@8zROC_m|w*F4&VB~ZU#$mDe^SIZ~@EIfIg(>()Y`z8{QgK)+I5a^-aZ*$%S7Ye22%Qjm;PwVz0HX*bfn4m zHOW}NNt)V-(4`y8O4TC^^#QZ zKQ-oUY23*7@mxGe|DZKF#3hL?f-)bPEV!|UqWQe}pXV-WFV6mDOvGI&^qD@es6qMBczN+o;h=0ps7bzB*zx*@iKF7zvPBOuJGg`SoqG!V~amtMl!;V`YJeNitR

`fm@ZOWF8M6aoAregY1G1CAbzoFnesuO zf-IL*1-@QA&wpF(rmSn{|E-D=B4x^d;{Hl=nH2=;wKkg-Db=)8QdVZ{5+JJ)E9mQ+ z&TQW|zImdpKTQ7Ev=&12k9WPTdVB*EFA>PlJS@Il4YsZObau_yz~>Z!$8GjhO3v3 zzHQTOqrMwV+Q;wZ<=wIVIvbVnXJ(f{C7r|Qw-T*22x-pN1=atxDH?`{=Gd&_juu? ziKue$rhDbQ$aE{zux0oW@U_CuVITUamg0U9?-j;g%W9qGFu-98(5Rgn4}Kn;v*( zf-F~il7y8ya*rzg97TLHFo4`y+|5t4KUSBpvNzcQ?QTA_I)kgp_LYS;Mb18w)mIN{ zei=R+bRNRrY%9@f6U4vrqT}26O$K@V()qm*^7elFWUFIxXZqE{if7IxlR51X>l5Z= zMZb2Fz;~lRDDOm%tyQ};etohv5i>P?{nx@jfE;JiTtB3;(sSxF^F7|jhBb|g?)ba~ zATid2cV`vc#o6C4?OeuWevbOlYY4G1;5qW=NaK?~S6-;g)>Juqc5pooIa_nO!b2&q zvbUvt$s`|ZjXZtW;a5Ys1#V#(TR4Ta*!TU5^Repxr`+s#ta%owfT;v|!1}&F%bzdJ z5(#yYI6WzR-o3JfF%@X_>$t&gDg8X+QDou=`~r0~W<#{=dWqaGFg5UypGSJv9R$_H z;ih-5#_9rQVsEdDMi%~FtWqF{(VB(mG3}K%O{Gp-(vGN6qCf}oR$L#~+6~X@4MN8Q zH_HAVIy!kL4(yn-?J1=~P?dFx|Gj*2dIFzw#UL7XIP;2rUaJusq-)slOH>{X!*EAN<3T;eek=Dq_;3AM~TexOQ1Cpx;TDd^3puZnn> z<)kF@??_oTri$;0ID42++_zefQ9k-*?uW~kHBiNSn5H*DvK_O#c$w6A0-Rx)YWXA0hpf?3`n>+>@|Xry!CZ82)|?uOtK)r$ z2+wq&Kt?c)ks#sH1ZdMLR%&PfuK_yH!}ck{ke?m1DbMr}`9ivDr3Kn8xgwqKfrmKG z5XG4NN$tJt=2JY5X))?e-mByK}YYf(F1CNzK<1%UAXyC2j8({lb4` z&EF6PO5HtK4f+ytUXFyFy6DS;uZ{Aua80tcK|y44#-? z$K&CNwM*;pjvFm0J9OC|@c8`1Aeq|mMcCxK-f>%9mg|9hMsQ$9>!Jl|xcS(E_TBI6 z>c7`p;){}(`Fj0>EZixg5XR6Dc9@l%HdYP`qNDM}X+#qt z`>jMSvpmTcbKU8>o-<>Z?C}VWL#|8qpO< zrWw2{FEPu1(Xn;@FGu7V%Osx)(j0LMzY~VJa-7D1em*XWP~K>y7QWThBSb;=#n?S zVJbLG0*H7w_yzv7)(AxL={`0U7?^q)Keq`Qz&;ocHR5XP#5j{14EVIwR)%V{kR66+ z?>!jQ{`8+%IA*plJ){hYx}$@nz0C_niN$MQc9rlX+9{kBcv1_nl`;0dJ; z_{yBedAK=njJxJ#^;(xLtzyUT)*^($u1jL|!&ef`1_DI1+N-r)*2^BO)<3&#e$qs# zj$5ZW(hi)DyG5y2I7U-Q3J$3dT|oxxl($W6Eh^qDm}HjV?+ z`OL&43a3tad4C-(GrIDnvHQuA@1IOAU3|{xgd&<7uNTNXODuGrd~Y2oACvi|i2pY` zct6!RW@6O~8cWDEz+!yBAK8*-NpD)bWN zj){vQU}3Rl4g+bHCc-i44gcsFnG-&zQ|MI*GkzUmWNb;Kdojn9J42^H=*+F!8se?+ zwK}DzSmKQu%Y>73sh|`v_l~z@V<^rIri)UK)~XIHUX9STLJfj+qsa;M?NZNvl2}=3 zQ7aoHGEJ*9Dexvz(CD=YrN85t%$P5Z_SzE*u@D}s#heSYdZ~WSsXpXsL3Tvmkf{60 z3K(W3gxLRlBi{N{Z1rM_H;5#=NOKrZbYbjUUD;}Xk-hHw?2aV#4M+UbmLAUhr}(H5 zxD9&gwUX0=4BcWph5D>e?S||K*Sv>pcp(XN`LS$gb#vyVk0y3uE+N`6Y}0PlvY{uk z(IppKdfj<>igmoqyT#YIA74yUFlN`CANyxfKp-iQzUmJuDd{dSvGX*^GnG$XEr-Tm2LfH`QnOepSz+ zC{d7?*n9<<^RM-^h!CHDq$V<4sU7#2#L^?WLiC=u>ms(uqZoJsVcnmSteC?VsozkZ zpq5T)20w+K-4%}C<1urE10A?Ev#KjXGcx*;T%w5sIE6ST?D|1qBr)l?>Li!w$bM@R$9{>L7)lA?^ z#F3@um0QpC5XtsCEvxZn(X>y2z^cLEVg&xdF*)AYrCcZt^2=cnR4`UYdcaojdj7E; z36hkfNBkSo$7h6l1EC}p1ebf8jq_*6PJiuABd_V@PT2k%`_!7hNuq|3m)Ys7%H~LN zY<@?71lPAdO-0^@f+SYXiImix_BmUxSBS>V2Tq%*rx_`xMk@Lb}y-XCP6s4tR;~t!7Hml2IiuBABrLKl^50Pth`$5xR^c*r zkC7LW(D=E5tjcnDkr}$F={u(YdX+jH4O@h~5La6Arpq6ArB$%nM#$mtw~MME!c&;w zFwK5jam4P5-bD!~wJV+tqnPPXar||!cckUbEww%84Kbo!#JgDj&nTwfMSz>1sT=A_ zV&4qcP3(yRQR4rDug_X&R4LjHuorV0R6T;A^P@#m#7D{0{)^AqVtp@H^5m9kn{v$A zZgiq~MkT^*Ux-;Mt>=>HIw;{5@vXI;DVqDdDh-z4|MGKuQ&gX?a;%_Aq#&z3K_-QE z$eGr~fbUdLQ~HP+sx?;nr3~q33T1n%Zl8SN!t#y=yW056aNjtM#qTerkIBngL3Ls4 z)f+}+lJu{q89t7`%_A?+FY~XKm%|C4u}0NojqIAG*W%A^-@I7y+zS?G$g#Tn^ovh` zo`#u&C2*-q>ocNkEX-D02zh|UgPo$&3pb^+Gk+@&{E4!q%sE?=i@m3hGR!ywiJZ!R z4OPDXlL>?6*|3LuZYmQflBv zf`ds{V%W}oS!7m;;V_oqVTI8G)h9*?$Im3Km5ge>>vG#2t6%eYgKGZFh!1I9P(7*m zJj5PPDf)^3Iv~V+J#O&Y{>zaZPNNrBZ!UUgLj^mxwi_<8Q{@+ANfu;j5Ww1SjQDO| zjbjt{gT6#~&T<@{lhsHgG0r!m;1?bgZ^ja!cU2qUl|#rScuZO$K>tAcfU`&?War_X ze9`T~0Y)Gt*Th11D?=fx8PWx>p^Wf>{|q_5B;8)vJM2HrowSh;9jhuwW?e)fmH*|$ z9AOm~{fXafevsMRcOO3SS6`@<|1m&H`6~Rp8&hVr%4Jv|*PEsOi{@(yp3?Ndn%G)0 z72f!S#0j6NpEw@!j(+*7;sr9jy*5~;3@U)3i~v_Oo+_a``?QE5pBmdJpy*{LeCPxU z(}qF)xNB#H(0M^ECXYKQkzY38(y^gi@Ex`PHsDw_5vagH$`B}kwhCIcRrLd$X-w4t zG6$26fy%&YV<0dSaW5#fAG-!cAM&@yfGbYzF%}o$LJ;UJ%gpIT4H|0E9uro!NXrc& zm-NzIfingHPdOASUVN0VOz|%iTY;h#rYwV^H#PL(M=~W2X!(rk(@r7|YhfCp-K5D{M_W*+%aSN-3>5jnt){JC|@D~s+RUdZABDGFY2+PwL8*Mef@L2*xxYY9U{E z(kUvxi?LXGU2MWRnG+Tmi51q?77S`10^zXwGpI1DQX<%mp)BO5jnajM(uM6UYi!Af z%p3hcxlN-`sB7gA=(1)F?gA*};bVM-G8+Q*hCh0qHW$VVo=q_1ek2ki=$qU zl@B%+>RJ*DZr;P7)|db|3UO4K;IKO?ZivYlJPaImXn2L1a`*xMaWr$%S-XoThuNwF z0O@T@I1O2A(T2LdU?d8qO#w4Mf-yoo=W%Vb3{D&(BKN?>F^U4A{HUSlIDjES@?*BPa?3bj5vtyrv7#Xpq_uvd>qK$60u2>#H3Xc)p(qSYw6A}KI&l91US2HOfUj)eL(#9t zQBd^U*uYh`bpV&#_VoYr4*A0?z&kzx{zTNya~g8_v2<(7nRFav|8eqq+XaZXOPKCe zNBf(8kjvuz@25^IQ1rcPpI~#5eZ8xv$DI0bts4!%L9))E+8v=hwo1#r7>H%p6GwwA z`Kz~~&?lVDrMd=v7 z&K#Eq6}UtKSt7`!XaycLJ`8p3Kh+z*!!612xAff2}}lg^wNDLLPIyk`wg zvMccLKR9+wFO9Jt5TA1TaGYCKX=mc)&J{|GV_o_60I=S%|23WPDRcuteAQ6^?DS=V z{L;NbjqcV1pHCDgHNSwxW`RYZe*gsTL1bz*R^T^(-^#hx4T1Q0A&r2N>l8Ux7TZLr zU>j$^JrL;|$DJOupTWCX>`>S7Q7069z8+6bJp&VP-1*)77Kn@lATqWw9XD4%whUne z62)u`;K~;i4078w`Wx`@qTf(7if9Rgn#YrC9Deu{yy^7yCs;L*DH2?KcMaKqze@+g z;uH3DHz--2_f&CX3V?x*F%#_nI&a0lJSGX;98`JIS;6lGt+L};#+^v6z=_Uayx!*hB&kx6`&{eA=P;0>L~Kk4+?Z~{IL zf!Z3LL92Ej4}oURZaO(TK4_rgw&4*g{WMDw=}Gjlq` z&H3pkk6YKCmMO7janNU1rI~dxSEZFCJWYWub3085E;prceJ51t@>T?m6eAA9G)zLf zBa!3$kxR1?*Xr8!$UJN-^auYv$(4+%pHMq z+4Q6wBVjkczYz)}+PeQSuze09!rT!)4~{8&wJ$#o)J+N62auqy46{zV*|s%Xg2^0B`Q>}Zl(%?Q!jVOJh8 z*C1UPp+U>|)I2~yYzzwz-Hs?c#gV&@368{N*ec-Lqvlb(j`|X|3OUdsG>kamsH2^m zNRF1EPxu?skkc>aZpJ$98Y)Y-+Tv*6TceQ}_$uua@sSxN?-Cv_Z=sLG3+LxUeU@)D zf;qvA18oWhT9D=6XbL^*{g~S7RsJaZ@Sz`4(~8UgL~q{YXNOeE-?Pk>k0;ZLO-bNo z5);TH7msE83nTJ>&Mfsr*6!(w#(We#O|pZGF)9?KJFn?o)8XKh`jlxo3U9Fv2Ya3a`5S}F6{pFhS&zwHZG3X90(DSHO6BJ#lY#!e=V#z<^&GOo2t)f^Rw?&Zy< zVxj6RKgZ6LRQxlyTIySoGHD&2pJgTeh2bSi!UE?KY}k$b_d&JyvD9yd5vBb4;maTN z!K?l!h+0zI{;M*yrFpE$#CJs=!kt=FFwSCZOAD&s<;#*!ymDF zF;rAe8MCHJd0Fgv_U09XGt#8aC#H9J9&4Ew$v(MAU9+?;k^1ANtHzw`y*>;178*71 zIOMF)-ic#YUP67uE6@Wp8>CJNKmTWct>tHordBM&wuyw_PusfiQBj3Xz?v zs~B~oZJ##DS(SYNv7C|RwWH*F=#J-RhN@gO^Gjqt-# zpWX&?jC${SDIE zM{BcKo0sVSrc?|C#sdUahEumL1;&L_w;+PpcA+4i{oLT2Gc?x1Z@Af$erZX*S9Pav zh^Tq#@5}v%?}UEoou4XnL!t}%G(seZY&~R-?W{Sdja{f>X=NW12+GIt_PpgC=~E@} zllt{GNaV2C8**QHr{me+y@&Z^&(|6c^{^+sB;p>g%}1Po5zHo^ zxlqk0y{UGW??>+^7P1~>Dk3@2JvTunqrw__{>qM9jva2@**57*-`6z(Z-{?IKD%1w zC_cy~{TNgxB3V$eq2o#3VtnJvkxDS9HND}sUw^#1&+&6mP>gROI+=mV)r^g4LNM&7 zg<|WK&e=dhRDPa(r249IRfIRO@1sk_FQ2~?Jr$QzzA1mo%fc5BxB0E+MaZ_pWT>x^ zda38(7nYa8i%yXo-Ai-99{kz{cxQ0L^YO`mbVal)k;3C2VKKgEYJsoKdDN5GcT(Kh zLbPPMmyaG2Nt^QpfH~)IN8JCww3H?nDkz<`9JCg4}A}5eP@(JV;Id0tq36T7`xY(O>WR=; zjHy(3r}UVvh`^M}B+pOkm#IA0N~Gt%9!6@Gmy3yc-frfb3u~M`2|Z()_oVK_ zK}rewK2wx-A-)`o3t-y?u(b5BTh$qPTbk3Ect1o6iM)Fr!)6f2>jRD%m@rLH#`@(F zE8#^ODAQu=`+Ae%)?(5Y_GMGa6`8vMT}E+F)Mwe``>%yYk~IYe|C@bKD655HBvI^7 z8;FTpf08d<)eg3YMaOMMQ;4dB#ECEsiz$X9WYaDTGJ&hTUsp0ym;0T~No79i$>wU3 zwnV>*8(%>6RrKFyo^bE?c9G$M{UZbUg^GF_H%GDRhJBvwKG@D&0%G= zMHf5fAMj73oEzbD?;}}ve0{-MfQ+r>-~Sj;g+AOF9|uHyy$Q_lW-pKOsuNgLCcQ&` zV%|r}%5p|7N<WTKX$zLpVfOEv}fsatz#S@7P zoUvEu`40XsbHpq?Cijm1D$X<3Gcsnee^T0x_>{)lbD}OHDzzp2s|H>(6sU4p^;Q=6 z&veEr|1gWYW_C_tEP;W(%)kRbhn8>1FhF$`LU!sMbL+iBzk`rlzGLTD!q6cU(+P%d zBnDN}%yS9SnVu507TxiXUBwG=8+XmmuKh})Y*R}f@pcL^6OpC;lyi`e!}u#rMo~!% z1=@NYDJDIb;zFM9`Y|v6v~v$Fn7JcJg>@((9(yf6E@)ghx&$RyzR+9xmz{DV(^$P1)w)^Hd^R@q(W%)pYQEjCu z?s?n%KRmoP=_jvBFVxHnpz?VbnKM%JCGPLI%ZMn{1KC8!F?xA=`lW~vXPbX&KjCrL z0_XFa#ylHLuPSwYuaXmk{6p@wEvDOOJ^TEk<8+}XLZSQr0Kh;$zrL5gV#Ym^AFq#p zpYeakgxSM9zvAnYe)c}=wG)FLNky-Q2OA$JVHE=qFCfK9$FE!HG{0Fr&gL321Xr$c zcj3928PmzF*pi)vvjelBmBCDRP?3t9Tca!69Y>-jedbM z(VTXPJL{F>sM7A)ciR6xn%?rPLXX`Ibc)OBF3Uz^NIGg2?5ds8z<&7k(d1Anv2Us7 zJgUFnL(V?S1LXxB(AYsSKJz@KP`%k|wp~A5FQ07a8|E1&M3IIl8Gw(EMi_rELbw)5i?In2W^u(&_v% zP6yOL04k9=WJ+F#@Zsxt?vN_Kui)>u#7%E1<9O0^KGT<~pmqO}8Q$UnMAtJfjg0?` zFU~LC2k%z#esGr_@1}cN!~69i65eSuH4Xpn!mxPHQ?u>b|D`tKJ#&bH_lSEWygw<{ zWIJME2)sw!$>1#c>jrcS-816x?4+ zpy#<)YCsR7lJuar+@*r<-Kz)v$%U|Wp_%T57%Qp*B~}9nZES6 z1i3QpcC2r-lxYq5juuNZJx($ukoR23(M&(LX)?9m83gj9bYw)~{=+Z9;2xucyPpp3 zL4jw;QM|YR1MuCT;{7@PepCGAbsgU8)~YaWTFc-Kwe#&~UVhE^|GYMA|NfOM(Eyy? zp9kyTC`)Fg?M?FVxznBUndxeqV*`1+`um_wG8v50UJnmH|>8QOcR z+WzD4_Y%?hS_|96Cf~Db)MkBw@z!=G)1%Mm`qM?Xx7MG^ua)}K z;c1%oy>&sD{*=mOy7)1Df9h~;YyD~30;xYO&}Mr2w$}Pn=OgE}KY3o%_NN6elK%9< zE4uzT%nO5e5tFH5y&mr$d$opl+&l^I zCnsw%?K!VCystlSUh#hAc`e>gK2Pxe^hF)sYgRC3?g|~=dzhE28Gply3&5A#=7!=+ zTTg{A8|H@O%g0?3Uz#Urn*Oh@2>9}a=JUjt{h!nFg0 zHOmX+v?}iWS55cpo?iPOIZlTX5f~S)+pYOCr za7Xd=2cujoh$y?a!CPFLc9DhSk8j}nOvHKF7>+mQF30aOIraV#&dE9TtD`J#JRiCK z(|AT{^2gWlz2y|0uVZlnCbb$rx1<2^q|vB4o6Zyd9n zuWy2u6gybGLs!y{?H3n<=;M$-X#*Z}?iQ^HhlR|8xTtYCy;O%zX1QN%p%aEV?r#OJ zvj>-l6Yf>yGwc?@EhcTCOO={02Yv1x8W<8~-JCv_}MO6KX5zf(g zFgPM6xR+Vo%k!Yd9wP{jdz-Lv5AVt2gih;;FkEi!y`FDeD;O>7dH3b@r$)Fw=Id7) zlD0!Ct`GS7)dny@P?+8$i|%F>&%UbT-hG7j-DlJs>X6P4pN% zz-umY128=evz_z<5pNRzjqdpW!(5RrTJR&T^Ne9f}oAF|C5Exe>(@_f~v{V zr8Dj5*NDu`u1)xjwg5Vlc-J?l{TK@ZyEaOn;O}V%^}lK7NwOMiveiSr_&|#a?jv~# z>jiS?{Xd8m2FY2H4iKYBRp9xanEiq$+d#{`P`c0z?+LTIo02wIJ+^G&_|Lp_ApzcO z(AeZmz!lb>Iw;2pW5b__M0Nrp!B4Gqo1r|pO>D%7ux^4+AY z#}_j5O~!_=h)`UC!TI`S(AN1R+7r#I^D@74b%726y6h(cEHXTPamz;O;JpP=yn7;t z_u?>j;!G!Z;#j={Hd-^AUDq`iGUW@9lO4_hlAOpQ0xjj*f?{JPk|IvtD2vGs5FX5* z+)iSl-*<9RaVgaUFnsf|Cw(9-wl`W6zQYS8fTv9SvJ>ryp3M)YXZu7+UyQyZ`dN-I zE8JzfN%`V=_iFJn=xl6^bohCsOFOQs3Gwj@IS z)`TC)nnY_HnEttP*!Q1k@mujm5;-s_9E4%AZ1yONAuXVevi)@V$DNP0_Fk<-n zlTmXM#d1)dbn%D(AzYb}_G~YdyVYQ6iXVgZ?emfBsXQh%-Pdj^^Sy`YV(F1w+;5ri zy7ls42N9kiUj`qG;-uX)o*HBgV?zn*3eTPJ172s~=U}jy=yEF*G((j9q196sOTK?& z7qWUPV{z3}X`aU(i{{^gJY>f_{8acFsu<6dSls#mT5AE&FuW3H5&U-Xo~fo!5LEG< z#V7-~kEC(CYYW7n&#msS#U~cyri1NHyWrl|{Fc>Yr>_w4yWkaOh#zBpE@IE*$?tJF zj7@s~8TPW9{nUat(CD6OlQ@DTeZZN53c}#)jV2Pdhm^oRYWiQXn5w^61n(LAjMkF` zw8`Q8;<3kQ8KD>L@~y!enmjklZ;y*x)wesz)gENjAQi)A5|>EtJL7H;ldqh7u+`A8 zc09$q2ZLGRyn^?%|M)r>*k^D~L;Yw&6t3$!Ogy^qAf@?KV(&75Y?k!}*}M)sUqftX zFW!@OHF)^U8cHA z5Ya#qJ4in36t(9>ekNV}Om1p3C2bJLJS1(7F~3U}I+$?JnYuZ~OcoT~_^7-&1~gxW zb}AxU>QhkPUjK@N#;9)><-7AIq8J}MTRG0(e#DVuqRXa|rR#_}9lgIs;P_xPhTa+tOPP}uHY}-^ex|v9|ZUY4- zZ2;*RC;s_<)bzMIrO63~$YkApH>9$f5vg(?O|SR+(0aT6g_Z3CIXAy3S|61vI}mdB zjeUn%V+(%y{uthUmek#>se3H_z->T**o12ju6dNMbTN_o%!UNm3sri*amue7Pv_`v zOzlnjp;{XpUz;W9$@p*Z5RdH?I01~U%+^_GT%h*y9_|CGP|ngR8`NbGFL$&5LGE#Z%JO{lGw_nB06kyQ*EgR8nadqCSQP;3a28jU z5TyE*6xTRh7508oxY>56T7WEbiITj-RZ?PiIA(#c*Fg&9&T^6K1y{4wNy2f7Xtg|5s@d;UyTuBASvf>L`(y;MRR$R4q$Q~rL2p-xd*o#Z+ z;uCt)c%}AP#jXmctrDOtsVH_hf@Y???JegQG+vm@Rp;^wKCmldNNC~`lr);_x&1hN*&g9V^H zh2yA78f3E})HWN4ysXF+MPaWpSI>p&mX~5R<&aXju@DuJOHxI3ajCr)Bz_i1fvcvb zy4GnfvZ`Xm^@XIMuT(qC@t~?o%22HP-sVa$4~O{Wj)B7+kQ9_w`xeU+kZ_i{T%!y~ z`jfC)QWdLRPDgnuG^u%JwF^4=ELRm#-2Jk-JH4vX-+P%&3eW2@>R{>#787Yj26W&< zDZI{NDz2%qSCs;NkFPc>rW2_mwo_^EpoYzGZm1=%plFI&dN-Ww+&RaY)tN&Dsb_Ld zVPDQX!BSw&8D+^ekDOu-Lct98s{M@eesKlLW}OYp9ByJ2ffjlO2)kJ#N-Vw8KHmw5 zpbsn`2Q+j(X-?9cQ9$~s{Y1xQ>Wi7Ka_G2K4oFBs)TX|~RP=q8(`YYU``T-vr*jT_ z#Vl^F9b1+dskO96gUWE&@s`xke}8za#Xov+i+>HchaYY6KLP&?@b66+->1UAoe#J8 zJ@D^ocpohO_zbzU=~QZyl56TDUE?gCZ3&lm@c4;R?BbeoVSJ%2y$p=83cJN((TA5_ zgi0?d5jC}qz-tNmo@mw3?*~gr*%Yw}Qkz02;FRYA$sjx~SjT7DY_qFe*dLRUZPx0N zIg&cDBwK>VPq!69_XVZ_Eg5kl5I>WRUuv(gL;ny80va|sEu@8ODHuJsFvVsmEEeWk zBC7%0UI9G`LdP%Ydq)Ctp;^t8cg8EKj6iRnq{$>}NSsp)Cy z>FI;hGtx5$Ck{>;oIE&XaO&W+!Rdns56&2znUR>0l#!f~l98H`mXV$@I3pt?GZTuK z3E5{t(wPt~6CTO+iIV-uSAm-Z?&B}2eW}oyoWkN)Gk?psa8nu$v}D)=h(!FL@#* zM2lAUwY~jwXdk!C2b01PHkF|9(P=%J^#MXdD`}-hjd5%Qnn8#!o6D&)NVVqo0!#m9e8OkQudlIw~rN{RpEbb#fud z`&ls7a_XiZxPD;sm)OhaqDAQdKkXIfn&MipGNBRVaH`FPWs}EL%IFoITWc-IyHKkcu)Lt`JfaMGZwiC=5 zb7k>-8y(2X?J|nS!vQxAWx<4nE{A#GfJh#gN{w#5mgBB9&w~L9nCt30IxXkGDLORm zX}UOMkb}UOmidLlf?()s^ODOa7YvJ3j8iOS)upruGppf$)CwlkbPARB3RGuHOU+hZD$3FeO>XAahX}Rvd!OBPhrNH!`CGKkaH0!w^dP>)bgPN?TX$!X}$d z=cs4&DX%K2a6vN;m6gT~4rj%nvf(pEnDc9yr`Y%v_WAH;Kxg%-+_J?OWdIwzthTzU znmh#KbC9{P8f3B9<*cqm>FCT(nP73NYiKG&Y~Jdr{@(rI(LHfEXq`~ z-2o#-oad;mt~5(Ci|kBvZE=;O0xbc8@oamQy%xrCn6@$uXD7u_g@j z@7d6zClkK#Et!_3?O^9}IwR#$NamI_`Df}fH%{(77q7+}8s`Z@E^m(ZEZ zmo$c5omd;xxidG&%rWx=9QICt7a#=@!TNA!_~VeFxXwV|)pMktmu2RLm^qp{_!rzU zj3e5W8(M3xsWsOYmpdRNp~~SL z;v97jY>#S}lbZ*Xw{x>9T#hnzKF?g`s;sG#HH6&T9Ma?A!^}y81`kT)IH@bhEk_hB zwL;9wput>dubnGVdU&&%g918jbc)bctQhf%Zq~(Da*z|E@oUPfK;~_gcF_2BVEk5B z*k;y&cp`(!dXCAbvbd@knAYYj!?vatw_Gnt|Df&$om2u+MYRU@bD3)B6s8oRuS1#_ zEVI+;w{jDO@kJTr|4He@Nl7M7dMY=%03MAlfCr-s z;OEFGMV3NtZJH^Fk_M0jR{#}kS;@K3Zz{!#gP1rv+(zl8DN*ev67 zl_&WVi|~;)7zU7&MnBEYM$;#0$d5+mj2Z_afmE;RF%r^IAK8p3wdPsiF@Y0Z(kD#m-`yg#1m$201X1Lp{eN^zR%vqA7vq9F0@& zoHCx1)G7@m{TQD=-oh2+L%Uff7g&W+LJ^lgdNlm7jIxc&pQ!JoltuE1+2kM!+(DjT zE8;JT8nAn{1Na9uf3dTShQmJRs-YpIC$&huGzv80tn%69xjYbsk#I?wouLhUNuQMh z527hjN1nmB%V`7X=!Y&QY2*@6vkr$EOu~*SEZRF--qF1S@_>ryU!jmXYKluxQt)X- z^=#X$@(R2113w8WKfna4QhuoAtdukY`e)a;G`~x9za>BYehcOgrO)D$IrgejU4vlz z+dvaR*V4X_gEOsNRt?>XPEKjEk%CMdKW=>fr12n16F_C;3VEPDa&o6|AU3w5d>cwn zt_1|80QFfAQ2||BUBf~86#=rrl{;)`*-Bk~CUU#{O$Kcx`k2_}#Y9~MzQn}=r0dY1 zWJC)?aA+}t9DRzGs7>hv4$f7jzSR666Yr&V{3wQlH;%}^Wdb)QXW|%3FjYRfxU#&W zF4!9m3=HU@c1dK&xP_Yo1{#+#C@GOs1{3U}0B$JZ;wPY;Ps$7upDk5u`KN^xX_`u8`P2BC`~$uprqgfu z*w4zUD@yI!8PWiG!343>1Oq!n`I3?f>eQZaPWZ?jflEo$$R8~awVOkfsEz54s+ zYtol!T(U;dP-!KXP~TqyBsE0*lnh<|ly~U_lC$*^1kFFKpiV2vq066=tZiS^!GN}X zeu$id<(DC67bKsc)AHTf?Oc(ry zqD1!nFfiwH{eBpj5)F^vX$F`VkZ&!)KCDacp^FFq?9RLg2kbl=tZ%6Dg$Q4SbJkPg z=K!4jn9h7Znps_45i-6NET^e`r!z%sZGC_?KI4RlB!yQB8lwKKy~kPv4G^k*L)9ZP zJa8Q1&;$&uzc!f=_=I>LSgkg;|0oy(;sn&xseK3FLSq8^0CQj2oj!mSiN5li>C*!u zYva*S14!u+el0uLo}mWB)P{9pcA}< zs!GW`-c&`B@J$p(lJQx8@k`Iy5I4--xm4vPjQtQts{G_vR;`?jB8izgFUTX zKvYpZTWh@P1|aaB%)+VTFYN$9{`xw*w%Sz_#w=ImBF>Jr)1rYyUK zp$!fa^zxaeonM$=rEYHtHUvo?tO-vhtilbFbn_R`TVZ@tS?gzBq@-kaZMaZ^Ym-Z$;+Si> z(;<}LIH<&8u@pKpV5~D z7pD8%$eg`?{R>b(Zfvcg=1tK4-s+s`xwKBWiP}m?(KeV^sy3%kVJ%Ne#~9(okv4Qt zee{2*0>O&UiV)L1D+5Ee+G7io&PkLj3zKaXxD+{f`)c^$KU~08{zC7cXw^o5sHyS} zQW{NY`z8u?JqKCVS!^zGIh@s%W;$g}*9MW5WX1AYvVqboSxeCJ>(|Dt%~PwE?`9EW4n@C($@P>>N)nI zH0VRAzYT_SZvR`(j{roL|8FoA)sjb4>T|@ubyJM#N$Kaq21~V_lekAzKW+Cvr$VRL z=znc3Ynw1v#?C=3+*TN-ZR3w~&)}TQn@W|S&&@tk69^U^-?pkcxiI-hkbw5{Fpge} z=2sO+Bka*I4C8`?lc#S+NnasDha=s40{vP_LWNn5mLs)LU^?1xRHBiFLYKOWIn@)T%tH-6K&2+`SQ;JR( zz*W0yOK42Z8_bf)Tlxq?A3MuxtLNFuXQ}Jf)%D$_)vlQpcJ<3AU4)IFrnHaHtkgcC zbk01ahxO>U3(yS)x>fTP6}Zk@dP=vpC=WG^ZmpgzS%><^LF{0~1j8L~dP5T}gTUvx zP)zca{*eYOud*q7k!(0=<#J;2YKmu;S4ajrK?=9*gxEb6&=houcfd9sSq?*lXIH+H zKi#;1&dZI#uieD@5BLkVz9ewDMCdgL?0SN8NlCkYNL`kIYa-gxLj%_*sZLvZdYj^h zSyT};ewZa2zsGtj^?l$nKK*(Uy*^}?rI3{|p<7;4pAhlGEdo-P3aNs^$W!?Bci|Vs zv<2VR*Y~xd{X^o{+WN`X>eps|V*8&|=#Y8q${)%4O=Z=kemzk5_U#)`pT0~!!B&Q9 z^+{k6|9`hX{+r+b&HR{y4U{ikWDl_|r_e^X3&$tOXTJ{8hOZk5c0mrAxX5u75~E2I z7XLzI+$iYJ2D$&18C(_ z>dbkJNwhC(54cp{r{bX`0fl3hJEZ`Y?7XH$8=t(^td9y7A*9~cub*aq2eW@7)%WK) z{5jjZA?#JDwry^YzPe}u;X?fd8$ZhX0rI8KyJRrOY<8ey1Z9Uru<^@pZogLS%ShAL z+P6WhXC=O5DYvEFgZ@_JSjnVs(D00X-I^VvwvTS}RG9a{?EIkR)xQt?9YA@M?QF%a z`5YH@Rl6ZpVBh%m9B1y@;x7aLy524RMaan4w)kHH_u2Fo|1Z~b+-`7t4&b<$!7ctZ z;1+@V6SyycTQQL1F3xE2?*{iMxJ76;4QcVe0q#g}*MWN)+|(O5t|`04{}i}UIgl>6 zZ-U#9$Z@}b`)_dhkuCnANgVeTxQoHH+Sr;C=*d;sj`qp&Zw< zsKp;Uoa2UqTLbPia6iiCxXCv``{r=mZQvS4a@Q5+Y2bBq54 zaG#tF?E~%(a80>T?+VBd+}c~=Hz?mXYg_#Hfm`8(@{H!VuU&u}xU=U$`eUGcw?jR_ z{R-SSz%|^_;_t!(KHye>`^lY9KXC853))ZMxJe5kUvMA3yTu8`GNcP1dh88-1~|+ZZWtg!2JZ=riqXrxT!Zm{lM)$3B;%#+6~-U;KoemxPO3q zJGckI{RQ0Q1}M)IXjgFeg8Kou)BeD5XTjYB?#RX#zjZ3d{Sn+x!Tk)}q0>O#{s`p& zcLKNrrgPjQ;976yxVOQr0r%>ATm11hzy;i=z#R|nN8q-ATQmdY;69KyaQA@wMlr`- z;)Zse3F(1b3hu+;z76hXaAQh1F7{6?{sM6S0PYLmz78%|%5k^<8RQ4&_PkJDaF>H? z1-4FhI*E9-2Y?k48Y@<);B)cY};rf7DX;W z5ET`X668-^+cs&_HZ%x=x)!}|kxLN7Rc&Z)1!+YUL9qx*gCGcsAP9ocsGbUlkR3F&1&O$>dl_-``-7QGiT16bLPy<{JZf3Gw-1s=hE)LWuV{Rqk`Z` z_mZFU3?u%&9@PTA3TB>f7^U~)cY$F%0lF`w9fGO!4j0u>UhpT-a|!Ve(*D5@!A5$e zJ07BaUTPS%U@`a-*bKIU$(K>myVs(374-vFgB!uI^9`fqQFy=~!1x7*;d_jB z2Cf0!WrmUWIQ0R#>Zoth0gOYcLCJ1`EMG8t_|0e*(@0mw*-ESKz7}INspc zO2ZiO9PJt01#ANM2a^`l9fBp`F<=XL2AHyhdIINx3&A$<7BKBb!}uPQ&tf0-Jna;` z32dn1JO@2D8OF9RaQwhyz-I6UFl(t{JPO9&%<%_{z-v~K4{-L2)b}#OSPoW!{{c6E z-C*`DhLQCW`M=dLD#1+fEpR#bJs5YJVQlj<`3K(z+re8JY3J3{<14f)@N+Qbc8>FF zKa1I&Jd_Hifm_$KEWct$h%UTzp~fkfDK>;m~t=25iA4OgLCdPjO$ylgKc2k{j@7^HkkP?=RG(B zG-_yP;977AnDu~RZ2unh1pXad4?YKaAEe&jr~QCuf}P;gVD3ZIPb=*QJRR%?p8)e$ z&>lYEItrcw8V_@Pz(O$QL-fHD!MH~Z<6+PT_JIxHajQAbD-GiTa4y&bHiE~rasJjC z#=T%M_&eAPPXCDG`>0{u1(t&U1zW+KHJqo98OH5k8Tb>p7M%1k=gH$7N3a6?9&87X zT1)xsXm4N@_${~|{J&2q{}Xg`U^Vy^*a;r`Ddm6CFs=jr;OAgBIN>wO|CC|;8(aZ? z0vh#(ao{@24_*z{fggc!PjjApPWi#Bzy|OG(DMxK<_pRXUI8|O?}CZXa^3!t@`IOx z&EQ*LLxW*_1{OYN7<;x;elQQreV%x51-JwZg3p6VFL2zyBH!RoU^jTy*W`N@{SdeU zTn8q-NdCT~p9Amxo_^>h!+7vV+86lh&*b}M!}t_z0DlENjfOG$7xE464Ys`sFPQe4 zVN`+T;7ec!xEjoAV!Y6a9@q%FUuV1}zrlI`<@X!p8!Q0*pdWk^+yFL#S#Q!mg5}_P zuodhDlbShCekC8^9$-C~0UB>{T?O;N0^Co`2&66 zhu|u(159aQywycJ0#60o!6LByUB(^1^BYV7jrR=WZZHi@{Db2JE(PTi)$#wMJ%EeA zbzl&Td!KPZH~j~=0rY|A^w5uhpMl+AY%e^mhH(H`2Obh2pC1^;X;=2Pz5D3@RvX3~upZoRBl&Bi9|CK@aYnCle?))j>QyD+NO!Ml10NXGt8&*E zh9|C9)q&;U*pKN?!BX)2ZFQzl(>K^E<=en_Huc`ry zz_{;ek6;m43ATdE!OR~RPk`0nDliDPfW99&f50Yi1DLvj;|f-S@yWfa6HEdNexjcT z8^DQR+Ruhj2G)R0VA3zNqrH07GB6K}@8moJ%fU7<2qx}LJ^h#bf*U|TIOkW|C%6tw z-3OiDCKQBqyTIjOQ4i-Y*a4>OM?Lo%#$2!stOfG|91n0Un6iJb$`2AhzE?Gal?Tuc`p{3K zKESR6dsW6p@_!Ki;IUw~GK^xd#)vWQ1k++-jJ04H*a^;#jWIGaI6h!0nC*@+R)Gzm zF#-P(F-A7%2P?p_BV&vfumbcPOnrg{V9BT$qZ-@*t^?=9#TdzlP@mvAV8J#qMlIL^ zc7oo~=w_k=mV!xR;05P`$%m4E@NO_KKE}vBtXHi7>%pXLV~n2aKltk;@(pHYlkXj4jP9w#f$vQt zUt?p8FTffwJ-1hRc8W3PgXQ3RU?(_oI_29r#yAOF3qA{ZFTk`K@i?w!YR-7UuWH`oB~d<^xR6l3IrDeW@P>{X4-uYF(} zSO%^K>p(;1+h8K-nZ@x3z2IC)2kXGrw* zk9>mp3#b>c>>A>k=ld7Z56FBUY!Iw~hxvZz4df3jUd-_WeM=|@^L>96?F_71O8%Jd zS1sc_0b6gO9D)zf4w(06tmswUpdUK1TllRyEUC#qkh3 zxB_ehn?TRo)SKAf;rRX)Ua(4D|MxC-(DNSo2aCb6M^j!f18jevehT!qk{>Yt1N^~a za2@FXkoN4QJ+7wSK~Edy6I_EGEC##9zLtDVB0rzde}TTwD92>@KgSPD?&f$-!JZsY z&0r^(n@#;?22`V9RzPLSvz!wHsuuKk11e=I@zVmT9_&0WpwgyMo)ZG96f7$Us8+#~ z11cjIJSCv2z`9cdDhT>c52)GGrQM46pe0euH^(paShs1+H zi3i6XM?9DYc7oZU=Ugz0YpQSy4RbU&~38tJ) zJv7kXz*evcENi45<`4&#fpxE;4>rC>IseY@4=E?;*(s=e=kR;ypsEF{b`7f3e~{1J zNC)%5d0-h>4c3A6U_00fdXj=_1Ly_g-9l~C?k*aSL`j^S|gYtmJY2**|%nqvD%jG&kI%xcjbg&N0E#>!_)F;>pc7p9^ zQ9oCJXLB6EjK7nAuoHCui{p6?I$-f!>Jv213#zeKQV-{Ie8A*?k{__~0*?DUbc?B1 z(6~6LvaiDL63P!IUm8>s=aY}i&~8^Ej?x{`{co2HO_| zRngUy^J?k?OkNmN1=pZg&T#?#|0X{Ru`lBIf_2xz54KkXm8YEgxSrz!dT$^f<+9Gr z@%=Y;unw$R0x#&j5&uQ-S8;s6{F`WxVA)dI&u1l20A_@1EZ0IU4?T@U}g)CbslAN2@!-VYz> zd4Tg3%y=-U+Q9sW&;^TEaQ@vu{KK>}Fk>bDU|nrct&`u6!B@%e$2qQG#uMnt?v9L#v0^BMHM0RN5Xt_rFe(Dx$xpz#vzri$`{1)%rk zpeh6N8)-kF?-lCpCh#@dEm+({JCWaSP`;)3y@~!(88-%13)ub^$Ny&V9r6qM-^U*` zKA?V=aa>l@{=mjI>gyKb*U*1}WgnAY(6^TQ0oy+zKex&_j^hosuY>nC%JVhn16cMQ z?YbJ>pMt6w^#4Y^fMwm3^LFytLpj05Udjn}2H?8`{XUKZ=-Ej8o%p-^R4JG-vQM>v z$)oyI+H&Hy=~HE3@|Zrg7W9H$pbzxiMI4xRm#nMwsa&uToCCIlWuOt?r0fk-q9Yw z##x*f&&m9#PZd2+-0^*?39LGS^1Ogw0p$TZ!NgVkK8f8N^81q9|K&UajccgKmt>p@zu3XGVh1;feIfP!GInq*SO%ub?{e}Fdj5?+ z*a~JelD|dpgLPmH*a|i_%DA>qtph#Ra(o-*zAb#OP;M{-EC%zyDsb*Aa-HIQ0b4;o z*a@xzJr(G^BI9G~N8&-xtHgt8U=^4P)`4@tR=b{{{R#eHB3KNjfn}fnHj7 zT&|n&ff?}|Rq5w)ecY(3!B(*0bGZ&~RBJz%>)%FYd_f$T3VIVZs){eS27=?zQ2|pmQMg19l!tzB|zQ>qg}VJyWUIZ_zoH z{`Hc05n$5X1PCdBklDU_-}$A^xn+z>?A+8Y*Z^ip|LQ_tNpAE%ysh;N{tLI3mAb1(6$s0YyZ67>vv8>#1j%*Saj zU{y2i0Q9#~4?$UHqp?HkpcKI(5h=XIalCnbLyN&heH0c;0XY?OY3b^&_2D33x1 zEC=g;ryO9}9~=)O#>oF4b}*xx^2pb{dx(?Yy&F|tOpH+++^Cj=`F-etjbJzE*+{uu zF-Dal9`qYZ)qu&Pl!}jyFd_tJFHMYCEO!N5mN3?UiZ* zJ9khjYa~2lm8u2vcUCHS6!t`=s=?%4mGZ=qza*t9!1mphGPWTR1nOVpj2@@JO?Y)47MJwRQ9&mvy_?#wjZTbH)tHK zR7L{nQ^*(S$x*5UG-fE}+m3wAB!8eMU#YPkc#l&m4@^D@UeJ3o`P`oL+3GzdZx?v3r5?b}3Z-%riMyUSuxhbVseg$v>cEMhZ>du4pyy_#yt|UGWlB|o zo?Db^1B=0K&<`f=#_wC<7rYJr-9lf5AFKCu#{BDeBc{|7vs?VAxGGF3q$TmM?wf8^ zCr$~TsR@ZQwi|PKT$yq7u1AbNBxP?aKAV48-Q6m~*z}*I&*on~>He{s(&c^7veZ$U z(yMj}OJn+Ml#`Ym7S9yQsFHh{l8~F=IXNNDOlJsdtZ+zYh!DC5Iv`!T(6s~7llse@ zpJ$eQ)SZj(iZONzFTeb2C5|be;eMG8DJQ|R$mN6CI61*H*;EJ} z2YG8|&6pk~eD0Iszdr zksT^WhO0Lrby8R%CcXAX%e(qG7ZX1*wO9Q`*PFcmyPULK(#{JVpA6S*>UPo?LvH!` z&{>Xcos|0@ojRYBLix;hoif_W+{`g1m(=;H(ImZhR8!mYKK#9Uuh;;wfV7|@Aksvn zlW-LU5fK#;=_=BsL`r}p5WFHF(gXygMmh*2^gw7TORUmB|`K9efU}E2&dG@h@n>$P^-9*S;s2G+nG;~V?Gwv+E zO?+CyuzcMk>U$2t*+^`eQrFa|`-J&droBd9+R->?kfgd6 za%HdQDta%82oD>AW~f5*_D_#=(~>s6>@?!Wi~m91*+q*WV-Q2ochV;EuCclrmml>~ zqAkvW*!Lw37-HmO#eG$MLefpu7m3>W=aOnF^0gqbUsY0?-M*`JcZFW^-Hou#jx{S4 zTHO%X`q78^)oS^PZR~N*?6rczRkc8iNB1kfYE`+<7kn&_$alRO(A7cF^b5En>QM_1 zk+=JxQCMLHO>Nf|8UWw**B)Apwb6My^ru%OhHUm49I(&>jmMMv9U+oS&rhq$$1t`K zjVcR%31aU=c+-VO78B5$AXWBRZKB54gPcR(W{F1w0X_k;9U3&tiS91&}X9onz`GfZf{Ko;EjG|QeD(1(4J022&olZCF^lB4Lcf2r&0 z$-8_kf3D{wmd|c|Rc8Zp--S z4&)tGemm&(CwTNbwbVc-#nAOUk*CpZzK?5UGa7rS=FO74d%xSz>m1Rzk^r@MhqJx# z*=w7=Z_s+rSK(eD=7_+2%X?CQ64UVDd}5HUn^`4%8f3ix*&@u3p(MY}zA$oj#U7;? z9&o`gHe-4c(4tge&XjX0^^ZW-Jk1HbiFx$^ZR>hbt0jxqy8L-4@c`;v05*$G)~s@b z)NGxIvI&i~MAXQ&4M%kGMCV9759{~qpXv}}*G|MKo<(SJM*a((^b2`Vyz}GwP+Z&{ zqSz!uIn$Xia%}g5J4$pw4RI5neZ#Z{*d3=8>(bcr7SHo^&V||+0)}-Q$4K#CT6e}S zCuG_P*VxAkpxOYv(*}OU0J**xh5*8^JW3)YL<62;A3haii`C666(Ba1JgO@l7Kw2c z3$f0He$q|dHQkq|{sd8ELwBmS{!%yIec>#TpXBDgZ#f_YUHRt4%X!VqmYh9ynqBTF zHwl_v*v|wDoxua7S1bm0ETrA0N0inyD}OZt{M%Qmt}mYLw#^b9_WFR%p3`&O|A5!} z*!A(%LQl$0!p@I#uraVIoP!y2s-nN`cD*}Oz#PA!S4;cxa0^BL&WS8dBwkYeQ01Xs zU-o57*kgX-*@{!WSEB{_rK7;M=ATt~>MOKfqjjj#PLHG)@feYt`=oUIudGIoFY;3r z9Yw#S)vEF%n)ubugiwW@ufH7^Qv6raWTF`GW>kxaD73OcAZ>PC|@#vtuAytGZ)yl&;4 zZJ2(qxc{BqDcb;zXbqHboXC_l0!>S=F*b8~IOcghVlrxl`))&bV6*X$lv%LLL3rqq zz?wKIZVSsGQtND{lykh3LPVUpVA;XDXUSpEN>WQ7dml#c1xRC6J9^Ze`l3f8@I#5T zld)p9g*S;VvC_JrugiQd1Z{`YgS~ehp}4zN!nzDGNR$yTNKd#yGxho}-C{W*bji(0 zjWg9Vw?OyiZWL7VNVKsP}GHw(D*OKYAe}QS%Y9W z3$%W@EP3t=5?|;ySfvKa8e~gTVW^Rl{=i ze~VOu@;l)X8qEBeX6R+gt<8xy8@SK&DNsKxTHwsAqviuVQ0$Z&AN!eDryODvap%E9 zMfG&E-_AvX01cIAZgm~OJ^?7atJgn5TH@A_mu0Q$w`f9j?OA_(e+8U$1iftiCOK

6uB%%4Tu&RyV&FhjX&6cP2GiAQzF2Vo!SaQ}pg=?!L+-L(KoiqTP_1_;Kd0R`#pdNz4qa5Zo@xg6dBE^GFp23;SK(`svA<4^ph7zD0cwDEKaiS7D~7D<2X2%7e?>EL&$ vqyKdgBFzr~`hPHP@Bwfe&|3YUx2pgdjkX3J{-4M+zqb-07FNmz@cZ;H3vVO5 delta 4357 zcmZu!cQo8v*A_pW(Sp%C(MtqFh~7IPYJ^}2A_+;<{6vXJlrYT1AbNN85+!=?y+)62 z^e*_4taa~u-}QcHt+Vz%&))ky=lt`my)g@96ANSzeJuco3JVL10E-8aUJNk-fM2%M z0Z_yL?W7eENkd$64Na|w8f-I&uzn?dQ`QEZ-KIOJFAJ+tzMgf5!#GtmUPY(gNZozDQ?=7BQaAzYt*AOZG%cwt zoV&unfBt+@G>H#iG!ZmZDp z_lf}`rk~xBd%hf9(S@eSEIr%ckNctmd(Ag)wy(pZd~FTT`8cA~SZoy#VYRPzAZFdT z$m1{S4@1-hd8t8M_5><~V!rB0YsMgkq3Mx;h8D$h04Md4=sZD2EU(VAF~RLL7!|{p z_>fWmuMuKp^hM8&70Rzaa%;zB`}sz3~%zO_xJ*Np{>;|AWBkJP(7OPY*y&9Lo338&7u zeTscIT*0U}M0VkoUHmH5LUKl=BTlU!!qj4n=c!^`rIw9SZ|CvEuwm^LGDdnG5MQ^w zaDBHKd^d#dI4E&$F$3aEQ(55dwOpEa=3!gId~rIsFMl?y*MrvCJ)sdfmV*{xyj-?# zuX>BVY#@^#N1sJt=z8M9v${h;v6?#*XG8GPs{YKr@}@SG*ll?+)A6u6Sc$xnX>Q>XpScu zgFQY#5CSJ-to2x-99a`ns3sn(nW>flVd z`g`dNS>MF*RbQsNp114zB%a+VGDRDkAOR0sKS@|tt-3$%tYT9L8k3Vnumh_iyFU-Nchaa)5O^0ZqkU2O~~8Gc!ZD1I5wr(3T(St%k88)v7w9FCVqs>m^bOjL;XXv5xTy^>CK z<%b7p33)K$v^jBb+)jEy&vEKXLdi(@)pn#}vH~@MDrBL%MyVw#XB!~7No0|hhmZ*_ zHZ?oDHMxS3xz@$9E7W_rkxCgr=sUM&ZG8g?|X^pi=tPV z2;nZUWCJ4;v!0zPDBhEdTKM&PM|*hc23}1{GXYZqDIORI-r<$~RG+`|c~G86*-Yn7 zh9zZ@Cl^B+yUro^jQbXm*wSK^S|J2rAT> zL$+HNj$-k~7LRAeX?e)H2{SZe(8 zP#Byue~Fy%ZYs$WHfu>w=)U-}cohYOj#xQa9Tv!t;nnwe{p`#BfQZg^I;>>rGusl4 zzZ7C;FV3<}xk~owh@#OWGhe*ymZw_Ivj&Xy+^`K|^Ds&FuHk@TJ}Vx}xbYtcq3<<` zG8H%G{IW9cuJp6Ta_-mn#z(Ep2{F^4Mpo`0(_30$#xG=dGNgXogUgEl ze&D*(quX#Dc-FCUS`@))seFG^3Wv8c-n=Ev&;o}JeHgyByNk*es``2I4QH?0-4>U< zd$u_>Ndgwx8s8ms(tfBoPLrPyW}$+-9n`tNca|d3>Z*Zk_=MAOl1w08wuNVSQqi2h3ad{ z?U4D0@`o{vlb?)=VT@V8Oa{;|MM#?aJeMEXLf1aLxh1DWS$ubC zlt01BsWnQ=aCr4B$xMLVpv^+%@apmcT$!QEgR0Q|SkMR!xs{b^9ut6m;yIOgBG>dd zxA?{DlD^qm;vc=8vi{Le5U#x<{mq!&?d|Zh^Q)^765_7T6z|H2;UeDy-4u*~(P6l& zUgW&sohT>Ke3hh#G8R!EgkiI-QGuR-Qn+Jo(|~zy4dTLxD}JP=9&GESIw^<;>$N7i z$GQyQ*41C={X&vPs9eIYSn_8@Eed}GzKUfWg=>^r@l3}SRZ_5z={)cLOLtq)^ZdL) zb!>r)-OFP_VZdTUNc|fn=HC4)#$w}A^=uWq?6OXOUFN)ym&8OHOAT~`Nyo=gLPT${ zQbrJY9g?gV$=TKMfPzjnhjFnqFt7XDboyOwA6ep+t%_A4ZT$|t^u;+}YDRaLO@gTR z`&;Z`#Iqi!fG**Et1wa~bu#%;Cqfw-95cDGf>D~4*w2;*5vCX`m*y4ejJUV zmvu6siWMsK25Esu#OTs{59TP1gsBvQeZ5ZdRt6$Z);5hf*51esY9H0eX82eqM)1|e zhvJ)BYhHXwE>&8r=YmO*HH~szVCSwFv+O=G_ZpTkizQEV%855fN*}K}?yNq_hI`!y z1NBL_B0ZJP{qR%s_mw#ySHz{wdLx>@``SP>Is}cr@oQsOqR{|};%_ht!jH1v0jP?# zp#a6NvuWW1odN|m7S;`1EG&LRqcYR)Fkz*{Jivkdy8vam2qgb;t``CGKP6|Nn*SB> ztDpYWEA#$Sg8H-s5P}hD>f~4hJ|OYe36v6xNEO^fR-P2#%W^z79o3px|HcyyVXXSRGet zv=!Kpx(SvhZ*4p9JJ9LN4+p(J*zp;86yM%FGD)jaReSa!s}6)D-Ntz08Z+0)${np- zcp=E@>Yt#}gOiu(;%9R29DbBMHnHa_eUb3Bmax{zyBFmB|b zoi7JVAnVL=j-Xq}WL-|C;jPfbv4(Kg*fHd=Z!r`mVJn>+1;uHnr3YuOpM#A18*y_r z0|l@9xF*+E6%t%gUv|1}2Bj%#T34phl@sHYi&7*i@%wFmKu>casr#J{w+1JwskjSb zg$(?V?~m$SD+35wUk$;V8oZMZlO-#Jn)X_teAbP3z~1=7&WOl&jBPD78UZtA!W%&> zDz{$2qDsqxn@jTQAP4v3I`#s{san519T&kHMdW8r25W(|CJ3%;b=)iY1)KhDsOq zWXbCbrA#qu`}>a5$0JS}#1SsFd7Q)gba~2dAbwKNj-{!hZ+H!n;{2VyT01px!)1BV zilizrQ;b!~)0hO~%O?%Yq+Qw%M6|r+pMmug(a2O2_tsTZUvsar+}Y)~F)@}j-;82- zbdV!a&3^FNY;JguUCLXLzb}!coQfS5`3-$oSjDoU^V4+UlJ5zMEfAu)NGt6g*01jf z#!4=QW%hpESWxpw$SA`7YyFkofkkAORIE{3uwa}}d#+hpBRsgWcg936JF_`j0dwi^jA&&vm|5rH9VroJr8W&h=xKu67b3M*B>$AO=b&&B5`pUzX{@sCnZjR8gXUB!Ak-JHk?e*4j4P%83 zhmL)E<@8n$BwJhUnJq9|FkG6bgVy@q=9jg@u5GG{bFHj!%VmQ+u+=o$vrK}3avvkC zuXO{P3ixN23jh*e|U91I}9!qUwHw&;9-ec~C-_U%af1|WYP z!nh4EV825DC+xNjSO8W%04k~n&?NhRKEMD3e+Li-NQU3)$jbIV!ud-K|6b{DXXpM& zB}BFS65Btd(>nkI09*W5=+h#A`#)r&yT8(=|DEQ(3();x{4d@8o9>q_dkb6o_q9s^ Q9%_6SAjk7t{jJjf2lm-Cxc~qF diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/README.md b/Triton_Inference_Server_Python_API/examples/fastapi/README.md index 8aa5c5e5..ab9eb060 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/README.md +++ b/Triton_Inference_Server_Python_API/examples/fastapi/README.md @@ -109,6 +109,18 @@ curl -s http://localhost:8000/models/llama-3-8b-instruct | jq . } ``` + +#### Completion + +#### chat completion + +curl http://localhost:8000/chat/completions -H "Content-Type: application/json" -d '{"model":"llama-3-8b-instruct","messages":[{"role":"system","content":"you are a helpful assistant."},{"role":"user","content":"Hello!"}]}' + + ## Comparison python3 -m vllm.entrypoints.openai.api_server --model "meta-llama/Meta-Llama-3-8B-Instruct" + +curl http://localhost:8000/v1/chat/completions -H "Content-Type: application/json" -d '{"model":"meta-llama/Meta-Llama-3-8B-Instruct","messages":[{"role":"system","content":"you are a helpful assistant."},{"role":"user","content":"Hello!"}]}' + + diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index 8673ce57..1f1e72c8 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -11,7 +11,11 @@ from fastapi import FastAPI, HTTPException, Request from fastapi.responses import StreamingResponse +from huggingface_hub.utils import chunk_iterable from openai_protocol_types import ( + ChatCompletionChoice, + ChatCompletionFinishReason, + ChatCompletionResponseMessage, ChatCompletionStreamingResponseChoice, ChatCompletionStreamResponseDelta, Choice, @@ -29,7 +33,8 @@ from vllm.transformers_utils.tokenizer import get_tokenizer owned_by = "ACME" - +default_role = "assistant" +add_generation_prompt_default = True model_map = { "llama-3-8b-instruct": Model( @@ -73,27 +78,52 @@ def streaming_chat_completion_response(request_id, created, model, role, responses): - first_response = True + # first chunk + + choice = ChatCompletionStreamingResponseChoice( + index=0, + delta=ChatCompletionStreamResponseDelta( + role=role, content=None, function_call=None + ), + logprobs=None, + finish_reason=None, + ) + chunk = CreateChatCompletionStreamResponse( + id=request_id, + choices=[choice], + created=created, + model=model, + system_fingerprint=None, + object=ObjectType.chat_completion_chunk, + ) + yield f"data: {chunk.json(exclude_unset=True)}\n\n" for response in responses: - choice = ChatCompletionResponse + try: + text = response.outputs["text_output"].to_string_array()[0] + except: + text = str(response.outputs["text_output"].to_bytes_array()[0]) - choice = Choice( - finish_reason=FinishReason.stop if response.final else None, + choice = ChatCompletionStreamingResponseChoice( index=0, + delta=ChatCompletionStreamResponseDelta( + role=None, content=text, function_call=None + ), logprobs=None, - text=response.outputs["text_output"].to_string_array()[0], + finish_reason=ChatCompletionFinishReason.stop if response.final else None, ) - response = CreateCompletionResponse( + + chunk = CreateChatCompletionStreamResponse( id=request_id, choices=[choice], - system_fingerprint=None, - object=ObjectType.text_completion, created=created, model=model, + system_fingerprint=None, + object=ObjectType.chat_completion_chunk, ) - yield f"data: {response.json(exclude_unset=True)}\n\n" + yield f"data: {chunk.json(exclude_unset=True)}\n\n" + yield "data: [DONE]\n\n" @@ -122,7 +152,9 @@ def create_chat_completion( ] prompt = tokenizer.apply_chat_template( - conversation=conversation, tokenize=False, add_generation_prompt=False + conversation=conversation, + tokenize=False, + add_generation_prompt=add_generation_prompt_default, ) request_id = f"cmpl-{uuid.uuid1()}" @@ -147,16 +179,44 @@ def create_chat_completion( ) ) - pass + response = list(responses)[-1] + + try: + text = response.outputs["text_output"].to_string_array()[0] + except: + text = str(response.outputs["text_output"].to_bytes_array()[0]) + + return CreateChatCompletionResponse( + id=request_id, + choices=[ + ChatCompletionChoice( + index=0, + message=ChatCompletionResponseMessage( + content=text, role=default_role, function_call=None + ), + logprobs=None, + finish_reason=ChatCompletionFinishReason.stop, + ) + ], + created=created, + model=request.model, + system_fingerprint=None, + object=ObjectType.chat_completion, + ) def streaming_completion_response(request_id, created, model, responses): for response in responses: + try: + text = response.outputs["text_output"].to_string_array()[0] + except: + text = str(response.outputs["text_output"].to_bytes_array()[0]) + choice = Choice( finish_reason=FinishReason.stop if response.final else None, index=0, logprobs=None, - text=response.outputs["text_output"].to_string_array()[0], + text=text, ) response = CreateCompletionResponse( id=request_id, diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py index 5a509616..b548076c 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py @@ -7,7 +7,7 @@ from enum import Enum from typing import Any, Dict, List, Optional, Union -from pydantic import AnyUrl, BaseModel, Extra, Field, confloat, conint +from pydantic import AnyUrl, BaseModel, Extra, Field, RootModel, confloat, conint class Error(BaseModel): @@ -37,8 +37,8 @@ class Model1(Enum): babbage_002 = "babbage-002" -class PromptItem(BaseModel): - __root__: List[Any] +class PromptItem(RootModel): + root: List[Any] class CreateCompletionRequest(BaseModel): @@ -363,7 +363,7 @@ class ChatCompletionStreamResponseDelta(BaseModel): description="Deprecated and replaced by `tool_calls`. The name and arguments of a function that should be called, as generated by the model.", ) tool_calls: Optional[List[ChatCompletionMessageToolCallChunk]] = None - role: Optional[Role6] = Field( + role: Optional[str] = Field( None, description="The role of the author of this message." ) @@ -408,7 +408,7 @@ class FunctionCall3(Enum): auto = "auto" -class FinishReason1(Enum): +class ChatCompletionFinishReason(Enum): stop = "stop" length = "length" tool_calls = "tool_calls" @@ -475,7 +475,7 @@ class ChatCompletionStreamingResponseChoice(BaseModel): logprobs: Optional[Logprobs2] = Field( None, description="Log probability information for the choice." ) - finish_reason: ChatCompletionFinishReason = Field( + finish_reason: ChatCompletionFinishReason | None = Field( ..., description="The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence,\n`length` if the maximum number of tokens specified in the request was reached,\n`content_filter` if content was omitted due to a flag from our content filters,\n`tool_calls` if the model called a tool, or `function_call` (deprecated) if the model called a function.\n", ) @@ -592,8 +592,8 @@ class CreateCompletionResponse(BaseModel): usage: Optional[CompletionUsage] = None -class ChatCompletionRequestMessageContentPart(BaseModel): - __root__: Union[ +class ChatCompletionRequestMessageContentPart(RootModel): + root: Union[ ChatCompletionRequestMessageContentPartText, ChatCompletionRequestMessageContentPartImage, ] @@ -620,17 +620,15 @@ class ChatCompletionTool(BaseModel): function: FunctionObject -class ChatCompletionToolChoiceOption(BaseModel): - __root__: Union[ - ChatCompletionToolChoiceOption1, ChatCompletionNamedToolChoice - ] = Field( +class ChatCompletionToolChoiceOption(RootModel): + root: Union[ChatCompletionToolChoiceOption1, ChatCompletionNamedToolChoice] = Field( ..., description='Controls which (if any) tool is called by the model.\n`none` means the model will not call any tool and instead generates a message.\n`auto` means the model can pick between generating a message or calling one or more tools.\n`required` means the model must call one or more tools.\nSpecifying a particular tool via `{"type": "function", "function": {"name": "my_function"}}` forces the model to call that tool.\n\n`none` is the default when no tools are present. `auto` is the default if tools are present.\n', ) -class ChatCompletionMessageToolCalls(BaseModel): - __root__: List[ChatCompletionMessageToolCall] = Field( +class ChatCompletionMessageToolCalls(RootModel): + root: List[ChatCompletionMessageToolCall] = Field( ..., description="The tool calls generated by the model, such as function calls.", ) @@ -639,15 +637,15 @@ class ChatCompletionMessageToolCalls(BaseModel): class ChatCompletionResponseMessage(BaseModel): content: str = Field(..., description="The contents of the message.") tool_calls: Optional[ChatCompletionMessageToolCalls] = None - role: Role5 = Field(..., description="The role of the author of this message.") + role: str = Field(..., description="The role of the author of this message.") function_call: Optional[FunctionCall] = Field( None, description="Deprecated and replaced by `tool_calls`. The name and arguments of a function that should be called, as generated by the model.", ) -class Choice1(BaseModel): - finish_reason: FinishReason1 = Field( +class ChatCompletionChoice(BaseModel): + finish_reason: ChatCompletionFinishReason = Field( ..., description="The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence,\n`length` if the maximum number of tokens specified in the request was reached,\n`content_filter` if content was omitted due to a flag from our content filters,\n`tool_calls` if the model called a tool, or `function_call` (deprecated) if the model called a function.\n", ) @@ -655,14 +653,14 @@ class Choice1(BaseModel): ..., description="The index of the choice in the list of choices." ) message: ChatCompletionResponseMessage - logprobs: Logprobs2 = Field( + logprobs: Logprobs2 | None = Field( ..., description="Log probability information for the choice." ) class CreateChatCompletionResponse(BaseModel): id: str = Field(..., description="A unique identifier for the chat completion.") - choices: List[Choice1] = Field( + choices: List[ChatCompletionChoice] = Field( ..., description="A list of chat completion choices. Can be more than one if `n` is greater than 1.", ) @@ -732,8 +730,8 @@ class ChatCompletionRequestAssistantMessage(BaseModel): ) -class ChatCompletionRequestMessage(BaseModel): - __root__: Union[ +class ChatCompletionRequestMessage(RootModel): + root: Union[ ChatCompletionRequestSystemMessage, ChatCompletionRequestUserMessage, ChatCompletionRequestAssistantMessage, @@ -743,11 +741,11 @@ class ChatCompletionRequestMessage(BaseModel): @property def role(self): - return self.role() + return self.root.role @property def content(self): - return self.content() + return self.root.content class CreateChatCompletionRequest(BaseModel): @@ -778,7 +776,7 @@ class CreateChatCompletionRequest(BaseModel): description="An integer between 0 and 20 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to `true` if this parameter is used.", ) max_tokens: Optional[int] = Field( - None, + 8168, description="The maximum number of [tokens](/tokenizer) that can be generated in the chat completion.\n\nThe total length of input tokens and generated tokens is limited by the model's context length. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens.\n", ) n: Optional[conint(ge=1, le=128)] = Field( @@ -807,7 +805,7 @@ class CreateChatCompletionRequest(BaseModel): description="If set, partial message deltas will be sent, like in ChatGPT. Tokens will be sent as data-only [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) as they become available, with the stream terminated by a `data: [DONE]` message. [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions).\n", ) temperature: Optional[confloat(ge=0.0, le=2.0)] = Field( - 1, + 0.7, description="What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or `top_p` but not both.\n", example=1, ) @@ -848,3 +846,4 @@ class ObjectType: list = Object.list text_completion = Object1.text_completion chat_completion_chunk = Object4.chat_completion_chunk + chat_completion = Object2.chat_completion From 73b3ce00637b6435df1a3e438af0839c39196878 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 7 May 2024 16:05:04 -0700 Subject: [PATCH 31/62] update with assitant strings --- .../fastapi-codegen/openai_protocol_types.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py index b548076c..5082bbea 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py @@ -181,6 +181,9 @@ class ChatCompletionRequestMessageContentPartText(BaseModel): class Role(Enum): system = "system" + def __str__(self): + return self.name + class ChatCompletionRequestSystemMessage(BaseModel): content: str = Field(..., description="The contents of the system message.") @@ -196,10 +199,16 @@ class ChatCompletionRequestSystemMessage(BaseModel): class Role1(Enum): user = "user" + def __str__(self): + return self.name + class Role2(Enum): assistant = "assistant" + def __str__(self): + return self.name + class FunctionCall(BaseModel): arguments: str = Field( @@ -212,6 +221,9 @@ class FunctionCall(BaseModel): class Role3(Enum): tool = "tool" + def __str__(self): + return self.name + class ChatCompletionRequestToolMessage(BaseModel): role: Role3 = Field( @@ -226,6 +238,9 @@ class ChatCompletionRequestToolMessage(BaseModel): class Role4(Enum): function = "function" + def __str__(self): + return self.name + class ChatCompletionRequestFunctionMessage(BaseModel): role: Role4 = Field( @@ -338,6 +353,9 @@ class ChatCompletionRole(Enum): class Role5(Enum): assistant = "assistant" + def __str__(self): + return self.name + class FunctionCall2(BaseModel): arguments: Optional[str] = Field( @@ -353,6 +371,9 @@ class Role6(Enum): assistant = "assistant" tool = "tool" + def __str__(self): + return self.name + class ChatCompletionStreamResponseDelta(BaseModel): content: Optional[str] = Field( From 57b5ac242109ede3cc006cf5f0e2aaa32a4bacdc Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 7 May 2024 19:30:45 -0700 Subject: [PATCH 32/62] updates for trt-llm - note code is not portable between vllm and trt-llm yet --- .../deps/requirements_trt_llm.txt | 1 - .../deps/requirements_vllm.txt | 2 +- .../docker/Dockerfile | 4 ++-- .../examples/fastapi/fastapi-codegen/main.py | 21 +++++++++++++------ 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt b/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt index 5c07da65..c86a4946 100644 --- a/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt +++ b/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt @@ -25,7 +25,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --extra-index-url https://pypi.nvidia.com/ -litellm[all] psutil pynvml>=11.5.0 tensorrt_llm==0.8.0 diff --git a/Triton_Inference_Server_Python_API/deps/requirements_vllm.txt b/Triton_Inference_Server_Python_API/deps/requirements_vllm.txt index ca4149ae..f3c4524d 100644 --- a/Triton_Inference_Server_Python_API/deps/requirements_vllm.txt +++ b/Triton_Inference_Server_Python_API/deps/requirements_vllm.txt @@ -24,4 +24,4 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -litellm[all] +vllm[all]==0.4.1 diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index 7566c804..79dc3057 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -55,7 +55,7 @@ RUN find /opt/tritonserver/python -maxdepth 1 -type f -name \ RUN pip3 install --force-reinstall --upgrade /tmp/tritonserver-2.46.0.dev0-py3-none-any.whl[all] -ARG TRITON_CLI_TAG="0.0.6" +ARG TRITON_CLI_TAG="0.0.7" RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${TRITON_CLI_TAG} @@ -69,7 +69,7 @@ RUN if [[ "$INCLUDE_PYTHON_REPL" != "" ]] ; then pip install --timeout=2000 -r / ARG FRAMEWORK=DIFFUSION -RUN if [[ "$FRAMEWORK" == "TRT_LLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_trt_llm.txt ; fi +# RUN if [[ "$FRAMEWORK" == "TRT_LLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_trt_llm.txt ; fi RUN if [[ "$FRAMEWORK" == "VLLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_vllm.txt ; fi diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index 1f1e72c8..acd197f5 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -9,6 +9,7 @@ from dataclasses import dataclass from typing import TypedDict +import numpy from fastapi import FastAPI, HTTPException, Request from fastapi.responses import StreamingResponse from huggingface_hub.utils import chunk_iterable @@ -30,7 +31,7 @@ ObjectType, ) from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast -from vllm.transformers_utils.tokenizer import get_tokenizer +from transformers_utils.tokenizer import get_tokenizer owned_by = "ACME" default_role = "assistant" @@ -63,6 +64,11 @@ for model in model_map.values(): server.load(model.id) +server.load("preprocessing") +server.load("postprocessing") +server.load("tensorrt_llm") + + app = FastAPI( title="OpenAI API", description="The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details.", @@ -165,11 +171,12 @@ def create_chat_completion( responses = model.infer( inputs={ - "text_input": [prompt], - "stream": [request.stream], - "exclude_input_in_output": [exclude_input_in_output], + "text_input": [[prompt]], + "stream": [[request.stream]], + "max_tokens": [[numpy.int32(request.max_tokens)]] + # "exclude_input_in_output": [exclude_input_in_output], }, - parameters=sampling_parameters, + # parameters=sampling_parameters, ) if request.stream: @@ -179,9 +186,11 @@ def create_chat_completion( ) ) - response = list(responses)[-1] + response = list(responses)[0] try: + print(response) + print(response.outputs) text = response.outputs["text_output"].to_string_array()[0] except: text = str(response.outputs["text_output"].to_bytes_array()[0]) From b1ec52faaf462819e08df7c79eae8a17b31f7cf1 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 7 May 2024 19:44:52 -0700 Subject: [PATCH 33/62] adding stop words --- .../examples/fastapi/fastapi-codegen/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index acd197f5..ca97ded4 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -173,7 +173,8 @@ def create_chat_completion( inputs={ "text_input": [[prompt]], "stream": [[request.stream]], - "max_tokens": [[numpy.int32(request.max_tokens)]] + "max_tokens": [[numpy.int32(request.max_tokens)]], + "stop_words": [["<|eot_id|>", "<|end_of_text|>"]] # "exclude_input_in_output": [exclude_input_in_output], }, # parameters=sampling_parameters, From c1dfd0a615d44cee9d1993e19ad351d32965c575 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 8 May 2024 00:49:08 -0700 Subject: [PATCH 34/62] updates to make vllm and trt-llm work in same code base --- .../deps/requirements_trt_llm.txt | 7 +- .../docker/Dockerfile | 2 +- .../examples/fastapi/fastapi-codegen/main.py | 99 +++++++++++-------- 3 files changed, 59 insertions(+), 49 deletions(-) diff --git a/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt b/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt index c86a4946..8f618a1a 100644 --- a/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt +++ b/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt @@ -24,8 +24,5 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---extra-index-url https://pypi.nvidia.com/ -psutil -pynvml>=11.5.0 -tensorrt_llm==0.8.0 -torch==2.1.2 +fastapi==0.111.0 +pydantic==2.7.1 diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index 79dc3057..5aebb0bb 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -69,7 +69,7 @@ RUN if [[ "$INCLUDE_PYTHON_REPL" != "" ]] ; then pip install --timeout=2000 -r / ARG FRAMEWORK=DIFFUSION -# RUN if [[ "$FRAMEWORK" == "TRT_LLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_trt_llm.txt ; fi +RUN if [[ "$FRAMEWORK" == "TRT_LLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_trt_llm.txt ; fi RUN if [[ "$FRAMEWORK" == "VLLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_vllm.txt ; fi diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index ca97ded4..55cf36cc 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -6,13 +6,11 @@ import time import uuid -from dataclasses import dataclass -from typing import TypedDict import numpy +import tritonserver from fastapi import FastAPI, HTTPException, Request from fastapi.responses import StreamingResponse -from huggingface_hub.utils import chunk_iterable from openai_protocol_types import ( ChatCompletionChoice, ChatCompletionFinishReason, @@ -30,29 +28,9 @@ Model, ObjectType, ) -from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast from transformers_utils.tokenizer import get_tokenizer - -owned_by = "ACME" -default_role = "assistant" -add_generation_prompt_default = True - -model_map = { - "llama-3-8b-instruct": Model( - id="llama-3-8b-instruct", - created=int(time.time()), - object=ObjectType.model, - owned_by=owned_by, - ) -} -tokenizer_map = { - "llama-3-8b-instruct": get_tokenizer( - tokenizer_name="meta-llama/Meta-Llama-3-8B-Instruct" - ) -} - - -import tritonserver +from triton_cli.constants import SUPPORTED_BACKENDS +from triton_cli.parser import KNOWN_MODEL_SOURCES as KNOWN_MODELS server = tritonserver.Server( model_repository="/workspace/llm-models", @@ -61,13 +39,30 @@ model_control_mode=tritonserver.ModelControlMode.EXPLICIT, ).start(wait_until_ready=True) -for model in model_map.values(): - server.load(model.id) -server.load("preprocessing") -server.load("postprocessing") -server.load("tensorrt_llm") +def load_model(server): + model = None + backends = [] + tokenizer = None + for model_name, version in server.models().keys(): + if version != -1: + continue + current_model = server.load(model_name) + backends.append(current_model.config()["backend"]) + if model_name in KNOWN_MODELS.keys(): + model = current_model + tokenizer = get_tokenizer(KNOWN_MODELS[model_name].replace("hf:", "")) + if model and tokenizer: + for backend in backends: + if backend in SUPPORTED_BACKENDS: + return model, int(time.time()), backend, tokenizer + return None, None, None, None + +model, model_create_time, backend, tokenizer = load_model(server) + +if not (model and backend and tokenizer and model_create_time): + raise Exception("Unknown Model") app = FastAPI( title="OpenAI API", @@ -105,10 +100,10 @@ def streaming_chat_completion_response(request_id, created, model, role, respons yield f"data: {chunk.json(exclude_unset=True)}\n\n" for response in responses: - try: - text = response.outputs["text_output"].to_string_array()[0] - except: + if "text_output" in response.outputs: text = str(response.outputs["text_output"].to_bytes_array()[0]) + else: + text = None choice = ChatCompletionStreamingResponseChoice( index=0, @@ -143,15 +138,15 @@ def create_chat_completion( Creates a model response for the given chat conversation. """ - if request.model not in model_map: + add_generation_prompt_default = True + default_role = "assistant" + + if request.model != model.name: raise HTTPException(status_code=404, detail=f"Unknown model: {request.model}") if request.n and request.n > 1: raise HTTPException(status_code=400, detail=f"Only single choice is supported") - model = server.model(request.model) - tokenizer = tokenizer_map[request.model] - conversation = [ {"role": str(message.role), "content": str(message.content)} for message in request.messages @@ -316,21 +311,39 @@ def create_completion( ) +owned_by = "ACME" + + @app.get("/models", response_model=ListModelsResponse, tags=["Models"]) def list_models() -> ListModelsResponse: """ Lists the currently available models, and provides basic information about each one such as the owner and availability. """ - return ListModelsResponse(object=ObjectType.list, data=list(model_map.values())) + + model_list = [ + Model( + id=model.name, + created=model_create_time, + object=ObjectType.model, + owned_by=owned_by, + ) + ] + + return ListModelsResponse(object=ObjectType.list, data=model_list) -@app.get("/models/{model}", response_model=Model, tags=["Models"]) -def retrieve_model(model: str) -> Model: +@app.get("/models/{model_name}", response_model=Model, tags=["Models"]) +def retrieve_model(model_name: str) -> Model: """ Retrieves a model instance, providing basic information about the model such as the owner and permissioning. """ - if model in model_map: - return model_map[model] + if model_name == model.name: + return Model( + id=model.name, + created=model_create_time, + object=ObjectType.model, + owned_by=owned_by, + ) - raise HTTPException(status_code=404, detail=f"Unknown model: {model}") + raise HTTPException(status_code=404, detail=f"Unknown model: {model_name}") From e759096b0ddf08f430c9eeb957035f83e2198bd0 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 8 May 2024 02:27:45 -0700 Subject: [PATCH 35/62] updates to support both trtlm and vllm --- .../examples/fastapi/fastapi-codegen/main.py | 120 +++++++++++------- 1 file changed, 72 insertions(+), 48 deletions(-) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index 55cf36cc..207cde96 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -100,10 +100,9 @@ def streaming_chat_completion_response(request_id, created, model, role, respons yield f"data: {chunk.json(exclude_unset=True)}\n\n" for response in responses: + text = None if "text_output" in response.outputs: text = str(response.outputs["text_output"].to_bytes_array()[0]) - else: - text = None choice = ChatCompletionStreamingResponseChoice( index=0, @@ -128,6 +127,59 @@ def streaming_chat_completion_response(request_id, created, model, role, respons yield "data: [DONE]\n\n" +def create_vllm_inference_request( + model, prompt, request: CreateChatCompletionRequest | CreateCompletionRequest +): + inputs = {} + sampling_parameters = request.copy( + exclude={"model", "stream", "messages", "prompt", "echo"} + ).dict() + inputs["text_input"] = [prompt] + inputs["stream"] = [request.stream] + exclude_input_in_output = True + echo = getattr(request, "echo", None) + if echo: + exclude_input_in_output = not echo + inputs["exclude_input_in_output"] = [exclude_input_in_output] + return model.create_request(inputs=inputs, parameters=sampling_parameters) + + +def create_trtllm_inference_request( + model, prompt, request: CreateChatCompletionRequest | CreateCompletionRequest +): + inputs = {} + if model.name == "llama-3-8b-instruct": + inputs["stop_words"] = [["<|eot_id|>", "<|end_of_text|>"]] + inputs["text_input"] = [[prompt]] + inputs["stream"] = [[request.stream]] + if request.max_tokens: + inputs["max_tokens"] = [[numpy.int32(request.max_tokens)]] + if request.stop: + if isinstance(request.stop, str): + request.stop = [request.stop] + inputs["stop_words"] = request.stop + if request.top_p: + inputs["top_p"] = [[numpy.float32(request.top_p)]] + if request.frequency_penalty: + inputs["frequence_penalty"] = [[numpy.int32(request.frequency_penalty)]] + if request.presence_penalty: + inputs["presence_penalty":] = [[numpy.int32(request.presence_penalty)]] + if request.seed: + inputs["random_seed"] = [[numpy.uint64(request.seed)]] + if request.temperature: + inputs["temperature"] = [[numpy.float32(request.temperature)]] + + return model.create_request(inputs=inputs) + + +create_inference_request = None + +if backend == "vllm": + create_inference_request = create_vllm_inference_request +elif backend == "tensorrtllm": + create_inference_request = create_trtllm_inference_request + + @app.post( "/chat/completions", response_model=CreateChatCompletionResponse, tags=["Chat"] ) @@ -138,6 +190,9 @@ def create_chat_completion( Creates a model response for the given chat conversation. """ + if not model or not tokenizer or not create_inference_request: + raise Exception("Unknown Model") + add_generation_prompt_default = True default_role = "assistant" @@ -160,20 +215,8 @@ def create_chat_completion( request_id = f"cmpl-{uuid.uuid1()}" created = int(time.time()) - exclude_input_in_output = True - sampling_parameters = request.copy(exclude={"model", "stream", "messages"}).dict() - - responses = model.infer( - inputs={ - "text_input": [[prompt]], - "stream": [[request.stream]], - "max_tokens": [[numpy.int32(request.max_tokens)]], - "stop_words": [["<|eot_id|>", "<|end_of_text|>"]] - # "exclude_input_in_output": [exclude_input_in_output], - }, - # parameters=sampling_parameters, - ) + responses = model.infer(create_inference_request(model, prompt, request)) if request.stream: return StreamingResponse( @@ -184,11 +227,8 @@ def create_chat_completion( response = list(responses)[0] - try: - print(response) - print(response.outputs) - text = response.outputs["text_output"].to_string_array()[0] - except: + text = None + if "text_output" in response.outputs: text = str(response.outputs["text_output"].to_bytes_array()[0]) return CreateChatCompletionResponse( @@ -212,9 +252,8 @@ def create_chat_completion( def streaming_completion_response(request_id, created, model, responses): for response in responses: - try: - text = response.outputs["text_output"].to_string_array()[0] - except: + text = None + if "text_output" in response.outputs: text = str(response.outputs["text_output"].to_bytes_array()[0]) choice = Choice( @@ -243,13 +282,15 @@ def create_completion( """ Creates a completion for the provided prompt and parameters. """ + + if not model or not tokenizer or not create_inference_request: + raise Exception("Unknown Model") + if request.suffix is not None: raise HTTPException(status_code=400, detail="suffix is not currently supported") - if request.model not in model_map: + if request.model != model.name: raise HTTPException(status_code=404, detail=f"Unknown model: {request.model}") - else: - model = server.model(model_map[request.model].id) if request.prompt is None: request.prompt = "<|endoftext|>" @@ -267,32 +308,15 @@ def create_completion( request_id = f"cmpl-{uuid.uuid1()}" created = int(time.time()) - exclude_input_in_output = True - - if request.echo: - exclude_input_in_output = False - - sampling_parameters = request.copy( - exclude={"model", "prompt", "stream", "echo"} - ).dict() - responses = model.infer( - inputs={ - "text_input": [request.prompt], - "stream": [request.stream], - "exclude_input_in_output": [exclude_input_in_output], - }, - parameters=sampling_parameters, - ) + responses = model.infer(create_inference_request(model, request.prompt, request)) if request.stream: return StreamingResponse( - streaming_completion_response(request_id, created, request.model, responses) + streaming_completion_response(request_id, created, model.name, responses) ) response = list(responses)[0] - - try: - text = response.outputs["text_output"].to_string_array()[0] - except: + text = None + if "text_output" in response.outputs: text = str(response.outputs["text_output"].to_bytes_array()[0]) choice = Choice( @@ -307,7 +331,7 @@ def create_completion( system_fingerprint=None, object=ObjectType.text_completion, created=created, - model=request.model, + model=model.name, ) From 25de682156a07f3c9b2cdc3f5f9f576bef87e123 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 8 May 2024 02:29:50 -0700 Subject: [PATCH 36/62] adding transformer utilities to get tokenizer - may need further tweaks --- .../transformers_utils/__init__.py | 0 .../transformers_utils/config.py | 68 ++++ .../transformers_utils/configs/__init__.py | 17 + .../transformers_utils/configs/chatglm.py | 71 ++++ .../transformers_utils/configs/dbrx.py | 277 +++++++++++++++ .../transformers_utils/configs/falcon.py | 85 +++++ .../transformers_utils/configs/jais.py | 242 +++++++++++++ .../transformers_utils/configs/mpt.py | 202 +++++++++++ .../transformers_utils/detokenizer.py | 325 ++++++++++++++++++ .../transformers_utils/tokenizer.py | 150 ++++++++ .../tokenizer_group/__init__.py | 34 ++ .../tokenizer_group/base_tokenizer_group.py | 55 +++ .../tokenizer_group/ray_tokenizer_group.py | 181 ++++++++++ .../tokenizer_group/tokenizer_group.py | 94 +++++ .../transformers_utils/tokenizers/__init__.py | 5 + .../transformers_utils/tokenizers/baichuan.py | 270 +++++++++++++++ 16 files changed, 2076 insertions(+) create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/__init__.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/config.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/__init__.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/chatglm.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/dbrx.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/falcon.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/jais.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/mpt.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/detokenizer.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/__init__.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/base_tokenizer_group.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/ray_tokenizer_group.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/tokenizer_group.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/__init__.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/baichuan.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/__init__.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/config.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/config.py new file mode 100644 index 00000000..9ae6b822 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/config.py @@ -0,0 +1,68 @@ +from typing import Dict, Optional + +from transformers import AutoConfig, PretrainedConfig +from vllm.transformers_utils.configs import ( + ChatGLMConfig, + DbrxConfig, + JAISConfig, + MPTConfig, + RWConfig, +) + +_CONFIG_REGISTRY: Dict[str, PretrainedConfig] = { + "chatglm": ChatGLMConfig, + "dbrx": DbrxConfig, + "mpt": MPTConfig, + "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct) + "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct) + "jais": JAISConfig, +} + + +def get_config( + model: str, + trust_remote_code: bool, + revision: Optional[str] = None, + code_revision: Optional[str] = None, +) -> PretrainedConfig: + try: + config = AutoConfig.from_pretrained( + model, + trust_remote_code=trust_remote_code, + revision=revision, + code_revision=code_revision, + ) + except ValueError as e: + if ( + not trust_remote_code + and "requires you to execute the configuration file" in str(e) + ): + err_msg = ( + "Failed to load the model config. If the model is a custom " + "model not yet available in the HuggingFace transformers " + "library, consider setting `trust_remote_code=True` in LLM " + "or using the `--trust-remote-code` flag in the CLI." + ) + raise RuntimeError(err_msg) from e + else: + raise e + if config.model_type in _CONFIG_REGISTRY: + config_class = _CONFIG_REGISTRY[config.model_type] + config = config_class.from_pretrained( + model, revision=revision, code_revision=code_revision + ) + return config + + +def get_hf_text_config(config: PretrainedConfig): + """Get the "sub" config relevant to llm for multi modal models. + No op for pure text models. + """ + if hasattr(config, "text_config"): + # The code operates under the assumption that text_config should have + # `num_attention_heads` (among others). Assert here to fail early + # if transformers config doesn't align with this assumption. + assert hasattr(config.text_config, "num_attention_heads") + return config.text_config + else: + return config diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/__init__.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/__init__.py new file mode 100644 index 00000000..8904e1b6 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/__init__.py @@ -0,0 +1,17 @@ +from vllm.transformers_utils.configs.chatglm import ChatGLMConfig +from vllm.transformers_utils.configs.dbrx import DbrxConfig + +# RWConfig is for the original tiiuae/falcon-40b(-instruct) and +# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the +# `FalconConfig` class from the official HuggingFace transformers library. +from vllm.transformers_utils.configs.falcon import RWConfig +from vllm.transformers_utils.configs.jais import JAISConfig +from vllm.transformers_utils.configs.mpt import MPTConfig + +__all__ = [ + "ChatGLMConfig", + "DbrxConfig", + "MPTConfig", + "RWConfig", + "JAISConfig", +] diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/chatglm.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/chatglm.py new file mode 100644 index 00000000..e6b95c79 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/chatglm.py @@ -0,0 +1,71 @@ +# coding=utf-8 +# Adapted from +# https://github.com/THUDM/ChatGLM2-6B +from transformers import PretrainedConfig + + +class ChatGLMConfig(PretrainedConfig): + model_type = "chatglm" + attribute_map = { + "num_hidden_layers": "num_layers", + "n_head_kv": "multi_query_group_num", + } + + def __init__( + self, + num_layers=28, + padded_vocab_size=65024, + hidden_size=4096, + ffn_hidden_size=13696, + kv_channels=128, + num_attention_heads=32, + seq_length=2048, + hidden_dropout=0.0, + attention_dropout=0.0, + layernorm_epsilon=1e-5, + rmsnorm=True, + apply_residual_connection_post_layernorm=False, + post_layer_norm=True, + add_bias_linear=False, + add_qkv_bias=False, + interleaved_qkv=False, + bias_dropout_fusion=True, + multi_query_attention=False, + multi_query_group_num=1, + apply_query_key_layer_scaling=True, + attention_softmax_in_fp32=True, + fp32_residual_connection=False, + quantization_bit=0, + pre_seq_len=None, + prefix_projection=False, + **kwargs + ): + self.num_layers = num_layers + self.vocab_size = padded_vocab_size + self.padded_vocab_size = padded_vocab_size + self.hidden_size = hidden_size + self.ffn_hidden_size = ffn_hidden_size + self.kv_channels = kv_channels + self.num_attention_heads = num_attention_heads + self.seq_length = seq_length + self.hidden_dropout = hidden_dropout + self.attention_dropout = attention_dropout + self.layernorm_epsilon = layernorm_epsilon + self.rmsnorm = rmsnorm + self.apply_residual_connection_post_layernorm = ( + apply_residual_connection_post_layernorm + ) + self.post_layer_norm = post_layer_norm + self.add_bias_linear = add_bias_linear + self.add_qkv_bias = add_qkv_bias + self.bias_dropout_fusion = bias_dropout_fusion + self.multi_query_attention = multi_query_attention + self.multi_query_group_num = multi_query_group_num + self.apply_query_key_layer_scaling = apply_query_key_layer_scaling + self.attention_softmax_in_fp32 = attention_softmax_in_fp32 + self.fp32_residual_connection = fp32_residual_connection + self.quantization_bit = quantization_bit + self.pre_seq_len = pre_seq_len + self.prefix_projection = prefix_projection + self.interleaved_qkv = interleaved_qkv + super().__init__(**kwargs) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/dbrx.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/dbrx.py new file mode 100644 index 00000000..1d2724f2 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/dbrx.py @@ -0,0 +1,277 @@ +# yapf: disable +# ruff: noqa: E501 +# coding=utf-8 +# Copied from +# https://huggingface.co/databricks/dbrx-base/blob/main/configuration_dbrx.py +"""Dbrx configuration.""" + +from typing import Any, Optional + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger(__name__) + +DBRX_PRETRAINED_CONFIG_ARCHIVE_MAP = {} # type: ignore + + +class DbrxAttentionConfig(PretrainedConfig): + """Configuration class for Dbrx Attention. + + [`DbrxAttention`] class. It is used to instantiate attention layers + according to the specified arguments, defining the layers architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + attn_pdrop (`float`, *optional*, defaults to 0.0): + The dropout probability for the attention layers. + clip_qkv (`float`, *optional*, defaults to None): + If not `None`, clip the queries, keys, and values in the attention layer to this value. + kv_n_heads (Optional[int]): For grouped_query_attention only, allow user to specify number of kv heads. + rope_theta (float): The base frequency for rope. + """ + + def __init__( + self, + attn_pdrop: float = 0, + clip_qkv: Optional[float] = None, + kv_n_heads: int = 1, + rope_theta: float = 10000.0, + **kwargs: Any, + ): + super().__init__(**kwargs) + self.attn_pdrop = attn_pdrop + self.clip_qkv = clip_qkv + self.kv_n_heads = kv_n_heads + self.rope_theta = rope_theta + + for k in ["model_type"]: + if k in kwargs: + kwargs.pop(k) + if len(kwargs) != 0: + raise ValueError(f"Found unknown {kwargs=}") + + @classmethod + def from_pretrained( + cls, pretrained_model_name_or_path: str, **kwargs: Any + ) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + + config_dict, kwargs = cls.get_config_dict( + pretrained_model_name_or_path, **kwargs + ) + + if config_dict.get("model_type") == "dbrx": + config_dict = config_dict["attn_config"] + + if ( + "model_type" in config_dict + and hasattr(cls, "model_type") + and config_dict["model_type"] != cls.model_type + ): + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." + ) + + return cls.from_dict(config_dict, **kwargs) + + +class DbrxFFNConfig(PretrainedConfig): + """Configuration class for Dbrx FFN. + + [`DbrxFFN`] class. It is used to instantiate feedforward layers according to + the specified arguments, defining the layers architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + ffn_act_fn (dict, optional): A dict specifying activation function for the FFN. + The dict should have a key 'name' with the value being the name of + the activation function along with any additional keyword arguments. + ffn_hidden_size (int, optional): The hidden size of the feedforward network. + moe_num_experts (int, optional): The number of experts in the mixture of experts layer. + moe_top_k (int, optional): The number of experts to use in the mixture of experts layer. + moe_jitter_eps (float, optional): The jitter epsilon for the mixture of experts layer. + moe_loss_weight (float, optional): The loss weight for the mixture of experts layer. + moe_normalize_expert_weights (float, optional): The normalization factor for the expert weights. + uniform_expert_assignment (bool, optional): Whether to use uniform expert assignment. + This should only be used for benchmarking purposes. + """ + + def __init__( + self, + ffn_act_fn: Optional[dict] = None, + ffn_hidden_size: int = 3584, + moe_num_experts: int = 4, + moe_top_k: int = 1, + moe_jitter_eps: Optional[float] = None, + moe_loss_weight: float = 0.01, + moe_normalize_expert_weights: Optional[float] = 1, + uniform_expert_assignment: bool = False, + **kwargs: Any, + ): + super().__init__() + if ffn_act_fn is None: + ffn_act_fn = {"name": "silu"} + self.ffn_act_fn = ffn_act_fn + self.ffn_hidden_size = ffn_hidden_size + self.moe_num_experts = moe_num_experts + self.moe_top_k = moe_top_k + self.moe_jitter_eps = moe_jitter_eps + self.moe_loss_weight = moe_loss_weight + self.moe_normalize_expert_weights = moe_normalize_expert_weights + self.uniform_expert_assignment = uniform_expert_assignment + + for k in ["model_type"]: + if k in kwargs: + kwargs.pop(k) + if len(kwargs) != 0: + raise ValueError(f"Found unknown {kwargs=}") + + @classmethod + def from_pretrained( + cls, pretrained_model_name_or_path: str, **kwargs: Any + ) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + + config_dict, kwargs = cls.get_config_dict( + pretrained_model_name_or_path, **kwargs + ) + + if config_dict.get("model_type") == "dbrx": + config_dict = config_dict["ffn_config"] + + if ( + "model_type" in config_dict + and hasattr(cls, "model_type") + and config_dict["model_type"] != cls.model_type + ): + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." + ) + + return cls.from_dict(config_dict, **kwargs) + + +class DbrxConfig(PretrainedConfig): + """Configuration class for Dbrx. + + [`DbrxModel`]. It is used to instantiate a Dbrx model according to the + specified arguments, defining the model architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + d_model (`int`, *optional*, defaults to 6144): + Dimensionality of the embeddings and hidden states. + n_heads (`int`, *optional*, defaults to 48): + Number of attention heads for each attention layer in the Transformer encoder. + n_layers (`int`, *optional*, defaults to 40): + Number of hidden layers in the Transformer encoder. + max_seq_len (`int`, *optional*, defaults to 32768): + The maximum sequence length of the model. + vocab_size (`int`, *optional*, defaults to 100352): + Vocabulary size of the Dbrx model. Defines the maximum number of different tokens that can be represented by + the `inputs_ids` passed when calling [`DbrxModel`]. + resid_pdrop (`float`, *optional*, defaults to 0.0): + The dropout probability applied to the attention output before combining with residual. + emb_pdrop (`float`, *optional*, defaults to 0.0): + The dropout probability for the embedding layer. + attn_config (`dict`, *optional*): + A dictionary used to configure the model's attention module. + ffn_config (`dict`, *optional*): + A dictionary used to configure the model's FFN module. + use_cache (`bool`, *optional*, defaults to `False`): + Whether or not the model should return the last key/values attentions (not used by all models). + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + output_router_logits (`bool`, *optional*, defaults to `False`): + Whether or not the router logits should be returned by the model. Enabling this will also + allow the model to output the auxiliary loss. See [here]() for more details + router_aux_loss_coef (`float`, *optional*, defaults to 0.001): + The aux loss factor for the total loss. + + + Example: + ```python + >>> from transformers import DbrxConfig, DbrxModel + + >>> # Initializing a Dbrx configuration + >>> configuration = DbrxConfig() + + >>> # Initializing a model (with random weights) from the configuration + >>> model = DbrxModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ``` + """ + + model_type = "dbrx" + attribute_map = { + "num_attention_heads": "n_heads", + "hidden_size": "d_model", + "num_hidden_layers": "n_layers", + "max_position_embeddings": "max_seq_len", + } + + def __init__( + self, + d_model: int = 2048, + n_heads: int = 16, + n_layers: int = 24, + max_seq_len: int = 2048, + vocab_size: int = 32000, + resid_pdrop: float = 0.0, + emb_pdrop: float = 0.0, + attn_config: Optional[DbrxAttentionConfig] = None, + ffn_config: Optional[DbrxFFNConfig] = None, + use_cache: bool = True, + initializer_range: float = 0.02, + output_router_logits: bool = False, + router_aux_loss_coef: float = 0.05, + **kwargs: Any, + ): + if attn_config is None: + self.attn_config = DbrxAttentionConfig() + elif isinstance(attn_config, dict): + self.attn_config = DbrxAttentionConfig(**attn_config) + else: + self.attn_config = attn_config + + if ffn_config is None: + self.ffn_config = DbrxFFNConfig() + elif isinstance(ffn_config, dict): + self.ffn_config = DbrxFFNConfig(**ffn_config) + else: + self.ffn_config = ffn_config + + self.d_model = d_model + self.n_heads = n_heads + self.n_layers = n_layers + self.max_seq_len = max_seq_len + self.vocab_size = vocab_size + self.resid_pdrop = resid_pdrop + self.emb_pdrop = emb_pdrop + self.use_cache = use_cache + self.initializer_range = initializer_range + self.output_router_logits = output_router_logits + self.router_aux_loss_coef = router_aux_loss_coef + + tie_word_embeddings = kwargs.pop("tie_word_embeddings", False) + if tie_word_embeddings: + raise ValueError( + "tie_word_embeddings is not supported for Dbrx models." + ) + + super().__init__( + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/falcon.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/falcon.py new file mode 100644 index 00000000..45366356 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/falcon.py @@ -0,0 +1,85 @@ +# Adapted from +# https://huggingface.co/tiiuae/falcon-7b/blob/main/configuration_RW.py +# Copyright 2023 The vLLM team. +# Copyright 2022 the Big Science Workshop and HuggingFace Inc. team. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Falcon configuration""" +from transformers.configuration_utils import PretrainedConfig + + +class RWConfig(PretrainedConfig): + model_type = "falcon" + keys_to_ignore_at_inference = ["past_key_values"] + attribute_map = { + "num_hidden_layers": "n_layer", + "num_attention_heads": "n_head", + "num_kv_heads": "n_head_kv", + } + + def __init__( + self, + vocab_size=250880, + hidden_size=64, + n_layer=2, + n_head=8, + layer_norm_epsilon=1e-5, + initializer_range=0.02, + use_cache=True, + bos_token_id=1, + eos_token_id=2, + hidden_dropout=0.0, + attention_dropout=0.0, + multi_query=True, + n_head_kv=None, + alibi=False, + bias=False, + parallel_attn=False, + new_decoder_architecture=False, + **kwargs, + ) -> None: + self.vocab_size = vocab_size + # Backward compatibility with n_embed kwarg + n_embed = kwargs.pop("n_embed", None) + self.hidden_size = hidden_size if n_embed is None else n_embed + self.n_layer = n_layer + self.n_head = n_head + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + self.use_cache = use_cache + self.hidden_dropout = hidden_dropout + self.attention_dropout = attention_dropout + + self.bos_token_id = bos_token_id + self.eos_token_id = eos_token_id + self.multi_query = multi_query + self.n_head_kv = 1 if n_head_kv is None else n_head_kv + self.alibi = alibi + self.bias = bias + self.parallel_attn = parallel_attn + self.new_decoder_architecture = new_decoder_architecture + + if self.hidden_size == 8192: + # Hack for falcon-40b + self.new_decoder_architecture = True + + super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs) + + @property + def head_dim(self): + return self.hidden_size // self.n_head + + @property + def rotary(self): + return not self.alibi diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/jais.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/jais.py new file mode 100644 index 00000000..eb36a37c --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/jais.py @@ -0,0 +1,242 @@ +# coding=utf-8 +# Copyright 2023 The OpenAI Team Authors and HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# Copyright 2023 Cerebras Systems. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""JAIS configuration""" + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger(__name__) + + +class JAISConfig(PretrainedConfig): + """ + This is the configuration class to store the configuration of a + [`JAISModel`]. It is used to instantiate a JAIS model according to the + specified arguments, defining the model architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used + to control the model outputs. Read the documentation from + [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 50257): + Vocabulary size of the JAIS model. Defines the number of different + tokens that can be represented by the + `inputs_ids` passed when calling [`JAISModel`]. + n_positions (`int`, *optional*, defaults to 1024): + The maximum sequence length that this model might ever be used + with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + n_embd (`int`, *optional*, defaults to 768): + Dimensionality of the embeddings and hidden states. + n_layer (`int`, *optional*, defaults to 12): + Number of hidden layers in the Transformer encoder. + n_head (`int`, *optional*, defaults to 12): + Number of attention heads for each attention layer in the + Transformer encoder. + n_inner (`int`, *optional*, defaults to None): + Dimensionality of the inner feed-forward layers. `None` will set + it to 4 times n_embd + activation_function (`str`, *optional*, defaults to `"gelu"`): + Activation function, to be selected in the list + `["relu", "silu", "gelu", "tanh", "gelu_new", "swiglu"]`. + resid_pdrop (`float`, *optional*, defaults to 0.1): + The dropout probability for all fully connected layers in + the embeddings, encoder, and pooler. + embd_pdrop (`float`, *optional*, defaults to 0.1): + The dropout ratio for the embeddings. + attn_pdrop (`float`, *optional*, defaults to 0.1): + The dropout ratio for the attention. + layer_norm_epsilon (`float`, *optional*, defaults to 1e-5): + The epsilon to use in the layer normalization layers. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for + initializing all weight matrices. + scale_attn_weights (`bool`, *optional*, defaults to `True`): + Scale attention weights by dividing by sqrt(hidden_size).. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values + attentions (not used by all models). + scale_attn_by_inverse_layer_idx (`bool`, *optional*, + defaults to `False`): + Whether to additionally scale attention weights by + `1 / layer_idx + 1`. + reorder_and_upcast_attn (`bool`, *optional*, defaults to `False`): + Whether to scale keys (K) prior to computing attention + (dot-product) + and upcast attention dot-product/softmax to float() when training + with mixed precision. + position_embedding_type (`str`, *optional*, defaults to `"learned"`): + Positional embedding can be either `"alibi"` or `"learned"`. + mup_width_scale (`float`, *optional*, defaults to 1.0): + muP parameter to scale learning rate and initializers. Calculated + as (`d_model,0 / d_model`), where + `d_model` is the model's width and `d_model,0` is the proxy + model's width. + mup_embeddings_scale (`float`, *optional*, defaults to 1.0): + muP parameter to scale token and position embeddings. + mup_output_alpha (`float`, *optional*, defaults to 1.0): + muP parameter to scale output logits + (`output_logits_scale = mup_output_alpha * mup_width_scale`). + mup_scale_qk_dot_by_d (`bool`, *optional*, defaults to `False`): + Scale attention weights by dividing by hidden_size instead of + sqrt(hidden_size). Need to set scale_attn_weights to `True` as + well. + alibi_scaling (`Dict`, *optional*): + Dictionary containing the scaling configuration for ALiBi + embeddings. Currently only supports linear + scaling strategy. Can specify either the scaling `factor` (must be + a float greater than 1) for fixed scaling + or `train_seq_len` for dynamic scaling on input samples with + sequence length > `train_seq_len`. The expected + formats are `{"type": strategy name, "factor": scaling factor}` or + `{"type": strategy name, + "train_seq_len": training sequence length}`. + architectures (`List`, *optional*, defaults to ['JAISLMHeadModel']): + architecture names for Jais. + + Example: + + ```python + >>> from transformers import JAISConfig, JAISModel + + >>> # Initializing a JAIS configuration + >>> configuration = JAISConfig() + + >>> # Initializing a model (with random weights) from the configuration + >>> model = JAISModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "jais" + keys_to_ignore_at_inference = ["past_key_values"] + attribute_map = { + "hidden_size": "n_embd", + "max_position_embeddings": "n_positions", + "num_attention_heads": "n_head", + "num_hidden_layers": "n_layer", + } + + def __init__( + self, + vocab_size=50257, + n_positions=1024, + n_embd=768, + n_layer=12, + n_head=12, + n_inner=None, + activation_function="gelu_new", + resid_pdrop=0.1, + embd_pdrop=0.1, + attn_pdrop=0.1, + layer_norm_epsilon=1e-5, + initializer_range=0.02, + scale_attn_weights=True, + use_cache=True, + bos_token_id=50256, + eos_token_id=50256, + scale_attn_by_inverse_layer_idx=False, + reorder_and_upcast_attn=False, + position_embedding_type="learned", + mup_width_scale=1.0, + mup_embeddings_scale=1.0, + mup_output_alpha=1.0, + mup_scale_qk_dot_by_d=False, + alibi_scaling=None, + architectures=None, + **kwargs, + ): + self.vocab_size = vocab_size + self.n_positions = n_positions + self.n_embd = n_embd + self.n_layer = n_layer + self.n_head = n_head + self.n_inner = n_inner + self.activation_function = activation_function + self.resid_pdrop = resid_pdrop + self.embd_pdrop = embd_pdrop + self.attn_pdrop = attn_pdrop + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + self.scale_attn_weights = scale_attn_weights + self.use_cache = use_cache + self.scale_attn_by_inverse_layer_idx = scale_attn_by_inverse_layer_idx + self.reorder_and_upcast_attn = reorder_and_upcast_attn + + self.bos_token_id = bos_token_id + self.eos_token_id = eos_token_id + + self.position_embedding_type = position_embedding_type + self.mup_width_scale = mup_width_scale + self.mup_embeddings_scale = mup_embeddings_scale + self.mup_output_alpha = mup_output_alpha + self.mup_scale_qk_dot_by_d = mup_scale_qk_dot_by_d + + self.alibi_scaling = alibi_scaling + self._alibi_scaling_validation() + if architectures is None: + architectures = ["JAISLMHeadModel"] + + super().__init__( + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + architectures=architectures, + **kwargs, + ) + + def _alibi_scaling_validation(self): + """ + Validate the `alibi_scaling` configuration. + """ + if self.alibi_scaling is None: + return + + if not isinstance(self.alibi_scaling, dict) or len(self.alibi_scaling) != 2: + raise ValueError( + "`alibi_scaling` must be a dictionary with two fields," + "`type` and `factor` or `type` and `train_seq_len`, " + f"got {self.alibi_scaling}" + ) + alibi_scaling_type = self.alibi_scaling.get("type", None) + alibi_scaling_factor = self.alibi_scaling.get("factor", None) + alibi_dynamic_scaling = self.alibi_scaling.get("train_seq_len", None) + if alibi_scaling_type is None or alibi_scaling_type != "linear": + raise ValueError( + f"`alibi_scaling`'s type field must be 'linear'," + f"got {alibi_scaling_type}" + ) + if ( + alibi_scaling_factor is not None + and not isinstance(alibi_scaling_factor, float) + or (alibi_scaling_factor is not None and alibi_scaling_factor <= 1.0) + ): + raise ValueError( + f"`alibi_scaling`'s factor field must be a float > 1.0," + f"got {alibi_scaling_factor}" + ) + if ( + alibi_dynamic_scaling is not None + and not isinstance(alibi_dynamic_scaling, int) + or (alibi_dynamic_scaling is not None and alibi_dynamic_scaling <= 1) + ): + raise ValueError( + f"`alibi_scaling`'s `train_seq_len` field must be an" + f"integer > 1, got {alibi_dynamic_scaling}" + ) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/mpt.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/mpt.py new file mode 100644 index 00000000..969af314 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/mpt.py @@ -0,0 +1,202 @@ +# coding=utf-8 +# Copied from +# https://huggingface.co/mosaicml/mpt-7b/blob/main/configuration_mpt.py +"""A HuggingFace-style model configuration.""" +import warnings +from typing import Any, Dict, Optional, Union + +from transformers import PretrainedConfig + +attn_config_defaults: Dict = { + "attn_type": "multihead_attention", + "attn_pdrop": 0.0, + "attn_impl": "triton", + "qk_ln": False, + "clip_qkv": None, + "softmax_scale": None, + "prefix_lm": False, + "attn_uses_sequence_id": False, + "alibi": False, + "alibi_bias_max": 8, +} +ffn_config_defaults: Dict = {"ffn_type": "mptmlp"} +init_config_defaults: Dict = { + "name": "kaiming_normal_", + "fan_mode": "fan_in", + "init_nonlinearity": "relu", + "init_div_is_residual": True, + "emb_init_std": None, + "emb_init_uniform_lim": None, + "init_std": None, + "init_gain": 0.0, +} + + +class MPTConfig(PretrainedConfig): + model_type = "mpt" + attribute_map = { + "num_attention_heads": "n_heads", + "hidden_size": "d_model", + "num_hidden_layers": "n_layers", + } + + # pylint: disable=dangerous-default-value + def __init__( + self, + d_model: int = 2048, + n_heads: int = 16, + n_layers: int = 24, + expansion_ratio: int = 4, + max_seq_len: int = 2048, + vocab_size: int = 50368, + resid_pdrop: float = 0.0, + emb_pdrop: float = 0.0, + learned_pos_emb: bool = True, + attn_config: Dict = attn_config_defaults, + ffn_config: Dict = ffn_config_defaults, + init_device: str = "cpu", + logit_scale: Optional[Union[float, str]] = None, + no_bias: bool = False, + embedding_fraction: float = 1.0, + norm_type: str = "low_precision_layernorm", + use_cache: bool = False, + init_config: Dict = init_config_defaults, + fc_type: str = "torch", + verbose: Optional[int] = None, + **kwargs: Any, + ): + self.d_model = d_model + self.n_heads = n_heads + self.n_layers = n_layers + self.expansion_ratio = expansion_ratio + self.max_seq_len = max_seq_len + self.vocab_size = vocab_size + self.resid_pdrop = resid_pdrop + self.emb_pdrop = emb_pdrop + self.learned_pos_emb = learned_pos_emb + self.attn_config = attn_config + self.ffn_config = ffn_config + self.init_device = init_device + self.logit_scale = logit_scale + self.no_bias = no_bias + self.embedding_fraction = embedding_fraction + self.norm_type = norm_type + self.use_cache = use_cache + self.init_config = init_config + self.fc_type = fc_type + if verbose is not None: + warnings.warn( + DeprecationWarning( + "verbose argument for MPTConfig is now ignored and " + "will be removed. Use python_log_level instead." + ), + stacklevel=2, + ) + if "name" in kwargs: + del kwargs["name"] + if "loss_fn" in kwargs: + del kwargs["loss_fn"] + if self.attn_config.get("alibi", False): + self.learned_pos_emb = False + warnings.warn( + f"alibi is turned on, setting `learned_pos_emb` " + f"to {self.learned_pos_emb}`", + stacklevel=2, + ) + super().__init__(**kwargs) + self._validate_config() + + def _set_config_defaults( + self, config: Dict[str, Any], config_defaults: Dict[str, Any] + ) -> Dict[str, Any]: + for k, v in config_defaults.items(): + if k not in config: + config[k] = v + return config + + def _validate_config(self) -> None: + self.attn_config = self._set_config_defaults( + self.attn_config, attn_config_defaults + ) + self.ffn_config = self._set_config_defaults( + self.ffn_config, ffn_config_defaults + ) + self.init_config = self._set_config_defaults( + self.init_config, init_config_defaults + ) + if self.d_model % self.n_heads != 0: + raise ValueError("d_model must be divisible by n_heads") + if any( + ( + prob < 0 or prob > 1 + for prob in [ + self.attn_config["attn_pdrop"], + self.resid_pdrop, + self.emb_pdrop, + ] + ) + ): + raise ValueError( + "self.attn_config['attn_pdrop'], resid_pdrop, emb_pdrop are " + "probabilities and must be between 0 and 1" + ) + if self.attn_config["attn_impl"] not in ["torch", "flash", "triton"]: + raise ValueError(f"Unknown attn_impl={self.attn_config['attn_impl']}") + if self.attn_config["prefix_lm"] and self.attn_config["attn_impl"] not in [ + "torch", + "triton", + ]: + raise NotImplementedError( + "prefix_lm only implemented with torch and triton attention." + ) + if self.attn_config["alibi"] and self.attn_config["attn_impl"] not in [ + "torch", + "triton", + ]: + raise NotImplementedError( + "alibi only implemented with torch and triton attention." + ) + if self.attn_config["attn_uses_sequence_id"] and self.attn_config[ + "attn_impl" + ] not in ["torch", "triton"]: + raise NotImplementedError( + "attn_uses_sequence_id only implemented with torch " + "and triton attention." + ) + if self.embedding_fraction > 1 or self.embedding_fraction <= 0: + raise ValueError( + "model.embedding_fraction must be between 0 (exclusive) " + "and 1 (inclusive)!" + ) + if isinstance(self.logit_scale, str) and self.logit_scale != "inv_sqrt_d_model": + raise ValueError( + f"self.logit_scale={self.logit_scale!r} is not recognized as " + "an option; use numeric value or 'inv_sqrt_d_model'." + ) + if self.init_config.get("name", None) is None: + raise ValueError( + f"self.init_config={self.init_config!r} 'name' needs to be set." + ) + if not self.learned_pos_emb and (not self.attn_config["alibi"]): + warnings.warn( + "Positional information not being provided to the model.", stacklevel=2 + ) + if self.fc_type == "te" or self.ffn_config["ffn_type"] == "te_ln_mlp": + try: + # pylint: disable=import-outside-toplevel + import transformer_engine.pytorch as te + + del te + except Exception as exc: + raise ImportError( + "TransformerEngine import fail. `fc_type: te` requires " + "TransformerEngine be installed. " + "The required version of transformer_engine also requires " + "FlashAttention v1.0.6 is installed:\n" + "pip install flash-attn==1.0.6 --no-build-isolation \n" + "pip install git+https://github.com/NVIDIA/TransformerEngine.git@144e4888b2cdd60bd52e706d5b7a79cb9c1a7156" + ) from exc + if self.ffn_config["ffn_type"] == "mptmlp": + self.ffn_config["fc_type"] = self.fc_type + elif self.ffn_config["ffn_type"] == "te_ln_mlp": + self.ffn_config["bias"] = not self.no_bias diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/detokenizer.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/detokenizer.py new file mode 100644 index 00000000..b7ca62e0 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/detokenizer.py @@ -0,0 +1,325 @@ +from typing import Dict, List, Optional, Tuple, Union + +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast +from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup +from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import ( + BaseTokenizerGroup, +) + +# Used eg. for marking rejected tokens in spec decoding. +INVALID_TOKEN_ID = -1 + + +class Detokenizer: + """Provides methods to decode the output of a model into text.""" + + def __init__(self, tokenizer_group: BaseTokenizerGroup): + self.tokenizer_group = tokenizer_group + + def get_tokenizer_for_seq(self, sequence: Sequence) -> "PreTrainedTokenizer": + """Returns the HF tokenizer to use for a given sequence.""" + return self.tokenizer_group.get_lora_tokenizer(sequence.lora_request) + + def decode_prompt_logprobs_inplace( + self, + seq_group: SequenceGroup, + prompt_logprobs: List[Optional[Dict[int, Logprob]]], + ) -> None: + """Decodes the logprobs for the prompt of a sequence group. + + Args: + seq_group: The sequence group to decode. + prompt_logprobs: The logprobs to decode. + + Returns: + The prompt logprobs with the decoded tokens. + """ + prms = seq_group.sampling_params + # We can pick any sequence for the prompt. + seq = next(iter(seq_group.seqs_dict.values())) + # Only prompt, without the generated token. + all_token_ids = seq.get_token_ids() + prompt_token_ids = all_token_ids[:-1] + tokenizer = self.get_tokenizer_for_seq(seq) + prefix_offset = 0 + read_offset = 0 + next_iter_prefix_offset = 0 + next_iter_read_offset = 0 + next_iter_tokens = [] + prev_tokens = None + + for token_position, prompt_logprobs_for_token in enumerate(prompt_logprobs): + if not prompt_logprobs_for_token: + continue + for token_id, sample_logprob in prompt_logprobs_for_token.items(): + if ( + sample_logprob.decoded_token is None + and token_id != INVALID_TOKEN_ID + ): + prompt_token_ids_with_token = prompt_token_ids[:token_position] + [ + token_id + ] + ( + new_tokens, + new_text, + new_prefix_offset, + new_read_offset, + ) = detokenize_incrementally( + tokenizer=tokenizer, + all_input_ids=prompt_token_ids_with_token, + prev_tokens=prev_tokens, + prefix_offset=prefix_offset, + read_offset=read_offset, + skip_special_tokens=prms.skip_special_tokens, + spaces_between_special_tokens=prms.spaces_between_special_tokens, + ) + + sample_logprob.decoded_token = new_text + + # Use the offsets & prev tokens corresponding to + # real tokens to ensure detokenization is consistent + # actual with prompt. + if token_id == all_token_ids[token_position]: + next_iter_prefix_offset = new_prefix_offset + next_iter_read_offset = new_read_offset + next_iter_tokens = new_tokens + + # Advance to the next token position. + prefix_offset = next_iter_prefix_offset + read_offset = next_iter_read_offset + if prev_tokens is None: + prev_tokens = next_iter_tokens + else: + prev_tokens.extend(next_iter_tokens) + + def decode_sequence_inplace(self, seq: Sequence, prms: SamplingParams) -> int: + """Decodes the new token for a sequence. In-place operation. + + Args: + seq: The sequence to decode. + prms: The sampling parameters used to generate the sequence. + + Returns: + The number of characters added to the output text. + """ + all_input_ids = seq.get_token_ids() + token_id_generated_this_iteration = all_input_ids[-1] + tokenizer = self.get_tokenizer_for_seq(seq) + + # Convert prompt token IDs to tokens if necessary. + # Do it here so that we don't have to repeat this + # computation for each logprob. + if seq.tokens is None: + ( + seq.tokens, + seq.prefix_offset, + seq.read_offset, + ) = convert_prompt_ids_to_tokens( + tokenizer=tokenizer, + prompt_ids=all_input_ids[:-1], + skip_special_tokens=prms.skip_special_tokens, + ) + + ( + new_tokens, + new_decoded_token_text, + prefix_offset, + read_offset, + ) = detokenize_incrementally( + tokenizer=tokenizer, + all_input_ids=all_input_ids, + prev_tokens=seq.tokens, + prefix_offset=seq.prefix_offset, + read_offset=seq.read_offset, + skip_special_tokens=prms.skip_special_tokens, + spaces_between_special_tokens=prms.spaces_between_special_tokens, + ) + + # Decode logprobs + logprobs = seq.output_logprobs[-1] + if logprobs: + previous_tokens = all_input_ids[:-1] + for token_id, sample_logprob in logprobs.items(): + # If the token was generated this iteration, + # use the provided text. + if token_id == token_id_generated_this_iteration: + sample_logprob.decoded_token = new_decoded_token_text + continue + + if ( + sample_logprob.decoded_token is None + and token_id != INVALID_TOKEN_ID + ): + all_input_ids_with_logprob = previous_tokens + [token_id] + (_, new_text, _, _) = detokenize_incrementally( + tokenizer=tokenizer, + all_input_ids=all_input_ids_with_logprob, + prev_tokens=seq.tokens, + prefix_offset=seq.prefix_offset, + read_offset=seq.read_offset, + skip_special_tokens=prms.skip_special_tokens, + spaces_between_special_tokens=prms.spaces_between_special_tokens, + ) + sample_logprob.decoded_token = new_text + + seq.tokens.extend(new_tokens) + seq.prefix_offset = prefix_offset + seq.read_offset = read_offset + seq.output_text += new_decoded_token_text + + return len(new_decoded_token_text) + + +def _convert_tokens_to_string_with_added_encoders( + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], + output_tokens: List[str], + skip_special_tokens: bool, + spaces_between_special_tokens: bool, +) -> str: + # Adapted from + # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921 + # NOTE(woosuk): The following code is slow because it runs a for loop over + # the output_tokens. In Python, running a for loop over a list can be slow + # even when the loop body is very simple. + sub_texts: List[str] = [] + current_sub_text: List[str] = [] + all_special_tokens = set(tokenizer.all_special_tokens) + for token in output_tokens: + if skip_special_tokens and token in all_special_tokens: + continue + if token in tokenizer.get_added_vocab(): + if current_sub_text: + sub_text = tokenizer.convert_tokens_to_string(current_sub_text) + sub_texts.append(sub_text) + current_sub_text = [] + sub_texts.append(token) + else: + current_sub_text.append(token) + if current_sub_text: + sub_text = tokenizer.convert_tokens_to_string(current_sub_text) + sub_texts.append(sub_text) + if spaces_between_special_tokens: + return " ".join(sub_texts) + else: + return "".join(sub_texts) + + +# 5 is an arbitrary value that should work for all +# tokenizers (bigger = more conservative). +INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5 + + +def convert_prompt_ids_to_tokens( + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], + prompt_ids: List[int], + skip_special_tokens: bool = False, +) -> Tuple[List[str], int, int]: + """Converts the prompt ids to tokens and returns the tokens and offsets + for incremental detokenization. + + Note that not all tokens are converted to strings. Only the tokens that + are necessary for incremental detokenization are converted to strings. + """ + # We do not need to convert the whole prompt to tokens. + # Offset a little more in case we have special tokens. + new_tokens = tokenizer.convert_ids_to_tokens( + prompt_ids[-INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET - 2 :], + skip_special_tokens=skip_special_tokens, + ) + read_offset = len(new_tokens) + prefix_offset = max(read_offset - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0) + return new_tokens, prefix_offset, read_offset + + +# Based on +# https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15 +# under Apache 2.0 license +def detokenize_incrementally( + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], + all_input_ids: List[int], + prev_tokens: Optional[List[str]], + prefix_offset: int, + read_offset: int, + skip_special_tokens: bool = False, + spaces_between_special_tokens: bool = True, +) -> Tuple[List[str], str, int, int]: + """Detokenizes the input ids incrementally and returns the new tokens + and the new text. + + If `prev_tokens` is None, this function will convert the input ids to + tokens and return the tokens and the new text. Otherwise, it will return the + new tokens and the new text. + + This function will also return the new prefix offset and the new read + offset to be used in the next iteration. + + The offsets are necessary to defeat cleanup algorithms in the decode which + decide to add a space or not depending on the surrounding ids. + + Args: + tokenizer: The tokenizer to use. + all_input_ids: The input ids. The last id is the new token id. + prev_tokens: The previous tokens. If None, this function will convert + the input ids to tokens and return the tokens and the new text. + prefix_offset: The prefix offset. + read_offset: The read offset. + skip_special_tokens: Whether to skip special tokens. + spaces_between_special_tokens: Whether to add spaces between special + tokens. + """ + new_token_id = all_input_ids[-1] + # This is the first iteration for this sequence + is_first_iter = prev_tokens is None + if is_first_iter: + (prev_tokens, prefix_offset, read_offset) = convert_prompt_ids_to_tokens( + tokenizer, all_input_ids[:-1], skip_special_tokens=skip_special_tokens + ) + assert prev_tokens is not None + + # If the new token id is out of bounds, return an empty string. + if new_token_id >= len(tokenizer): + new_tokens = [""] + else: + # Put new_token_id in a list so skip_special_tokens is respected + new_tokens = tokenizer.convert_ids_to_tokens( + [new_token_id], skip_special_tokens=skip_special_tokens + ) + if isinstance(new_tokens, str): + new_tokens = [new_tokens] + output_tokens = prev_tokens + new_tokens + + # If this is the first iteration, return all tokens. + if is_first_iter: + new_tokens = output_tokens + + # The prefix text is necessary only to defeat cleanup algorithms in + # the decode which decide to add a space or not depending on the + # surrounding ids. + if tokenizer.is_fast or not tokenizer.get_added_vocab(): + prefix_text = tokenizer.convert_tokens_to_string( + output_tokens[prefix_offset:read_offset] + ) + new_text = tokenizer.convert_tokens_to_string(output_tokens[prefix_offset:]) + else: + prefix_text = _convert_tokens_to_string_with_added_encoders( + tokenizer, + output_tokens[prefix_offset:read_offset], + skip_special_tokens=skip_special_tokens, + spaces_between_special_tokens=spaces_between_special_tokens, + ) + new_text = _convert_tokens_to_string_with_added_encoders( + tokenizer, + output_tokens[prefix_offset:], + skip_special_tokens=skip_special_tokens, + spaces_between_special_tokens=spaces_between_special_tokens, + ) + + if len(new_text) <= len(prefix_text) or new_text.endswith("�"): + # utf-8 char at the end means it's a potential unfinished byte sequence + # from byte fallback tokenization. + # If it's in the middle, it's probably a real invalid id generated + # by the model + return new_tokens, "", prefix_offset, read_offset + + new_text = new_text[len(prefix_text) :] + return new_tokens, new_text, read_offset, len(output_tokens) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer.py new file mode 100644 index 00000000..0011172c --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer.py @@ -0,0 +1,150 @@ +import os +from typing import Optional, Union + +from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast + +# from vllm.config import VLLM_USE_MODELSCOPE +# from vllm.logger import init_logger +# from vllm.lora.request import LoRARequest +# from vllm.transformers_utils.tokenizers import BaichuanTokenizer +# from vllm.utils import make_async + +# logger = init_logger(__name__) + +VLLM_USE_MODELSCOPE = False + + +def get_cached_tokenizer( + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast] +) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: + """Get tokenizer with cached properties. + + This will patch the tokenizer object in place. + + By default, transformers will recompute multiple tokenizer properties + each time they are called, leading to a significant slowdown. This + function caches these properties for faster access.""" + + tokenizer_all_special_ids = set(tokenizer.all_special_ids) + tokenizer_all_special_tokens_extended = tokenizer.all_special_tokens_extended + tokenizer_all_special_tokens = set(tokenizer.all_special_tokens) + tokenizer_len = len(tokenizer) + + class CachedTokenizer(tokenizer.__class__): # type: ignore + @property + def all_special_ids(self): + return tokenizer_all_special_ids + + @property + def all_special_tokens(self): + return tokenizer_all_special_tokens + + @property + def all_special_tokens_extended(self): + return tokenizer_all_special_tokens_extended + + def __len__(self): + return tokenizer_len + + CachedTokenizer.__name__ = f"Cached{tokenizer.__class__.__name__}" + + tokenizer.__class__ = CachedTokenizer + return tokenizer + + +def get_tokenizer( + tokenizer_name: str, + *args, + tokenizer_mode: str = "auto", + trust_remote_code: bool = False, + tokenizer_revision: Optional[str] = None, + download_dir: Optional[str] = None, + **kwargs, +) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: + """Gets a tokenizer for the given model name via Huggingface/modelscope.""" + if VLLM_USE_MODELSCOPE: + # download model from ModelScope hub, + # lazy import so that modelscope is not required for normal use. + # pylint: disable=C. + from modelscope.hub.snapshot_download import snapshot_download + + # Only set the tokenizer here, model will be downloaded on the workers. + if not os.path.exists(tokenizer_name): + tokenizer_path = snapshot_download( + model_id=tokenizer_name, + cache_dir=download_dir, + revision=tokenizer_revision, + # Ignore weights - we only need the tokenizer. + ignore_file_pattern=["*.pt", "*.safetensors", "*.bin"], + ) + tokenizer_name = tokenizer_path + + if tokenizer_mode == "slow": + if kwargs.get("use_fast", False): + raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.") + kwargs["use_fast"] = False + + try: + tokenizer = AutoTokenizer.from_pretrained( + tokenizer_name, + *args, + trust_remote_code=trust_remote_code, + tokenizer_revision=tokenizer_revision, + **kwargs, + ) + except ValueError as e: + # If the error pertains to the tokenizer class not existing or not + # currently being imported, suggest using the --trust-remote-code flag. + if not trust_remote_code and ( + "does not exist or is not currently imported." in str(e) + or "requires you to execute the tokenizer file" in str(e) + ): + err_msg = ( + "Failed to load the tokenizer. If the tokenizer is a custom " + "tokenizer not yet available in the HuggingFace transformers " + "library, consider setting `trust_remote_code=True` in LLM " + "or using the `--trust-remote-code` flag in the CLI." + ) + raise RuntimeError(err_msg) from e + else: + raise e + except AttributeError as e: + # if "BaichuanTokenizer" in str(e): + # # This is for the error "'BaichuanTokenizer' object has no + # # attribute 'sp_model'". + # tokenizer = BaichuanTokenizer.from_pretrained( + # tokenizer_name, + # *args, + # trust_remote_code=trust_remote_code, + # tokenizer_revision=tokenizer_revision, + # **kwargs) + # else: + raise e + + if not isinstance(tokenizer, PreTrainedTokenizerFast): + print( + "Using a slow tokenizer. This might cause a significant " + "slowdown. Consider using a fast tokenizer instead." + ) + return get_cached_tokenizer(tokenizer) + + +# def get_lora_tokenizer(lora_request: LoRARequest, *args, +# **kwargs) -> Optional[PreTrainedTokenizer]: +# if lora_request is None: +# return None +# try: +# tokenizer = get_tokenizer(lora_request.lora_local_path, *args, +# **kwargs) +# except OSError as e: +# # No tokenizer was found in the LoRA folder, +# # use base model tokenizer +# logger.warning( +# f"No tokenizer found in {lora_request.lora_local_path}, " +# "using base model tokenizer instead. " +# f"(Exception: {str(e)})") +# tokenizer = None +# return tokenizer + + +# get_lora_tokenizer_async = make_async(get_lora_tokenizer) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/__init__.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/__init__.py new file mode 100644 index 00000000..d952b789 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/__init__.py @@ -0,0 +1,34 @@ +from typing import Optional + +from vllm.config import TokenizerPoolConfig +from vllm.engine.ray_utils import ray +from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import ( + BaseTokenizerGroup, +) +from vllm.transformers_utils.tokenizer_group.tokenizer_group import TokenizerGroup + +if ray: + from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import ( + RayTokenizerGroupPool, + ) +else: + RayTokenizerGroupPool = None # type: ignore + + +def get_tokenizer_group( + tokenizer_pool_config: Optional[TokenizerPoolConfig], **init_kwargs +) -> BaseTokenizerGroup: + if tokenizer_pool_config is None: + return TokenizerGroup(**init_kwargs) + if tokenizer_pool_config.pool_type == "ray": + if RayTokenizerGroupPool is None: + raise ImportError( + "RayTokenizerGroupPool is not available. Please install " + "the ray package to use the Ray tokenizer group pool." + ) + return RayTokenizerGroupPool.from_config(tokenizer_pool_config, **init_kwargs) + else: + raise ValueError(f"Unknown pool type: {tokenizer_pool_config.pool_type}") + + +__all__ = ["get_tokenizer_group", "BaseTokenizerGroup"] diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/base_tokenizer_group.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/base_tokenizer_group.py new file mode 100644 index 00000000..ecffa071 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/base_tokenizer_group.py @@ -0,0 +1,55 @@ +from abc import ABC, abstractmethod +from typing import List, Optional + +from transformers import PreTrainedTokenizer +from vllm.lora.request import LoRARequest + + +class BaseTokenizerGroup(ABC): + """A group of tokenizers that can be used for LoRA adapters.""" + + @abstractmethod + def ping(self) -> bool: + """Check if the tokenizer group is alive.""" + pass + + @abstractmethod + def get_max_input_len( + self, lora_request: Optional[LoRARequest] = None + ) -> Optional[int]: + """Get the maximum input length for the LoRA request.""" + pass + + @abstractmethod + def encode( + self, + prompt: str, + request_id: Optional[str] = None, + lora_request: Optional[LoRARequest] = None, + ) -> List[int]: + """Encode a prompt using the tokenizer group.""" + pass + + @abstractmethod + async def encode_async( + self, + prompt: str, + request_id: Optional[str] = None, + lora_request: Optional[LoRARequest] = None, + ) -> List[int]: + """Encode a prompt using the tokenizer group.""" + pass + + @abstractmethod + def get_lora_tokenizer( + self, lora_request: Optional[LoRARequest] = None + ) -> "PreTrainedTokenizer": + """Get a tokenizer for a LoRA request.""" + pass + + @abstractmethod + async def get_lora_tokenizer_async( + self, lora_request: Optional[LoRARequest] = None + ) -> "PreTrainedTokenizer": + """Get a tokenizer for a LoRA request.""" + pass diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/ray_tokenizer_group.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/ray_tokenizer_group.py new file mode 100644 index 00000000..a8182ee7 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/ray_tokenizer_group.py @@ -0,0 +1,181 @@ +import asyncio +import os +from typing import List, Optional + +from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy +from transformers import PreTrainedTokenizer +from vllm.config import TokenizerPoolConfig +from vllm.engine.ray_utils import ray +from vllm.lora.request import LoRARequest +from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import ( + BaseTokenizerGroup, +) +from vllm.transformers_utils.tokenizer_group.tokenizer_group import TokenizerGroup + + +class RayTokenizerGroupPool(BaseTokenizerGroup): + """A Ray-based pool of TokenizerGroups for async tokenization.""" + + # Class to use for workers making up the pool. + _worker_cls = TokenizerGroup + + @classmethod + def from_config( + cls, tokenizer_pool_config: TokenizerPoolConfig, **init_kwargs + ) -> "RayTokenizerGroupPool": + ray_actor_options = tokenizer_pool_config.extra_config or {"num_cpus": 0} + ray_actor_options.setdefault( + "scheduling_strategy", + NodeAffinitySchedulingStrategy( + node_id=ray.get_runtime_context().get_node_id(), soft=True + ), + ) + + # Carry over the env vars to the actors. + # This is necessary for API keys and such. + ray_actor_options.setdefault("runtime_env", {}) + _carry_over_env_vars_to_runtime_env(ray_actor_options["runtime_env"]) + + init_kwargs["num_actors"] = tokenizer_pool_config.pool_size + init_kwargs["ray_actor_options"] = ray_actor_options + + return cls(**init_kwargs) + + def __init__( + self, + tokenizer_id: str, + enable_lora: bool, + max_num_seqs: int, + max_input_length: Optional[int], + num_actors: int, + ray_actor_options: dict, + **tokenizer_config + ): + # Store a local copy of the TokenizerGroup for quick access + # to underlying HF tokenizers. + self._local_tokenizer_group = self._worker_cls( + tokenizer_id=tokenizer_id, + enable_lora=enable_lora, + max_num_seqs=max_num_seqs, + max_input_length=max_input_length, + **tokenizer_config, + ) + + ray_tokenizer_group_cls = ray.remote(self._worker_cls).options( + **ray_actor_options + ) + self.tokenizer_actors = [ + ray_tokenizer_group_cls.remote( + tokenizer_id, + enable_lora, + max_num_seqs, + max_input_length, + **tokenizer_config, + ) + for _ in range(num_actors) + ] + self._idle_actors: Optional[asyncio.Queue] = None + + @property + def pool_size(self) -> int: + return len(self.tokenizer_actors) + + def ping(self): + return ray.get([actor.ping.remote() for actor in self.tokenizer_actors]) + + def _ensure_queue_initialized(self): + if self._idle_actors is None: + self._idle_actors = asyncio.Queue() + for actor in self.tokenizer_actors: + self._idle_actors.put_nowait(actor) + + def encode( + self, + prompt: str, + request_id: Optional[str] = None, + lora_request: Optional[LoRARequest] = None, + ) -> List[int]: + """Encode a prompt using the tokenizer group. + + We pick an idle actor and use it to encode the prompt. + The actor is then put back in the queue for future use. + This is blocking. + """ + self._ensure_queue_initialized() + assert self._idle_actors is not None + + if self._idle_actors.empty(): + raise RuntimeError("No idle actors available.") + actor = self._idle_actors.get_nowait() + try: + ret = ray.get( + actor.encode.remote( + request_id=request_id, prompt=prompt, lora_request=lora_request + ) + ) + finally: + # Put the actor back in the queue. + # This is done in a finally block to ensure that the actor is + # always put back in the queue, even if an exception/cancellation + # is raised. + self._idle_actors.put_nowait(actor) + return ret + + async def encode_async( + self, + prompt: str, + request_id: Optional[str] = None, + lora_request: Optional[LoRARequest] = None, + ) -> List[int]: + """Encode a prompt using the tokenizer group. + + We pick an idle actor and use it to encode the prompt. + If there are no idle actors, we wait until one becomes + available. + The actor is then put back in the queue for future use. + This is non-blocking. + """ + self._ensure_queue_initialized() + assert self._idle_actors is not None + + actor = await self._idle_actors.get() + try: + ret = await actor.encode.remote( + request_id=request_id, prompt=prompt, lora_request=lora_request + ) + finally: + # Put the actor back in the queue. + # This is done in a finally block to ensure that the actor is + # always put back in the queue, even if an exception/cancellation + # is raised. + self._idle_actors.put_nowait(actor) + return ret + + def get_max_input_len( + self, lora_request: Optional[LoRARequest] = None + ) -> Optional[int]: + """Get the maximum input length for the LoRA request.""" + return self._local_tokenizer_group.get_max_input_len(lora_request) + + def get_lora_tokenizer( + self, lora_request: Optional[LoRARequest] = None + ) -> "PreTrainedTokenizer": + return self._local_tokenizer_group.get_lora_tokenizer(lora_request) + + async def get_lora_tokenizer_async( + self, lora_request: Optional[LoRARequest] = None + ) -> "PreTrainedTokenizer": + return await self._local_tokenizer_group.get_lora_tokenizer_async(lora_request) + + +def _carry_over_env_vars_to_runtime_env(runtime_env: dict) -> None: + """Copy over all current process environment variables to the runtime_env. + + The variables in runtime_env will take precedence over the current process + environment variables. + + runtime_env will be modified in place.""" + env_vars = os.environ.copy() + runtime_env.setdefault("env_vars", {}) + env_vars.update(runtime_env["env_vars"]) + runtime_env["env_vars"] = env_vars diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/tokenizer_group.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/tokenizer_group.py new file mode 100644 index 00000000..cc9736fa --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/tokenizer_group.py @@ -0,0 +1,94 @@ +from typing import List, Optional + +from transformers import PreTrainedTokenizer +from vllm.lora.request import LoRARequest +from vllm.transformers_utils.tokenizer import ( + get_lora_tokenizer, + get_lora_tokenizer_async, + get_tokenizer, +) +from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import ( + BaseTokenizerGroup, +) +from vllm.utils import LRUCache + + +class TokenizerGroup(BaseTokenizerGroup): + """A group of tokenizers that can be used for LoRA adapters.""" + + def __init__( + self, + tokenizer_id: str, + enable_lora: bool, + max_num_seqs: int, + max_input_length: Optional[int], + **tokenizer_config + ): + self.tokenizer_id = tokenizer_id + self.tokenizer_config = tokenizer_config + self.enable_lora = enable_lora + self.max_input_length = max_input_length + self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config) + self.lora_tokenizers = ( + LRUCache[PreTrainedTokenizer](capacity=max_num_seqs) + if enable_lora + else None + ) + + def ping(self) -> bool: + """Check if the tokenizer group is alive.""" + return True + + def get_max_input_len( + self, lora_request: Optional[LoRARequest] = None + ) -> Optional[int]: + """Get the maximum input length for the LoRA request.""" + return self.max_input_length + + def encode( + self, + prompt: str, + request_id: Optional[str] = None, + lora_request: Optional[LoRARequest] = None, + ) -> List[int]: + tokenizer = self.get_lora_tokenizer(lora_request) + return tokenizer.encode(prompt) + + async def encode_async( + self, + prompt: str, + request_id: Optional[str] = None, + lora_request: Optional[LoRARequest] = None, + ) -> List[int]: + tokenizer = await self.get_lora_tokenizer_async(lora_request) + return tokenizer.encode(prompt) + + def get_lora_tokenizer( + self, lora_request: Optional[LoRARequest] = None + ) -> "PreTrainedTokenizer": + if not lora_request or not self.enable_lora: + return self.tokenizer + if lora_request.lora_int_id not in self.lora_tokenizers: + tokenizer = ( + get_lora_tokenizer(lora_request, **self.tokenizer_config) + or self.tokenizer + ) + self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer) + return tokenizer + else: + return self.lora_tokenizers.get(lora_request.lora_int_id) + + async def get_lora_tokenizer_async( + self, lora_request: Optional[LoRARequest] = None + ) -> "PreTrainedTokenizer": + if not lora_request or not self.enable_lora: + return self.tokenizer + if lora_request.lora_int_id not in self.lora_tokenizers: + tokenizer = ( + await get_lora_tokenizer_async(lora_request, **self.tokenizer_config) + or self.tokenizer + ) + self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer) + return tokenizer + else: + return self.lora_tokenizers.get(lora_request.lora_int_id) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/__init__.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/__init__.py new file mode 100644 index 00000000..e6b59722 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/__init__.py @@ -0,0 +1,5 @@ +from vllm.transformers_utils.tokenizers.baichuan import BaichuanTokenizer + +__all__ = [ + "BaichuanTokenizer", +] diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/baichuan.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/baichuan.py new file mode 100644 index 00000000..ba4baaaf --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/baichuan.py @@ -0,0 +1,270 @@ +# Adapted from +# https://huggingface.co/baichuan-inc/Baichuan2-13B-Chat/blob/8f6e343d545c503b91429582231d1d354dac2740/tokenization_baichuan.py +# This includes a fix suggested in +# https://github.com/vllm-project/vllm/issues/1403#issuecomment-1767503058 +# Copyright (c) 2023, Baichuan Intelligent Technology. All rights reserved. + +import os +from shutil import copyfile +from typing import Any, Dict, List, Optional, Tuple + +import sentencepiece as spm +from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer +from transformers.utils import logging + +logger = logging.get_logger(__name__) + +VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"} + +PRETRAINED_VOCAB_FILES_MAP = { # type: ignore + "vocab_file": {}, + "tokenizer_file": {}, +} +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {} # type: ignore + + +class BaichuanTokenizer(PreTrainedTokenizer): + """ + Construct a Baichuan tokenizer. Based on byte-level Byte-Pair-Encoding. + + Args: + vocab_file (`str`): + Path to the vocabulary file. + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + model_input_names = ["input_ids", "attention_mask"] + + def __init__( + self, + vocab_file, + unk_token="", + bos_token="", + eos_token="", + pad_token=None, + sp_model_kwargs: Optional[Dict[str, Any]] = None, + add_bos_token=True, + add_eos_token=False, + clean_up_tokenization_spaces=False, + **kwargs, + ): + self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs + bos_token = ( + AddedToken(bos_token, lstrip=False, rstrip=False) + if isinstance(bos_token, str) + else bos_token + ) + eos_token = ( + AddedToken(eos_token, lstrip=False, rstrip=False) + if isinstance(eos_token, str) + else eos_token + ) + unk_token = ( + AddedToken(unk_token, lstrip=False, rstrip=False) + if isinstance(unk_token, str) + else unk_token + ) + pad_token = ( + AddedToken(pad_token, lstrip=False, rstrip=False) + if isinstance(pad_token, str) + else pad_token + ) + self.vocab_file = vocab_file + self.add_bos_token = add_bos_token + self.add_eos_token = add_eos_token + self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) + self.sp_model.Load(vocab_file) + super().__init__( + bos_token=bos_token, + eos_token=eos_token, + unk_token=unk_token, + pad_token=pad_token, + add_bos_token=add_bos_token, + add_eos_token=add_eos_token, + sp_model_kwargs=self.sp_model_kwargs, + clean_up_tokenization_spaces=clean_up_tokenization_spaces, + **kwargs, + ) + + def __getstate__(self): + state = self.__dict__.copy() + state["sp_model"] = None + return state + + def __setstate__(self, d): + self.__dict__ = d + self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) + self.sp_model.Load(self.vocab_file) + + @property + def vocab_size(self): + """Returns vocab size""" + return self.sp_model.get_piece_size() + + def get_vocab(self): + """Returns vocab as a dict""" + vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} + vocab.update(self.added_tokens_encoder) + return vocab + + def _tokenize(self, text): + """Returns a tokenized string.""" + return self.sp_model.encode(text, out_type=str) + + def _convert_token_to_id(self, token): + """Converts a token (str) in an id using the vocab.""" + return self.sp_model.piece_to_id(token) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + token = self.sp_model.IdToPiece(index) + return token + + def convert_tokens_to_string(self, tokens: List[str]): + """Converts a sequence of tokens (string) in a single string.""" + current_sub_tokens: List[str] = [] + out_string = "" + prev_is_special = False + for i, token in enumerate(tokens): + # make sure that special tokens are not decoded using + # sentencepiece model + if token in self.all_special_tokens: + if not prev_is_special and i != 0: + out_string += " " + out_string += self.sp_model.decode(current_sub_tokens) + token + prev_is_special = True + current_sub_tokens = [] + else: + current_sub_tokens.append(token) + prev_is_special = False + out_string += self.sp_model.decode(current_sub_tokens) + return out_string + + def save_vocabulary( + self, save_directory, filename_prefix: Optional[str] = None + ) -> Tuple[str]: + """ + Save the vocabulary and special tokens file to a directory. + + Args: + save_directory (`str`): + The directory in which to save the vocabulary. + + Returns: + `Tuple(str)`: Paths to the files saved. + """ + if not os.path.isdir(save_directory): + raise ValueError( + f"Vocabulary path ({save_directory}) " "should be a directory" + ) + + out_vocab_file = os.path.join( + save_directory, + (filename_prefix + "-" if filename_prefix else "") + + VOCAB_FILES_NAMES["vocab_file"], + ) + + if os.path.abspath(self.vocab_file) != os.path.abspath( + out_vocab_file + ) and os.path.isfile(self.vocab_file): + copyfile(self.vocab_file, out_vocab_file) + elif not os.path.isfile(self.vocab_file): + with open(out_vocab_file, "wb") as fi: + content_spiece_model = self.sp_model.serialized_model_proto() + fi.write(content_spiece_model) + + return (out_vocab_file,) + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + bos_token_id = [self.bos_token_id] if self.add_bos_token else [] + eos_token_id = [self.eos_token_id] if self.add_eos_token else [] + + output = bos_token_id + token_ids_0 + eos_token_id + + if token_ids_1 is not None: + output = output + bos_token_id + token_ids_1 + eos_token_id + + return output + + def get_special_tokens_mask( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None, + already_has_special_tokens: bool = False, + ) -> List[int]: + """ + Retrieve sequence ids from a token list that has no special tokens + added. This method is called when adding + special tokens using the tokenizer `prepare_for_model` method. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + already_has_special_tokens (`bool`, *optional*, defaults to + `False`): + Whether or not the token list is already formatted with + special tokens for the model. + + Returns: + `List[int]`: A list of integers in the range [0, 1]: + 1 for a special token, 0 for a sequence token. + """ + if already_has_special_tokens: + return super().get_special_tokens_mask( + token_ids_0=token_ids_0, + token_ids_1=token_ids_1, + already_has_special_tokens=True, + ) + + bos_token_id = [1] if self.add_bos_token else [] + eos_token_id = [1] if self.add_eos_token else [] + + if token_ids_1 is None: + return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id + return ( + bos_token_id + + ([0] * len(token_ids_0)) + + eos_token_id + + bos_token_id + + ([0] * len(token_ids_1)) + + eos_token_id + ) + + def create_token_type_ids_from_sequences( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Creates a mask from the two sequences passed to be used in a + sequence-pair classification task. An ALBERT + sequence pair mask has the following format: + + ``` + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence | + ``` + + if token_ids_1 is None, only returns the first portion of the mask (0s). + + Args: + token_ids_0 (`List[int]`): + List of ids. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [token type IDs](../glossary#token-type-ids) + according to the given sequence(s). + """ + bos_token_id = [self.bos_token_id] if self.add_bos_token else [] + eos_token_id = [self.eos_token_id] if self.add_eos_token else [] + + output = [0] * len(bos_token_id + token_ids_0 + eos_token_id) + + if token_ids_1 is not None: + output += [1] * len(bos_token_id + token_ids_1 + eos_token_id) + + return output From 52dc3f44cd9ddc0cb1c6b414a1fefcae669ec973 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 8 May 2024 10:16:08 -0700 Subject: [PATCH 37/62] updated to return string unless decode exception --- .../examples/fastapi/README.md | 6 +++++ .../examples/fastapi/fastapi-codegen/main.py | 25 ++++++++++--------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/README.md b/Triton_Inference_Server_Python_API/examples/fastapi/README.md index ab9eb060..e78e0750 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/README.md +++ b/Triton_Inference_Server_Python_API/examples/fastapi/README.md @@ -112,6 +112,12 @@ curl -s http://localhost:8000/models/llama-3-8b-instruct | jq . #### Completion +### Comparison + +curl http://localhost:8000/v1/completions -H "Content-Type: application/json" -d '{"model":"meta-llama/Meta-Llama-3-8B-Instruct","prompt":"say this is a test, but is it?","seed":800}' + + + #### chat completion curl http://localhost:8000/chat/completions -H "Content-Type: application/json" -d '{"model":"llama-3-8b-instruct","messages":[{"role":"system","content":"you are a helpful assistant."},{"role":"user","content":"Hello!"}]}' diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index 207cde96..f02cad0b 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -78,6 +78,15 @@ def load_model(server): ) +def get_output(response): + if "text_output" in response.outputs: + try: + return response.outputs["text_output"].to_string_array()[0] + except: + return str(response.outputs["text_output"].to_bytes_array()[0]) + return None + + def streaming_chat_completion_response(request_id, created, model, role, responses): # first chunk @@ -100,9 +109,7 @@ def streaming_chat_completion_response(request_id, created, model, role, respons yield f"data: {chunk.json(exclude_unset=True)}\n\n" for response in responses: - text = None - if "text_output" in response.outputs: - text = str(response.outputs["text_output"].to_bytes_array()[0]) + text = get_output(response) choice = ChatCompletionStreamingResponseChoice( index=0, @@ -227,9 +234,7 @@ def create_chat_completion( response = list(responses)[0] - text = None - if "text_output" in response.outputs: - text = str(response.outputs["text_output"].to_bytes_array()[0]) + text = get_output(response) return CreateChatCompletionResponse( id=request_id, @@ -252,9 +257,7 @@ def create_chat_completion( def streaming_completion_response(request_id, created, model, responses): for response in responses: - text = None - if "text_output" in response.outputs: - text = str(response.outputs["text_output"].to_bytes_array()[0]) + text = get_output(response) choice = Choice( finish_reason=FinishReason.stop if response.final else None, @@ -315,9 +318,7 @@ def create_completion( streaming_completion_response(request_id, created, model.name, responses) ) response = list(responses)[0] - text = None - if "text_output" in response.outputs: - text = str(response.outputs["text_output"].to_bytes_array()[0]) + text = get_output(response) choice = Choice( finish_reason=FinishReason.stop if response.final else None, From 2b42bee8ef3c61f16fd2344e58d5ea0dd52080da Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 8 May 2024 10:31:59 -0700 Subject: [PATCH 38/62] updated with metrics --- .../examples/fastapi/fastapi-codegen/main.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index f02cad0b..dd20c228 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -9,6 +9,7 @@ import numpy import tritonserver +import uvicorn from fastapi import FastAPI, HTTPException, Request from fastapi.responses import StreamingResponse from openai_protocol_types import ( @@ -339,6 +340,11 @@ def create_completion( owned_by = "ACME" +@app.get("/metrics") +def metrics() -> str: + return server.metrics() + + @app.get("/models", response_model=ListModelsResponse, tags=["Models"]) def list_models() -> ListModelsResponse: """ @@ -372,3 +378,7 @@ def retrieve_model(model_name: str) -> Model: ) raise HTTPException(status_code=404, detail=f"Unknown model: {model_name}") + + +if __name__ == "__main__": + uvicorn.run(app) From e53e76357fcda19ef686e76ad8a66148fd05848b Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 8 May 2024 10:37:58 -0700 Subject: [PATCH 39/62] deleting unused files --- .../transformers_utils/config.py | 68 ---- .../transformers_utils/configs/__init__.py | 17 - .../transformers_utils/configs/chatglm.py | 71 ---- .../transformers_utils/configs/dbrx.py | 277 --------------- .../transformers_utils/configs/falcon.py | 85 ----- .../transformers_utils/configs/jais.py | 242 ------------- .../transformers_utils/configs/mpt.py | 202 ----------- .../transformers_utils/detokenizer.py | 325 ------------------ .../tokenizer_group/__init__.py | 34 -- .../tokenizer_group/base_tokenizer_group.py | 55 --- .../tokenizer_group/ray_tokenizer_group.py | 181 ---------- .../tokenizer_group/tokenizer_group.py | 94 ----- .../transformers_utils/tokenizers/__init__.py | 5 - .../transformers_utils/tokenizers/baichuan.py | 270 --------------- 14 files changed, 1926 deletions(-) delete mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/config.py delete mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/__init__.py delete mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/chatglm.py delete mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/dbrx.py delete mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/falcon.py delete mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/jais.py delete mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/mpt.py delete mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/detokenizer.py delete mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/__init__.py delete mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/base_tokenizer_group.py delete mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/ray_tokenizer_group.py delete mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/tokenizer_group.py delete mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/__init__.py delete mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/baichuan.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/config.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/config.py deleted file mode 100644 index 9ae6b822..00000000 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/config.py +++ /dev/null @@ -1,68 +0,0 @@ -from typing import Dict, Optional - -from transformers import AutoConfig, PretrainedConfig -from vllm.transformers_utils.configs import ( - ChatGLMConfig, - DbrxConfig, - JAISConfig, - MPTConfig, - RWConfig, -) - -_CONFIG_REGISTRY: Dict[str, PretrainedConfig] = { - "chatglm": ChatGLMConfig, - "dbrx": DbrxConfig, - "mpt": MPTConfig, - "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct) - "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct) - "jais": JAISConfig, -} - - -def get_config( - model: str, - trust_remote_code: bool, - revision: Optional[str] = None, - code_revision: Optional[str] = None, -) -> PretrainedConfig: - try: - config = AutoConfig.from_pretrained( - model, - trust_remote_code=trust_remote_code, - revision=revision, - code_revision=code_revision, - ) - except ValueError as e: - if ( - not trust_remote_code - and "requires you to execute the configuration file" in str(e) - ): - err_msg = ( - "Failed to load the model config. If the model is a custom " - "model not yet available in the HuggingFace transformers " - "library, consider setting `trust_remote_code=True` in LLM " - "or using the `--trust-remote-code` flag in the CLI." - ) - raise RuntimeError(err_msg) from e - else: - raise e - if config.model_type in _CONFIG_REGISTRY: - config_class = _CONFIG_REGISTRY[config.model_type] - config = config_class.from_pretrained( - model, revision=revision, code_revision=code_revision - ) - return config - - -def get_hf_text_config(config: PretrainedConfig): - """Get the "sub" config relevant to llm for multi modal models. - No op for pure text models. - """ - if hasattr(config, "text_config"): - # The code operates under the assumption that text_config should have - # `num_attention_heads` (among others). Assert here to fail early - # if transformers config doesn't align with this assumption. - assert hasattr(config.text_config, "num_attention_heads") - return config.text_config - else: - return config diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/__init__.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/__init__.py deleted file mode 100644 index 8904e1b6..00000000 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -from vllm.transformers_utils.configs.chatglm import ChatGLMConfig -from vllm.transformers_utils.configs.dbrx import DbrxConfig - -# RWConfig is for the original tiiuae/falcon-40b(-instruct) and -# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the -# `FalconConfig` class from the official HuggingFace transformers library. -from vllm.transformers_utils.configs.falcon import RWConfig -from vllm.transformers_utils.configs.jais import JAISConfig -from vllm.transformers_utils.configs.mpt import MPTConfig - -__all__ = [ - "ChatGLMConfig", - "DbrxConfig", - "MPTConfig", - "RWConfig", - "JAISConfig", -] diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/chatglm.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/chatglm.py deleted file mode 100644 index e6b95c79..00000000 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/chatglm.py +++ /dev/null @@ -1,71 +0,0 @@ -# coding=utf-8 -# Adapted from -# https://github.com/THUDM/ChatGLM2-6B -from transformers import PretrainedConfig - - -class ChatGLMConfig(PretrainedConfig): - model_type = "chatglm" - attribute_map = { - "num_hidden_layers": "num_layers", - "n_head_kv": "multi_query_group_num", - } - - def __init__( - self, - num_layers=28, - padded_vocab_size=65024, - hidden_size=4096, - ffn_hidden_size=13696, - kv_channels=128, - num_attention_heads=32, - seq_length=2048, - hidden_dropout=0.0, - attention_dropout=0.0, - layernorm_epsilon=1e-5, - rmsnorm=True, - apply_residual_connection_post_layernorm=False, - post_layer_norm=True, - add_bias_linear=False, - add_qkv_bias=False, - interleaved_qkv=False, - bias_dropout_fusion=True, - multi_query_attention=False, - multi_query_group_num=1, - apply_query_key_layer_scaling=True, - attention_softmax_in_fp32=True, - fp32_residual_connection=False, - quantization_bit=0, - pre_seq_len=None, - prefix_projection=False, - **kwargs - ): - self.num_layers = num_layers - self.vocab_size = padded_vocab_size - self.padded_vocab_size = padded_vocab_size - self.hidden_size = hidden_size - self.ffn_hidden_size = ffn_hidden_size - self.kv_channels = kv_channels - self.num_attention_heads = num_attention_heads - self.seq_length = seq_length - self.hidden_dropout = hidden_dropout - self.attention_dropout = attention_dropout - self.layernorm_epsilon = layernorm_epsilon - self.rmsnorm = rmsnorm - self.apply_residual_connection_post_layernorm = ( - apply_residual_connection_post_layernorm - ) - self.post_layer_norm = post_layer_norm - self.add_bias_linear = add_bias_linear - self.add_qkv_bias = add_qkv_bias - self.bias_dropout_fusion = bias_dropout_fusion - self.multi_query_attention = multi_query_attention - self.multi_query_group_num = multi_query_group_num - self.apply_query_key_layer_scaling = apply_query_key_layer_scaling - self.attention_softmax_in_fp32 = attention_softmax_in_fp32 - self.fp32_residual_connection = fp32_residual_connection - self.quantization_bit = quantization_bit - self.pre_seq_len = pre_seq_len - self.prefix_projection = prefix_projection - self.interleaved_qkv = interleaved_qkv - super().__init__(**kwargs) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/dbrx.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/dbrx.py deleted file mode 100644 index 1d2724f2..00000000 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/dbrx.py +++ /dev/null @@ -1,277 +0,0 @@ -# yapf: disable -# ruff: noqa: E501 -# coding=utf-8 -# Copied from -# https://huggingface.co/databricks/dbrx-base/blob/main/configuration_dbrx.py -"""Dbrx configuration.""" - -from typing import Any, Optional - -from transformers.configuration_utils import PretrainedConfig -from transformers.utils import logging - -logger = logging.get_logger(__name__) - -DBRX_PRETRAINED_CONFIG_ARCHIVE_MAP = {} # type: ignore - - -class DbrxAttentionConfig(PretrainedConfig): - """Configuration class for Dbrx Attention. - - [`DbrxAttention`] class. It is used to instantiate attention layers - according to the specified arguments, defining the layers architecture. - - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PretrainedConfig`] for more information. - - Args: - attn_pdrop (`float`, *optional*, defaults to 0.0): - The dropout probability for the attention layers. - clip_qkv (`float`, *optional*, defaults to None): - If not `None`, clip the queries, keys, and values in the attention layer to this value. - kv_n_heads (Optional[int]): For grouped_query_attention only, allow user to specify number of kv heads. - rope_theta (float): The base frequency for rope. - """ - - def __init__( - self, - attn_pdrop: float = 0, - clip_qkv: Optional[float] = None, - kv_n_heads: int = 1, - rope_theta: float = 10000.0, - **kwargs: Any, - ): - super().__init__(**kwargs) - self.attn_pdrop = attn_pdrop - self.clip_qkv = clip_qkv - self.kv_n_heads = kv_n_heads - self.rope_theta = rope_theta - - for k in ["model_type"]: - if k in kwargs: - kwargs.pop(k) - if len(kwargs) != 0: - raise ValueError(f"Found unknown {kwargs=}") - - @classmethod - def from_pretrained( - cls, pretrained_model_name_or_path: str, **kwargs: Any - ) -> "PretrainedConfig": - cls._set_token_in_kwargs(kwargs) - - config_dict, kwargs = cls.get_config_dict( - pretrained_model_name_or_path, **kwargs - ) - - if config_dict.get("model_type") == "dbrx": - config_dict = config_dict["attn_config"] - - if ( - "model_type" in config_dict - and hasattr(cls, "model_type") - and config_dict["model_type"] != cls.model_type - ): - logger.warning( - f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " - + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." - ) - - return cls.from_dict(config_dict, **kwargs) - - -class DbrxFFNConfig(PretrainedConfig): - """Configuration class for Dbrx FFN. - - [`DbrxFFN`] class. It is used to instantiate feedforward layers according to - the specified arguments, defining the layers architecture. - - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PretrainedConfig`] for more information. - - Args: - ffn_act_fn (dict, optional): A dict specifying activation function for the FFN. - The dict should have a key 'name' with the value being the name of - the activation function along with any additional keyword arguments. - ffn_hidden_size (int, optional): The hidden size of the feedforward network. - moe_num_experts (int, optional): The number of experts in the mixture of experts layer. - moe_top_k (int, optional): The number of experts to use in the mixture of experts layer. - moe_jitter_eps (float, optional): The jitter epsilon for the mixture of experts layer. - moe_loss_weight (float, optional): The loss weight for the mixture of experts layer. - moe_normalize_expert_weights (float, optional): The normalization factor for the expert weights. - uniform_expert_assignment (bool, optional): Whether to use uniform expert assignment. - This should only be used for benchmarking purposes. - """ - - def __init__( - self, - ffn_act_fn: Optional[dict] = None, - ffn_hidden_size: int = 3584, - moe_num_experts: int = 4, - moe_top_k: int = 1, - moe_jitter_eps: Optional[float] = None, - moe_loss_weight: float = 0.01, - moe_normalize_expert_weights: Optional[float] = 1, - uniform_expert_assignment: bool = False, - **kwargs: Any, - ): - super().__init__() - if ffn_act_fn is None: - ffn_act_fn = {"name": "silu"} - self.ffn_act_fn = ffn_act_fn - self.ffn_hidden_size = ffn_hidden_size - self.moe_num_experts = moe_num_experts - self.moe_top_k = moe_top_k - self.moe_jitter_eps = moe_jitter_eps - self.moe_loss_weight = moe_loss_weight - self.moe_normalize_expert_weights = moe_normalize_expert_weights - self.uniform_expert_assignment = uniform_expert_assignment - - for k in ["model_type"]: - if k in kwargs: - kwargs.pop(k) - if len(kwargs) != 0: - raise ValueError(f"Found unknown {kwargs=}") - - @classmethod - def from_pretrained( - cls, pretrained_model_name_or_path: str, **kwargs: Any - ) -> "PretrainedConfig": - cls._set_token_in_kwargs(kwargs) - - config_dict, kwargs = cls.get_config_dict( - pretrained_model_name_or_path, **kwargs - ) - - if config_dict.get("model_type") == "dbrx": - config_dict = config_dict["ffn_config"] - - if ( - "model_type" in config_dict - and hasattr(cls, "model_type") - and config_dict["model_type"] != cls.model_type - ): - logger.warning( - f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " - + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." - ) - - return cls.from_dict(config_dict, **kwargs) - - -class DbrxConfig(PretrainedConfig): - """Configuration class for Dbrx. - - [`DbrxModel`]. It is used to instantiate a Dbrx model according to the - specified arguments, defining the model architecture. - - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PretrainedConfig`] for more information. - - - Args: - d_model (`int`, *optional*, defaults to 6144): - Dimensionality of the embeddings and hidden states. - n_heads (`int`, *optional*, defaults to 48): - Number of attention heads for each attention layer in the Transformer encoder. - n_layers (`int`, *optional*, defaults to 40): - Number of hidden layers in the Transformer encoder. - max_seq_len (`int`, *optional*, defaults to 32768): - The maximum sequence length of the model. - vocab_size (`int`, *optional*, defaults to 100352): - Vocabulary size of the Dbrx model. Defines the maximum number of different tokens that can be represented by - the `inputs_ids` passed when calling [`DbrxModel`]. - resid_pdrop (`float`, *optional*, defaults to 0.0): - The dropout probability applied to the attention output before combining with residual. - emb_pdrop (`float`, *optional*, defaults to 0.0): - The dropout probability for the embedding layer. - attn_config (`dict`, *optional*): - A dictionary used to configure the model's attention module. - ffn_config (`dict`, *optional*): - A dictionary used to configure the model's FFN module. - use_cache (`bool`, *optional*, defaults to `False`): - Whether or not the model should return the last key/values attentions (not used by all models). - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - output_router_logits (`bool`, *optional*, defaults to `False`): - Whether or not the router logits should be returned by the model. Enabling this will also - allow the model to output the auxiliary loss. See [here]() for more details - router_aux_loss_coef (`float`, *optional*, defaults to 0.001): - The aux loss factor for the total loss. - - - Example: - ```python - >>> from transformers import DbrxConfig, DbrxModel - - >>> # Initializing a Dbrx configuration - >>> configuration = DbrxConfig() - - >>> # Initializing a model (with random weights) from the configuration - >>> model = DbrxModel(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ``` - """ - - model_type = "dbrx" - attribute_map = { - "num_attention_heads": "n_heads", - "hidden_size": "d_model", - "num_hidden_layers": "n_layers", - "max_position_embeddings": "max_seq_len", - } - - def __init__( - self, - d_model: int = 2048, - n_heads: int = 16, - n_layers: int = 24, - max_seq_len: int = 2048, - vocab_size: int = 32000, - resid_pdrop: float = 0.0, - emb_pdrop: float = 0.0, - attn_config: Optional[DbrxAttentionConfig] = None, - ffn_config: Optional[DbrxFFNConfig] = None, - use_cache: bool = True, - initializer_range: float = 0.02, - output_router_logits: bool = False, - router_aux_loss_coef: float = 0.05, - **kwargs: Any, - ): - if attn_config is None: - self.attn_config = DbrxAttentionConfig() - elif isinstance(attn_config, dict): - self.attn_config = DbrxAttentionConfig(**attn_config) - else: - self.attn_config = attn_config - - if ffn_config is None: - self.ffn_config = DbrxFFNConfig() - elif isinstance(ffn_config, dict): - self.ffn_config = DbrxFFNConfig(**ffn_config) - else: - self.ffn_config = ffn_config - - self.d_model = d_model - self.n_heads = n_heads - self.n_layers = n_layers - self.max_seq_len = max_seq_len - self.vocab_size = vocab_size - self.resid_pdrop = resid_pdrop - self.emb_pdrop = emb_pdrop - self.use_cache = use_cache - self.initializer_range = initializer_range - self.output_router_logits = output_router_logits - self.router_aux_loss_coef = router_aux_loss_coef - - tie_word_embeddings = kwargs.pop("tie_word_embeddings", False) - if tie_word_embeddings: - raise ValueError( - "tie_word_embeddings is not supported for Dbrx models." - ) - - super().__init__( - tie_word_embeddings=tie_word_embeddings, - **kwargs, - ) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/falcon.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/falcon.py deleted file mode 100644 index 45366356..00000000 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/falcon.py +++ /dev/null @@ -1,85 +0,0 @@ -# Adapted from -# https://huggingface.co/tiiuae/falcon-7b/blob/main/configuration_RW.py -# Copyright 2023 The vLLM team. -# Copyright 2022 the Big Science Workshop and HuggingFace Inc. team. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Falcon configuration""" -from transformers.configuration_utils import PretrainedConfig - - -class RWConfig(PretrainedConfig): - model_type = "falcon" - keys_to_ignore_at_inference = ["past_key_values"] - attribute_map = { - "num_hidden_layers": "n_layer", - "num_attention_heads": "n_head", - "num_kv_heads": "n_head_kv", - } - - def __init__( - self, - vocab_size=250880, - hidden_size=64, - n_layer=2, - n_head=8, - layer_norm_epsilon=1e-5, - initializer_range=0.02, - use_cache=True, - bos_token_id=1, - eos_token_id=2, - hidden_dropout=0.0, - attention_dropout=0.0, - multi_query=True, - n_head_kv=None, - alibi=False, - bias=False, - parallel_attn=False, - new_decoder_architecture=False, - **kwargs, - ) -> None: - self.vocab_size = vocab_size - # Backward compatibility with n_embed kwarg - n_embed = kwargs.pop("n_embed", None) - self.hidden_size = hidden_size if n_embed is None else n_embed - self.n_layer = n_layer - self.n_head = n_head - self.layer_norm_epsilon = layer_norm_epsilon - self.initializer_range = initializer_range - self.use_cache = use_cache - self.hidden_dropout = hidden_dropout - self.attention_dropout = attention_dropout - - self.bos_token_id = bos_token_id - self.eos_token_id = eos_token_id - self.multi_query = multi_query - self.n_head_kv = 1 if n_head_kv is None else n_head_kv - self.alibi = alibi - self.bias = bias - self.parallel_attn = parallel_attn - self.new_decoder_architecture = new_decoder_architecture - - if self.hidden_size == 8192: - # Hack for falcon-40b - self.new_decoder_architecture = True - - super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs) - - @property - def head_dim(self): - return self.hidden_size // self.n_head - - @property - def rotary(self): - return not self.alibi diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/jais.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/jais.py deleted file mode 100644 index eb36a37c..00000000 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/jais.py +++ /dev/null @@ -1,242 +0,0 @@ -# coding=utf-8 -# Copyright 2023 The OpenAI Team Authors and HuggingFace Inc. team. -# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. -# Copyright 2023 Cerebras Systems. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""JAIS configuration""" - -from transformers.configuration_utils import PretrainedConfig -from transformers.utils import logging - -logger = logging.get_logger(__name__) - - -class JAISConfig(PretrainedConfig): - """ - This is the configuration class to store the configuration of a - [`JAISModel`]. It is used to instantiate a JAIS model according to the - specified arguments, defining the model architecture. - - Configuration objects inherit from [`PretrainedConfig`] and can be used - to control the model outputs. Read the documentation from - [`PretrainedConfig`] for more information. - - - Args: - vocab_size (`int`, *optional*, defaults to 50257): - Vocabulary size of the JAIS model. Defines the number of different - tokens that can be represented by the - `inputs_ids` passed when calling [`JAISModel`]. - n_positions (`int`, *optional*, defaults to 1024): - The maximum sequence length that this model might ever be used - with. Typically set this to something large just in case - (e.g., 512 or 1024 or 2048). - n_embd (`int`, *optional*, defaults to 768): - Dimensionality of the embeddings and hidden states. - n_layer (`int`, *optional*, defaults to 12): - Number of hidden layers in the Transformer encoder. - n_head (`int`, *optional*, defaults to 12): - Number of attention heads for each attention layer in the - Transformer encoder. - n_inner (`int`, *optional*, defaults to None): - Dimensionality of the inner feed-forward layers. `None` will set - it to 4 times n_embd - activation_function (`str`, *optional*, defaults to `"gelu"`): - Activation function, to be selected in the list - `["relu", "silu", "gelu", "tanh", "gelu_new", "swiglu"]`. - resid_pdrop (`float`, *optional*, defaults to 0.1): - The dropout probability for all fully connected layers in - the embeddings, encoder, and pooler. - embd_pdrop (`float`, *optional*, defaults to 0.1): - The dropout ratio for the embeddings. - attn_pdrop (`float`, *optional*, defaults to 0.1): - The dropout ratio for the attention. - layer_norm_epsilon (`float`, *optional*, defaults to 1e-5): - The epsilon to use in the layer normalization layers. - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for - initializing all weight matrices. - scale_attn_weights (`bool`, *optional*, defaults to `True`): - Scale attention weights by dividing by sqrt(hidden_size).. - use_cache (`bool`, *optional*, defaults to `True`): - Whether or not the model should return the last key/values - attentions (not used by all models). - scale_attn_by_inverse_layer_idx (`bool`, *optional*, - defaults to `False`): - Whether to additionally scale attention weights by - `1 / layer_idx + 1`. - reorder_and_upcast_attn (`bool`, *optional*, defaults to `False`): - Whether to scale keys (K) prior to computing attention - (dot-product) - and upcast attention dot-product/softmax to float() when training - with mixed precision. - position_embedding_type (`str`, *optional*, defaults to `"learned"`): - Positional embedding can be either `"alibi"` or `"learned"`. - mup_width_scale (`float`, *optional*, defaults to 1.0): - muP parameter to scale learning rate and initializers. Calculated - as (`d_model,0 / d_model`), where - `d_model` is the model's width and `d_model,0` is the proxy - model's width. - mup_embeddings_scale (`float`, *optional*, defaults to 1.0): - muP parameter to scale token and position embeddings. - mup_output_alpha (`float`, *optional*, defaults to 1.0): - muP parameter to scale output logits - (`output_logits_scale = mup_output_alpha * mup_width_scale`). - mup_scale_qk_dot_by_d (`bool`, *optional*, defaults to `False`): - Scale attention weights by dividing by hidden_size instead of - sqrt(hidden_size). Need to set scale_attn_weights to `True` as - well. - alibi_scaling (`Dict`, *optional*): - Dictionary containing the scaling configuration for ALiBi - embeddings. Currently only supports linear - scaling strategy. Can specify either the scaling `factor` (must be - a float greater than 1) for fixed scaling - or `train_seq_len` for dynamic scaling on input samples with - sequence length > `train_seq_len`. The expected - formats are `{"type": strategy name, "factor": scaling factor}` or - `{"type": strategy name, - "train_seq_len": training sequence length}`. - architectures (`List`, *optional*, defaults to ['JAISLMHeadModel']): - architecture names for Jais. - - Example: - - ```python - >>> from transformers import JAISConfig, JAISModel - - >>> # Initializing a JAIS configuration - >>> configuration = JAISConfig() - - >>> # Initializing a model (with random weights) from the configuration - >>> model = JAISModel(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "jais" - keys_to_ignore_at_inference = ["past_key_values"] - attribute_map = { - "hidden_size": "n_embd", - "max_position_embeddings": "n_positions", - "num_attention_heads": "n_head", - "num_hidden_layers": "n_layer", - } - - def __init__( - self, - vocab_size=50257, - n_positions=1024, - n_embd=768, - n_layer=12, - n_head=12, - n_inner=None, - activation_function="gelu_new", - resid_pdrop=0.1, - embd_pdrop=0.1, - attn_pdrop=0.1, - layer_norm_epsilon=1e-5, - initializer_range=0.02, - scale_attn_weights=True, - use_cache=True, - bos_token_id=50256, - eos_token_id=50256, - scale_attn_by_inverse_layer_idx=False, - reorder_and_upcast_attn=False, - position_embedding_type="learned", - mup_width_scale=1.0, - mup_embeddings_scale=1.0, - mup_output_alpha=1.0, - mup_scale_qk_dot_by_d=False, - alibi_scaling=None, - architectures=None, - **kwargs, - ): - self.vocab_size = vocab_size - self.n_positions = n_positions - self.n_embd = n_embd - self.n_layer = n_layer - self.n_head = n_head - self.n_inner = n_inner - self.activation_function = activation_function - self.resid_pdrop = resid_pdrop - self.embd_pdrop = embd_pdrop - self.attn_pdrop = attn_pdrop - self.layer_norm_epsilon = layer_norm_epsilon - self.initializer_range = initializer_range - self.scale_attn_weights = scale_attn_weights - self.use_cache = use_cache - self.scale_attn_by_inverse_layer_idx = scale_attn_by_inverse_layer_idx - self.reorder_and_upcast_attn = reorder_and_upcast_attn - - self.bos_token_id = bos_token_id - self.eos_token_id = eos_token_id - - self.position_embedding_type = position_embedding_type - self.mup_width_scale = mup_width_scale - self.mup_embeddings_scale = mup_embeddings_scale - self.mup_output_alpha = mup_output_alpha - self.mup_scale_qk_dot_by_d = mup_scale_qk_dot_by_d - - self.alibi_scaling = alibi_scaling - self._alibi_scaling_validation() - if architectures is None: - architectures = ["JAISLMHeadModel"] - - super().__init__( - bos_token_id=bos_token_id, - eos_token_id=eos_token_id, - architectures=architectures, - **kwargs, - ) - - def _alibi_scaling_validation(self): - """ - Validate the `alibi_scaling` configuration. - """ - if self.alibi_scaling is None: - return - - if not isinstance(self.alibi_scaling, dict) or len(self.alibi_scaling) != 2: - raise ValueError( - "`alibi_scaling` must be a dictionary with two fields," - "`type` and `factor` or `type` and `train_seq_len`, " - f"got {self.alibi_scaling}" - ) - alibi_scaling_type = self.alibi_scaling.get("type", None) - alibi_scaling_factor = self.alibi_scaling.get("factor", None) - alibi_dynamic_scaling = self.alibi_scaling.get("train_seq_len", None) - if alibi_scaling_type is None or alibi_scaling_type != "linear": - raise ValueError( - f"`alibi_scaling`'s type field must be 'linear'," - f"got {alibi_scaling_type}" - ) - if ( - alibi_scaling_factor is not None - and not isinstance(alibi_scaling_factor, float) - or (alibi_scaling_factor is not None and alibi_scaling_factor <= 1.0) - ): - raise ValueError( - f"`alibi_scaling`'s factor field must be a float > 1.0," - f"got {alibi_scaling_factor}" - ) - if ( - alibi_dynamic_scaling is not None - and not isinstance(alibi_dynamic_scaling, int) - or (alibi_dynamic_scaling is not None and alibi_dynamic_scaling <= 1) - ): - raise ValueError( - f"`alibi_scaling`'s `train_seq_len` field must be an" - f"integer > 1, got {alibi_dynamic_scaling}" - ) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/mpt.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/mpt.py deleted file mode 100644 index 969af314..00000000 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/configs/mpt.py +++ /dev/null @@ -1,202 +0,0 @@ -# coding=utf-8 -# Copied from -# https://huggingface.co/mosaicml/mpt-7b/blob/main/configuration_mpt.py -"""A HuggingFace-style model configuration.""" -import warnings -from typing import Any, Dict, Optional, Union - -from transformers import PretrainedConfig - -attn_config_defaults: Dict = { - "attn_type": "multihead_attention", - "attn_pdrop": 0.0, - "attn_impl": "triton", - "qk_ln": False, - "clip_qkv": None, - "softmax_scale": None, - "prefix_lm": False, - "attn_uses_sequence_id": False, - "alibi": False, - "alibi_bias_max": 8, -} -ffn_config_defaults: Dict = {"ffn_type": "mptmlp"} -init_config_defaults: Dict = { - "name": "kaiming_normal_", - "fan_mode": "fan_in", - "init_nonlinearity": "relu", - "init_div_is_residual": True, - "emb_init_std": None, - "emb_init_uniform_lim": None, - "init_std": None, - "init_gain": 0.0, -} - - -class MPTConfig(PretrainedConfig): - model_type = "mpt" - attribute_map = { - "num_attention_heads": "n_heads", - "hidden_size": "d_model", - "num_hidden_layers": "n_layers", - } - - # pylint: disable=dangerous-default-value - def __init__( - self, - d_model: int = 2048, - n_heads: int = 16, - n_layers: int = 24, - expansion_ratio: int = 4, - max_seq_len: int = 2048, - vocab_size: int = 50368, - resid_pdrop: float = 0.0, - emb_pdrop: float = 0.0, - learned_pos_emb: bool = True, - attn_config: Dict = attn_config_defaults, - ffn_config: Dict = ffn_config_defaults, - init_device: str = "cpu", - logit_scale: Optional[Union[float, str]] = None, - no_bias: bool = False, - embedding_fraction: float = 1.0, - norm_type: str = "low_precision_layernorm", - use_cache: bool = False, - init_config: Dict = init_config_defaults, - fc_type: str = "torch", - verbose: Optional[int] = None, - **kwargs: Any, - ): - self.d_model = d_model - self.n_heads = n_heads - self.n_layers = n_layers - self.expansion_ratio = expansion_ratio - self.max_seq_len = max_seq_len - self.vocab_size = vocab_size - self.resid_pdrop = resid_pdrop - self.emb_pdrop = emb_pdrop - self.learned_pos_emb = learned_pos_emb - self.attn_config = attn_config - self.ffn_config = ffn_config - self.init_device = init_device - self.logit_scale = logit_scale - self.no_bias = no_bias - self.embedding_fraction = embedding_fraction - self.norm_type = norm_type - self.use_cache = use_cache - self.init_config = init_config - self.fc_type = fc_type - if verbose is not None: - warnings.warn( - DeprecationWarning( - "verbose argument for MPTConfig is now ignored and " - "will be removed. Use python_log_level instead." - ), - stacklevel=2, - ) - if "name" in kwargs: - del kwargs["name"] - if "loss_fn" in kwargs: - del kwargs["loss_fn"] - if self.attn_config.get("alibi", False): - self.learned_pos_emb = False - warnings.warn( - f"alibi is turned on, setting `learned_pos_emb` " - f"to {self.learned_pos_emb}`", - stacklevel=2, - ) - super().__init__(**kwargs) - self._validate_config() - - def _set_config_defaults( - self, config: Dict[str, Any], config_defaults: Dict[str, Any] - ) -> Dict[str, Any]: - for k, v in config_defaults.items(): - if k not in config: - config[k] = v - return config - - def _validate_config(self) -> None: - self.attn_config = self._set_config_defaults( - self.attn_config, attn_config_defaults - ) - self.ffn_config = self._set_config_defaults( - self.ffn_config, ffn_config_defaults - ) - self.init_config = self._set_config_defaults( - self.init_config, init_config_defaults - ) - if self.d_model % self.n_heads != 0: - raise ValueError("d_model must be divisible by n_heads") - if any( - ( - prob < 0 or prob > 1 - for prob in [ - self.attn_config["attn_pdrop"], - self.resid_pdrop, - self.emb_pdrop, - ] - ) - ): - raise ValueError( - "self.attn_config['attn_pdrop'], resid_pdrop, emb_pdrop are " - "probabilities and must be between 0 and 1" - ) - if self.attn_config["attn_impl"] not in ["torch", "flash", "triton"]: - raise ValueError(f"Unknown attn_impl={self.attn_config['attn_impl']}") - if self.attn_config["prefix_lm"] and self.attn_config["attn_impl"] not in [ - "torch", - "triton", - ]: - raise NotImplementedError( - "prefix_lm only implemented with torch and triton attention." - ) - if self.attn_config["alibi"] and self.attn_config["attn_impl"] not in [ - "torch", - "triton", - ]: - raise NotImplementedError( - "alibi only implemented with torch and triton attention." - ) - if self.attn_config["attn_uses_sequence_id"] and self.attn_config[ - "attn_impl" - ] not in ["torch", "triton"]: - raise NotImplementedError( - "attn_uses_sequence_id only implemented with torch " - "and triton attention." - ) - if self.embedding_fraction > 1 or self.embedding_fraction <= 0: - raise ValueError( - "model.embedding_fraction must be between 0 (exclusive) " - "and 1 (inclusive)!" - ) - if isinstance(self.logit_scale, str) and self.logit_scale != "inv_sqrt_d_model": - raise ValueError( - f"self.logit_scale={self.logit_scale!r} is not recognized as " - "an option; use numeric value or 'inv_sqrt_d_model'." - ) - if self.init_config.get("name", None) is None: - raise ValueError( - f"self.init_config={self.init_config!r} 'name' needs to be set." - ) - if not self.learned_pos_emb and (not self.attn_config["alibi"]): - warnings.warn( - "Positional information not being provided to the model.", stacklevel=2 - ) - if self.fc_type == "te" or self.ffn_config["ffn_type"] == "te_ln_mlp": - try: - # pylint: disable=import-outside-toplevel - import transformer_engine.pytorch as te - - del te - except Exception as exc: - raise ImportError( - "TransformerEngine import fail. `fc_type: te` requires " - "TransformerEngine be installed. " - "The required version of transformer_engine also requires " - "FlashAttention v1.0.6 is installed:\n" - "pip install flash-attn==1.0.6 --no-build-isolation \n" - "pip install git+https://github.com/NVIDIA/TransformerEngine.git@144e4888b2cdd60bd52e706d5b7a79cb9c1a7156" - ) from exc - if self.ffn_config["ffn_type"] == "mptmlp": - self.ffn_config["fc_type"] = self.fc_type - elif self.ffn_config["ffn_type"] == "te_ln_mlp": - self.ffn_config["bias"] = not self.no_bias diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/detokenizer.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/detokenizer.py deleted file mode 100644 index b7ca62e0..00000000 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/detokenizer.py +++ /dev/null @@ -1,325 +0,0 @@ -from typing import Dict, List, Optional, Tuple, Union - -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast -from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup -from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import ( - BaseTokenizerGroup, -) - -# Used eg. for marking rejected tokens in spec decoding. -INVALID_TOKEN_ID = -1 - - -class Detokenizer: - """Provides methods to decode the output of a model into text.""" - - def __init__(self, tokenizer_group: BaseTokenizerGroup): - self.tokenizer_group = tokenizer_group - - def get_tokenizer_for_seq(self, sequence: Sequence) -> "PreTrainedTokenizer": - """Returns the HF tokenizer to use for a given sequence.""" - return self.tokenizer_group.get_lora_tokenizer(sequence.lora_request) - - def decode_prompt_logprobs_inplace( - self, - seq_group: SequenceGroup, - prompt_logprobs: List[Optional[Dict[int, Logprob]]], - ) -> None: - """Decodes the logprobs for the prompt of a sequence group. - - Args: - seq_group: The sequence group to decode. - prompt_logprobs: The logprobs to decode. - - Returns: - The prompt logprobs with the decoded tokens. - """ - prms = seq_group.sampling_params - # We can pick any sequence for the prompt. - seq = next(iter(seq_group.seqs_dict.values())) - # Only prompt, without the generated token. - all_token_ids = seq.get_token_ids() - prompt_token_ids = all_token_ids[:-1] - tokenizer = self.get_tokenizer_for_seq(seq) - prefix_offset = 0 - read_offset = 0 - next_iter_prefix_offset = 0 - next_iter_read_offset = 0 - next_iter_tokens = [] - prev_tokens = None - - for token_position, prompt_logprobs_for_token in enumerate(prompt_logprobs): - if not prompt_logprobs_for_token: - continue - for token_id, sample_logprob in prompt_logprobs_for_token.items(): - if ( - sample_logprob.decoded_token is None - and token_id != INVALID_TOKEN_ID - ): - prompt_token_ids_with_token = prompt_token_ids[:token_position] + [ - token_id - ] - ( - new_tokens, - new_text, - new_prefix_offset, - new_read_offset, - ) = detokenize_incrementally( - tokenizer=tokenizer, - all_input_ids=prompt_token_ids_with_token, - prev_tokens=prev_tokens, - prefix_offset=prefix_offset, - read_offset=read_offset, - skip_special_tokens=prms.skip_special_tokens, - spaces_between_special_tokens=prms.spaces_between_special_tokens, - ) - - sample_logprob.decoded_token = new_text - - # Use the offsets & prev tokens corresponding to - # real tokens to ensure detokenization is consistent - # actual with prompt. - if token_id == all_token_ids[token_position]: - next_iter_prefix_offset = new_prefix_offset - next_iter_read_offset = new_read_offset - next_iter_tokens = new_tokens - - # Advance to the next token position. - prefix_offset = next_iter_prefix_offset - read_offset = next_iter_read_offset - if prev_tokens is None: - prev_tokens = next_iter_tokens - else: - prev_tokens.extend(next_iter_tokens) - - def decode_sequence_inplace(self, seq: Sequence, prms: SamplingParams) -> int: - """Decodes the new token for a sequence. In-place operation. - - Args: - seq: The sequence to decode. - prms: The sampling parameters used to generate the sequence. - - Returns: - The number of characters added to the output text. - """ - all_input_ids = seq.get_token_ids() - token_id_generated_this_iteration = all_input_ids[-1] - tokenizer = self.get_tokenizer_for_seq(seq) - - # Convert prompt token IDs to tokens if necessary. - # Do it here so that we don't have to repeat this - # computation for each logprob. - if seq.tokens is None: - ( - seq.tokens, - seq.prefix_offset, - seq.read_offset, - ) = convert_prompt_ids_to_tokens( - tokenizer=tokenizer, - prompt_ids=all_input_ids[:-1], - skip_special_tokens=prms.skip_special_tokens, - ) - - ( - new_tokens, - new_decoded_token_text, - prefix_offset, - read_offset, - ) = detokenize_incrementally( - tokenizer=tokenizer, - all_input_ids=all_input_ids, - prev_tokens=seq.tokens, - prefix_offset=seq.prefix_offset, - read_offset=seq.read_offset, - skip_special_tokens=prms.skip_special_tokens, - spaces_between_special_tokens=prms.spaces_between_special_tokens, - ) - - # Decode logprobs - logprobs = seq.output_logprobs[-1] - if logprobs: - previous_tokens = all_input_ids[:-1] - for token_id, sample_logprob in logprobs.items(): - # If the token was generated this iteration, - # use the provided text. - if token_id == token_id_generated_this_iteration: - sample_logprob.decoded_token = new_decoded_token_text - continue - - if ( - sample_logprob.decoded_token is None - and token_id != INVALID_TOKEN_ID - ): - all_input_ids_with_logprob = previous_tokens + [token_id] - (_, new_text, _, _) = detokenize_incrementally( - tokenizer=tokenizer, - all_input_ids=all_input_ids_with_logprob, - prev_tokens=seq.tokens, - prefix_offset=seq.prefix_offset, - read_offset=seq.read_offset, - skip_special_tokens=prms.skip_special_tokens, - spaces_between_special_tokens=prms.spaces_between_special_tokens, - ) - sample_logprob.decoded_token = new_text - - seq.tokens.extend(new_tokens) - seq.prefix_offset = prefix_offset - seq.read_offset = read_offset - seq.output_text += new_decoded_token_text - - return len(new_decoded_token_text) - - -def _convert_tokens_to_string_with_added_encoders( - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], - output_tokens: List[str], - skip_special_tokens: bool, - spaces_between_special_tokens: bool, -) -> str: - # Adapted from - # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921 - # NOTE(woosuk): The following code is slow because it runs a for loop over - # the output_tokens. In Python, running a for loop over a list can be slow - # even when the loop body is very simple. - sub_texts: List[str] = [] - current_sub_text: List[str] = [] - all_special_tokens = set(tokenizer.all_special_tokens) - for token in output_tokens: - if skip_special_tokens and token in all_special_tokens: - continue - if token in tokenizer.get_added_vocab(): - if current_sub_text: - sub_text = tokenizer.convert_tokens_to_string(current_sub_text) - sub_texts.append(sub_text) - current_sub_text = [] - sub_texts.append(token) - else: - current_sub_text.append(token) - if current_sub_text: - sub_text = tokenizer.convert_tokens_to_string(current_sub_text) - sub_texts.append(sub_text) - if spaces_between_special_tokens: - return " ".join(sub_texts) - else: - return "".join(sub_texts) - - -# 5 is an arbitrary value that should work for all -# tokenizers (bigger = more conservative). -INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5 - - -def convert_prompt_ids_to_tokens( - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], - prompt_ids: List[int], - skip_special_tokens: bool = False, -) -> Tuple[List[str], int, int]: - """Converts the prompt ids to tokens and returns the tokens and offsets - for incremental detokenization. - - Note that not all tokens are converted to strings. Only the tokens that - are necessary for incremental detokenization are converted to strings. - """ - # We do not need to convert the whole prompt to tokens. - # Offset a little more in case we have special tokens. - new_tokens = tokenizer.convert_ids_to_tokens( - prompt_ids[-INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET - 2 :], - skip_special_tokens=skip_special_tokens, - ) - read_offset = len(new_tokens) - prefix_offset = max(read_offset - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0) - return new_tokens, prefix_offset, read_offset - - -# Based on -# https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15 -# under Apache 2.0 license -def detokenize_incrementally( - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], - all_input_ids: List[int], - prev_tokens: Optional[List[str]], - prefix_offset: int, - read_offset: int, - skip_special_tokens: bool = False, - spaces_between_special_tokens: bool = True, -) -> Tuple[List[str], str, int, int]: - """Detokenizes the input ids incrementally and returns the new tokens - and the new text. - - If `prev_tokens` is None, this function will convert the input ids to - tokens and return the tokens and the new text. Otherwise, it will return the - new tokens and the new text. - - This function will also return the new prefix offset and the new read - offset to be used in the next iteration. - - The offsets are necessary to defeat cleanup algorithms in the decode which - decide to add a space or not depending on the surrounding ids. - - Args: - tokenizer: The tokenizer to use. - all_input_ids: The input ids. The last id is the new token id. - prev_tokens: The previous tokens. If None, this function will convert - the input ids to tokens and return the tokens and the new text. - prefix_offset: The prefix offset. - read_offset: The read offset. - skip_special_tokens: Whether to skip special tokens. - spaces_between_special_tokens: Whether to add spaces between special - tokens. - """ - new_token_id = all_input_ids[-1] - # This is the first iteration for this sequence - is_first_iter = prev_tokens is None - if is_first_iter: - (prev_tokens, prefix_offset, read_offset) = convert_prompt_ids_to_tokens( - tokenizer, all_input_ids[:-1], skip_special_tokens=skip_special_tokens - ) - assert prev_tokens is not None - - # If the new token id is out of bounds, return an empty string. - if new_token_id >= len(tokenizer): - new_tokens = [""] - else: - # Put new_token_id in a list so skip_special_tokens is respected - new_tokens = tokenizer.convert_ids_to_tokens( - [new_token_id], skip_special_tokens=skip_special_tokens - ) - if isinstance(new_tokens, str): - new_tokens = [new_tokens] - output_tokens = prev_tokens + new_tokens - - # If this is the first iteration, return all tokens. - if is_first_iter: - new_tokens = output_tokens - - # The prefix text is necessary only to defeat cleanup algorithms in - # the decode which decide to add a space or not depending on the - # surrounding ids. - if tokenizer.is_fast or not tokenizer.get_added_vocab(): - prefix_text = tokenizer.convert_tokens_to_string( - output_tokens[prefix_offset:read_offset] - ) - new_text = tokenizer.convert_tokens_to_string(output_tokens[prefix_offset:]) - else: - prefix_text = _convert_tokens_to_string_with_added_encoders( - tokenizer, - output_tokens[prefix_offset:read_offset], - skip_special_tokens=skip_special_tokens, - spaces_between_special_tokens=spaces_between_special_tokens, - ) - new_text = _convert_tokens_to_string_with_added_encoders( - tokenizer, - output_tokens[prefix_offset:], - skip_special_tokens=skip_special_tokens, - spaces_between_special_tokens=spaces_between_special_tokens, - ) - - if len(new_text) <= len(prefix_text) or new_text.endswith("�"): - # utf-8 char at the end means it's a potential unfinished byte sequence - # from byte fallback tokenization. - # If it's in the middle, it's probably a real invalid id generated - # by the model - return new_tokens, "", prefix_offset, read_offset - - new_text = new_text[len(prefix_text) :] - return new_tokens, new_text, read_offset, len(output_tokens) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/__init__.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/__init__.py deleted file mode 100644 index d952b789..00000000 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -from typing import Optional - -from vllm.config import TokenizerPoolConfig -from vllm.engine.ray_utils import ray -from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import ( - BaseTokenizerGroup, -) -from vllm.transformers_utils.tokenizer_group.tokenizer_group import TokenizerGroup - -if ray: - from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import ( - RayTokenizerGroupPool, - ) -else: - RayTokenizerGroupPool = None # type: ignore - - -def get_tokenizer_group( - tokenizer_pool_config: Optional[TokenizerPoolConfig], **init_kwargs -) -> BaseTokenizerGroup: - if tokenizer_pool_config is None: - return TokenizerGroup(**init_kwargs) - if tokenizer_pool_config.pool_type == "ray": - if RayTokenizerGroupPool is None: - raise ImportError( - "RayTokenizerGroupPool is not available. Please install " - "the ray package to use the Ray tokenizer group pool." - ) - return RayTokenizerGroupPool.from_config(tokenizer_pool_config, **init_kwargs) - else: - raise ValueError(f"Unknown pool type: {tokenizer_pool_config.pool_type}") - - -__all__ = ["get_tokenizer_group", "BaseTokenizerGroup"] diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/base_tokenizer_group.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/base_tokenizer_group.py deleted file mode 100644 index ecffa071..00000000 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/base_tokenizer_group.py +++ /dev/null @@ -1,55 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List, Optional - -from transformers import PreTrainedTokenizer -from vllm.lora.request import LoRARequest - - -class BaseTokenizerGroup(ABC): - """A group of tokenizers that can be used for LoRA adapters.""" - - @abstractmethod - def ping(self) -> bool: - """Check if the tokenizer group is alive.""" - pass - - @abstractmethod - def get_max_input_len( - self, lora_request: Optional[LoRARequest] = None - ) -> Optional[int]: - """Get the maximum input length for the LoRA request.""" - pass - - @abstractmethod - def encode( - self, - prompt: str, - request_id: Optional[str] = None, - lora_request: Optional[LoRARequest] = None, - ) -> List[int]: - """Encode a prompt using the tokenizer group.""" - pass - - @abstractmethod - async def encode_async( - self, - prompt: str, - request_id: Optional[str] = None, - lora_request: Optional[LoRARequest] = None, - ) -> List[int]: - """Encode a prompt using the tokenizer group.""" - pass - - @abstractmethod - def get_lora_tokenizer( - self, lora_request: Optional[LoRARequest] = None - ) -> "PreTrainedTokenizer": - """Get a tokenizer for a LoRA request.""" - pass - - @abstractmethod - async def get_lora_tokenizer_async( - self, lora_request: Optional[LoRARequest] = None - ) -> "PreTrainedTokenizer": - """Get a tokenizer for a LoRA request.""" - pass diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/ray_tokenizer_group.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/ray_tokenizer_group.py deleted file mode 100644 index a8182ee7..00000000 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/ray_tokenizer_group.py +++ /dev/null @@ -1,181 +0,0 @@ -import asyncio -import os -from typing import List, Optional - -from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy -from transformers import PreTrainedTokenizer -from vllm.config import TokenizerPoolConfig -from vllm.engine.ray_utils import ray -from vllm.lora.request import LoRARequest -from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import ( - BaseTokenizerGroup, -) -from vllm.transformers_utils.tokenizer_group.tokenizer_group import TokenizerGroup - - -class RayTokenizerGroupPool(BaseTokenizerGroup): - """A Ray-based pool of TokenizerGroups for async tokenization.""" - - # Class to use for workers making up the pool. - _worker_cls = TokenizerGroup - - @classmethod - def from_config( - cls, tokenizer_pool_config: TokenizerPoolConfig, **init_kwargs - ) -> "RayTokenizerGroupPool": - ray_actor_options = tokenizer_pool_config.extra_config or {"num_cpus": 0} - ray_actor_options.setdefault( - "scheduling_strategy", - NodeAffinitySchedulingStrategy( - node_id=ray.get_runtime_context().get_node_id(), soft=True - ), - ) - - # Carry over the env vars to the actors. - # This is necessary for API keys and such. - ray_actor_options.setdefault("runtime_env", {}) - _carry_over_env_vars_to_runtime_env(ray_actor_options["runtime_env"]) - - init_kwargs["num_actors"] = tokenizer_pool_config.pool_size - init_kwargs["ray_actor_options"] = ray_actor_options - - return cls(**init_kwargs) - - def __init__( - self, - tokenizer_id: str, - enable_lora: bool, - max_num_seqs: int, - max_input_length: Optional[int], - num_actors: int, - ray_actor_options: dict, - **tokenizer_config - ): - # Store a local copy of the TokenizerGroup for quick access - # to underlying HF tokenizers. - self._local_tokenizer_group = self._worker_cls( - tokenizer_id=tokenizer_id, - enable_lora=enable_lora, - max_num_seqs=max_num_seqs, - max_input_length=max_input_length, - **tokenizer_config, - ) - - ray_tokenizer_group_cls = ray.remote(self._worker_cls).options( - **ray_actor_options - ) - self.tokenizer_actors = [ - ray_tokenizer_group_cls.remote( - tokenizer_id, - enable_lora, - max_num_seqs, - max_input_length, - **tokenizer_config, - ) - for _ in range(num_actors) - ] - self._idle_actors: Optional[asyncio.Queue] = None - - @property - def pool_size(self) -> int: - return len(self.tokenizer_actors) - - def ping(self): - return ray.get([actor.ping.remote() for actor in self.tokenizer_actors]) - - def _ensure_queue_initialized(self): - if self._idle_actors is None: - self._idle_actors = asyncio.Queue() - for actor in self.tokenizer_actors: - self._idle_actors.put_nowait(actor) - - def encode( - self, - prompt: str, - request_id: Optional[str] = None, - lora_request: Optional[LoRARequest] = None, - ) -> List[int]: - """Encode a prompt using the tokenizer group. - - We pick an idle actor and use it to encode the prompt. - The actor is then put back in the queue for future use. - This is blocking. - """ - self._ensure_queue_initialized() - assert self._idle_actors is not None - - if self._idle_actors.empty(): - raise RuntimeError("No idle actors available.") - actor = self._idle_actors.get_nowait() - try: - ret = ray.get( - actor.encode.remote( - request_id=request_id, prompt=prompt, lora_request=lora_request - ) - ) - finally: - # Put the actor back in the queue. - # This is done in a finally block to ensure that the actor is - # always put back in the queue, even if an exception/cancellation - # is raised. - self._idle_actors.put_nowait(actor) - return ret - - async def encode_async( - self, - prompt: str, - request_id: Optional[str] = None, - lora_request: Optional[LoRARequest] = None, - ) -> List[int]: - """Encode a prompt using the tokenizer group. - - We pick an idle actor and use it to encode the prompt. - If there are no idle actors, we wait until one becomes - available. - The actor is then put back in the queue for future use. - This is non-blocking. - """ - self._ensure_queue_initialized() - assert self._idle_actors is not None - - actor = await self._idle_actors.get() - try: - ret = await actor.encode.remote( - request_id=request_id, prompt=prompt, lora_request=lora_request - ) - finally: - # Put the actor back in the queue. - # This is done in a finally block to ensure that the actor is - # always put back in the queue, even if an exception/cancellation - # is raised. - self._idle_actors.put_nowait(actor) - return ret - - def get_max_input_len( - self, lora_request: Optional[LoRARequest] = None - ) -> Optional[int]: - """Get the maximum input length for the LoRA request.""" - return self._local_tokenizer_group.get_max_input_len(lora_request) - - def get_lora_tokenizer( - self, lora_request: Optional[LoRARequest] = None - ) -> "PreTrainedTokenizer": - return self._local_tokenizer_group.get_lora_tokenizer(lora_request) - - async def get_lora_tokenizer_async( - self, lora_request: Optional[LoRARequest] = None - ) -> "PreTrainedTokenizer": - return await self._local_tokenizer_group.get_lora_tokenizer_async(lora_request) - - -def _carry_over_env_vars_to_runtime_env(runtime_env: dict) -> None: - """Copy over all current process environment variables to the runtime_env. - - The variables in runtime_env will take precedence over the current process - environment variables. - - runtime_env will be modified in place.""" - env_vars = os.environ.copy() - runtime_env.setdefault("env_vars", {}) - env_vars.update(runtime_env["env_vars"]) - runtime_env["env_vars"] = env_vars diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/tokenizer_group.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/tokenizer_group.py deleted file mode 100644 index cc9736fa..00000000 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizer_group/tokenizer_group.py +++ /dev/null @@ -1,94 +0,0 @@ -from typing import List, Optional - -from transformers import PreTrainedTokenizer -from vllm.lora.request import LoRARequest -from vllm.transformers_utils.tokenizer import ( - get_lora_tokenizer, - get_lora_tokenizer_async, - get_tokenizer, -) -from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import ( - BaseTokenizerGroup, -) -from vllm.utils import LRUCache - - -class TokenizerGroup(BaseTokenizerGroup): - """A group of tokenizers that can be used for LoRA adapters.""" - - def __init__( - self, - tokenizer_id: str, - enable_lora: bool, - max_num_seqs: int, - max_input_length: Optional[int], - **tokenizer_config - ): - self.tokenizer_id = tokenizer_id - self.tokenizer_config = tokenizer_config - self.enable_lora = enable_lora - self.max_input_length = max_input_length - self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config) - self.lora_tokenizers = ( - LRUCache[PreTrainedTokenizer](capacity=max_num_seqs) - if enable_lora - else None - ) - - def ping(self) -> bool: - """Check if the tokenizer group is alive.""" - return True - - def get_max_input_len( - self, lora_request: Optional[LoRARequest] = None - ) -> Optional[int]: - """Get the maximum input length for the LoRA request.""" - return self.max_input_length - - def encode( - self, - prompt: str, - request_id: Optional[str] = None, - lora_request: Optional[LoRARequest] = None, - ) -> List[int]: - tokenizer = self.get_lora_tokenizer(lora_request) - return tokenizer.encode(prompt) - - async def encode_async( - self, - prompt: str, - request_id: Optional[str] = None, - lora_request: Optional[LoRARequest] = None, - ) -> List[int]: - tokenizer = await self.get_lora_tokenizer_async(lora_request) - return tokenizer.encode(prompt) - - def get_lora_tokenizer( - self, lora_request: Optional[LoRARequest] = None - ) -> "PreTrainedTokenizer": - if not lora_request or not self.enable_lora: - return self.tokenizer - if lora_request.lora_int_id not in self.lora_tokenizers: - tokenizer = ( - get_lora_tokenizer(lora_request, **self.tokenizer_config) - or self.tokenizer - ) - self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer) - return tokenizer - else: - return self.lora_tokenizers.get(lora_request.lora_int_id) - - async def get_lora_tokenizer_async( - self, lora_request: Optional[LoRARequest] = None - ) -> "PreTrainedTokenizer": - if not lora_request or not self.enable_lora: - return self.tokenizer - if lora_request.lora_int_id not in self.lora_tokenizers: - tokenizer = ( - await get_lora_tokenizer_async(lora_request, **self.tokenizer_config) - or self.tokenizer - ) - self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer) - return tokenizer - else: - return self.lora_tokenizers.get(lora_request.lora_int_id) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/__init__.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/__init__.py deleted file mode 100644 index e6b59722..00000000 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from vllm.transformers_utils.tokenizers.baichuan import BaichuanTokenizer - -__all__ = [ - "BaichuanTokenizer", -] diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/baichuan.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/baichuan.py deleted file mode 100644 index ba4baaaf..00000000 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/transformers_utils/tokenizers/baichuan.py +++ /dev/null @@ -1,270 +0,0 @@ -# Adapted from -# https://huggingface.co/baichuan-inc/Baichuan2-13B-Chat/blob/8f6e343d545c503b91429582231d1d354dac2740/tokenization_baichuan.py -# This includes a fix suggested in -# https://github.com/vllm-project/vllm/issues/1403#issuecomment-1767503058 -# Copyright (c) 2023, Baichuan Intelligent Technology. All rights reserved. - -import os -from shutil import copyfile -from typing import Any, Dict, List, Optional, Tuple - -import sentencepiece as spm -from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer -from transformers.utils import logging - -logger = logging.get_logger(__name__) - -VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"} - -PRETRAINED_VOCAB_FILES_MAP = { # type: ignore - "vocab_file": {}, - "tokenizer_file": {}, -} -PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {} # type: ignore - - -class BaichuanTokenizer(PreTrainedTokenizer): - """ - Construct a Baichuan tokenizer. Based on byte-level Byte-Pair-Encoding. - - Args: - vocab_file (`str`): - Path to the vocabulary file. - """ - - vocab_files_names = VOCAB_FILES_NAMES - pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP - max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES - model_input_names = ["input_ids", "attention_mask"] - - def __init__( - self, - vocab_file, - unk_token="", - bos_token="", - eos_token="", - pad_token=None, - sp_model_kwargs: Optional[Dict[str, Any]] = None, - add_bos_token=True, - add_eos_token=False, - clean_up_tokenization_spaces=False, - **kwargs, - ): - self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs - bos_token = ( - AddedToken(bos_token, lstrip=False, rstrip=False) - if isinstance(bos_token, str) - else bos_token - ) - eos_token = ( - AddedToken(eos_token, lstrip=False, rstrip=False) - if isinstance(eos_token, str) - else eos_token - ) - unk_token = ( - AddedToken(unk_token, lstrip=False, rstrip=False) - if isinstance(unk_token, str) - else unk_token - ) - pad_token = ( - AddedToken(pad_token, lstrip=False, rstrip=False) - if isinstance(pad_token, str) - else pad_token - ) - self.vocab_file = vocab_file - self.add_bos_token = add_bos_token - self.add_eos_token = add_eos_token - self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) - self.sp_model.Load(vocab_file) - super().__init__( - bos_token=bos_token, - eos_token=eos_token, - unk_token=unk_token, - pad_token=pad_token, - add_bos_token=add_bos_token, - add_eos_token=add_eos_token, - sp_model_kwargs=self.sp_model_kwargs, - clean_up_tokenization_spaces=clean_up_tokenization_spaces, - **kwargs, - ) - - def __getstate__(self): - state = self.__dict__.copy() - state["sp_model"] = None - return state - - def __setstate__(self, d): - self.__dict__ = d - self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) - self.sp_model.Load(self.vocab_file) - - @property - def vocab_size(self): - """Returns vocab size""" - return self.sp_model.get_piece_size() - - def get_vocab(self): - """Returns vocab as a dict""" - vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} - vocab.update(self.added_tokens_encoder) - return vocab - - def _tokenize(self, text): - """Returns a tokenized string.""" - return self.sp_model.encode(text, out_type=str) - - def _convert_token_to_id(self, token): - """Converts a token (str) in an id using the vocab.""" - return self.sp_model.piece_to_id(token) - - def _convert_id_to_token(self, index): - """Converts an index (integer) in a token (str) using the vocab.""" - token = self.sp_model.IdToPiece(index) - return token - - def convert_tokens_to_string(self, tokens: List[str]): - """Converts a sequence of tokens (string) in a single string.""" - current_sub_tokens: List[str] = [] - out_string = "" - prev_is_special = False - for i, token in enumerate(tokens): - # make sure that special tokens are not decoded using - # sentencepiece model - if token in self.all_special_tokens: - if not prev_is_special and i != 0: - out_string += " " - out_string += self.sp_model.decode(current_sub_tokens) + token - prev_is_special = True - current_sub_tokens = [] - else: - current_sub_tokens.append(token) - prev_is_special = False - out_string += self.sp_model.decode(current_sub_tokens) - return out_string - - def save_vocabulary( - self, save_directory, filename_prefix: Optional[str] = None - ) -> Tuple[str]: - """ - Save the vocabulary and special tokens file to a directory. - - Args: - save_directory (`str`): - The directory in which to save the vocabulary. - - Returns: - `Tuple(str)`: Paths to the files saved. - """ - if not os.path.isdir(save_directory): - raise ValueError( - f"Vocabulary path ({save_directory}) " "should be a directory" - ) - - out_vocab_file = os.path.join( - save_directory, - (filename_prefix + "-" if filename_prefix else "") - + VOCAB_FILES_NAMES["vocab_file"], - ) - - if os.path.abspath(self.vocab_file) != os.path.abspath( - out_vocab_file - ) and os.path.isfile(self.vocab_file): - copyfile(self.vocab_file, out_vocab_file) - elif not os.path.isfile(self.vocab_file): - with open(out_vocab_file, "wb") as fi: - content_spiece_model = self.sp_model.serialized_model_proto() - fi.write(content_spiece_model) - - return (out_vocab_file,) - - def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): - bos_token_id = [self.bos_token_id] if self.add_bos_token else [] - eos_token_id = [self.eos_token_id] if self.add_eos_token else [] - - output = bos_token_id + token_ids_0 + eos_token_id - - if token_ids_1 is not None: - output = output + bos_token_id + token_ids_1 + eos_token_id - - return output - - def get_special_tokens_mask( - self, - token_ids_0: List[int], - token_ids_1: Optional[List[int]] = None, - already_has_special_tokens: bool = False, - ) -> List[int]: - """ - Retrieve sequence ids from a token list that has no special tokens - added. This method is called when adding - special tokens using the tokenizer `prepare_for_model` method. - - Args: - token_ids_0 (`List[int]`): - List of IDs. - token_ids_1 (`List[int]`, *optional*): - Optional second list of IDs for sequence pairs. - already_has_special_tokens (`bool`, *optional*, defaults to - `False`): - Whether or not the token list is already formatted with - special tokens for the model. - - Returns: - `List[int]`: A list of integers in the range [0, 1]: - 1 for a special token, 0 for a sequence token. - """ - if already_has_special_tokens: - return super().get_special_tokens_mask( - token_ids_0=token_ids_0, - token_ids_1=token_ids_1, - already_has_special_tokens=True, - ) - - bos_token_id = [1] if self.add_bos_token else [] - eos_token_id = [1] if self.add_eos_token else [] - - if token_ids_1 is None: - return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id - return ( - bos_token_id - + ([0] * len(token_ids_0)) - + eos_token_id - + bos_token_id - + ([0] * len(token_ids_1)) - + eos_token_id - ) - - def create_token_type_ids_from_sequences( - self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None - ) -> List[int]: - """ - Creates a mask from the two sequences passed to be used in a - sequence-pair classification task. An ALBERT - sequence pair mask has the following format: - - ``` - 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 - | first sequence | second sequence | - ``` - - if token_ids_1 is None, only returns the first portion of the mask (0s). - - Args: - token_ids_0 (`List[int]`): - List of ids. - token_ids_1 (`List[int]`, *optional*): - Optional second list of IDs for sequence pairs. - - Returns: - `List[int]`: List of [token type IDs](../glossary#token-type-ids) - according to the given sequence(s). - """ - bos_token_id = [self.bos_token_id] if self.add_bos_token else [] - eos_token_id = [self.eos_token_id] if self.add_eos_token else [] - - output = [0] * len(bos_token_id + token_ids_0 + eos_token_id) - - if token_ids_1 is not None: - output += [1] * len(bos_token_id + token_ids_1 + eos_token_id) - - return output From 0774e8454f1d6a9ed995e005d99a24004682a1c8 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 8 May 2024 13:27:58 -0700 Subject: [PATCH 40/62] adding basic support for arg parsing --- .../deps/requirements_trt_llm.txt | 1 + .../deps/requirements_vllm.txt | 1 + .../examples/fastapi/fastapi-codegen/main.py | 97 +++++++++++++++---- 3 files changed, 79 insertions(+), 20 deletions(-) diff --git a/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt b/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt index 8f618a1a..eb3e95b7 100644 --- a/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt +++ b/Triton_Inference_Server_Python_API/deps/requirements_trt_llm.txt @@ -25,4 +25,5 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. fastapi==0.111.0 +openai==1.26.0 pydantic==2.7.1 diff --git a/Triton_Inference_Server_Python_API/deps/requirements_vllm.txt b/Triton_Inference_Server_Python_API/deps/requirements_vllm.txt index f3c4524d..fd9a1fee 100644 --- a/Triton_Inference_Server_Python_API/deps/requirements_vllm.txt +++ b/Triton_Inference_Server_Python_API/deps/requirements_vllm.txt @@ -24,4 +24,5 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +openai==1.26.0 vllm[all]==0.4.1 diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index dd20c228..c6c5f7ad 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -4,8 +4,10 @@ from __future__ import annotations +import argparse import time import uuid +from typing import Optional, Union import numpy import tritonserver @@ -29,16 +31,20 @@ Model, ObjectType, ) +from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast from transformers_utils.tokenizer import get_tokenizer from triton_cli.constants import SUPPORTED_BACKENDS from triton_cli.parser import KNOWN_MODEL_SOURCES as KNOWN_MODELS -server = tritonserver.Server( - model_repository="/workspace/llm-models", - log_verbose=6, - strict_model_config=False, - model_control_mode=tritonserver.ModelControlMode.EXPLICIT, -).start(wait_until_ready=True) +TIMEOUT_KEEP_ALIVE = 5 # seconds + + +server: tritonserver.Server +model: tritonserver.Model +model_create_time: int +backend: str +tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast] +create_inference_request = None def load_model(server): @@ -60,11 +66,6 @@ def load_model(server): return None, None, None, None -model, model_create_time, backend, tokenizer = load_model(server) - -if not (model and backend and tokenizer and model_create_time): - raise Exception("Unknown Model") - app = FastAPI( title="OpenAI API", description="The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details.", @@ -180,14 +181,6 @@ def create_trtllm_inference_request( return model.create_request(inputs=inputs) -create_inference_request = None - -if backend == "vllm": - create_inference_request = create_vllm_inference_request -elif backend == "tensorrtllm": - create_inference_request = create_trtllm_inference_request - - @app.post( "/chat/completions", response_model=CreateChatCompletionResponse, tags=["Chat"] ) @@ -380,5 +373,69 @@ def retrieve_model(model_name: str) -> Model: raise HTTPException(status_code=404, detail=f"Unknown model: {model_name}") +def parse_args(): + parser = argparse.ArgumentParser( + description="Triton OpenAI Compatible RESTful API server." + ) + parser.add_argument("--host", type=str, default=None, help="host name") + parser.add_argument("--port", type=int, default=8000, help="port number") + parser.add_argument( + "--uvicorn-log-level", + type=str, + default="info", + choices=["debug", "info", "warning", "error", "critical", "trace"], + help="log level for uvicorn", + ) + parser.add_argument( + "--response-role", type=str, default="assistant", help="The role name to return" + ) + + parser.add_argument( + "--tritonserver-log-level", + type=int, + default=0, + help="The tritonserver log level", + ) + + parser.add_argument( + "--model-repository", + type=str, + default="/workspace/llm-models", + help="model repository", + ) + return parser.parse_args() + + if __name__ == "__main__": - uvicorn.run(app) + args = parse_args() + + print("Starting Triton Server Core", flush=True) + + server = tritonserver.Server( + model_repository=args.model_repository, + log_verbose=args.tritonserver_log_level, + strict_model_config=False, + model_control_mode=tritonserver.ModelControlMode.EXPLICIT, + ).start(wait_until_ready=True) + + print("Loading Model...\n\n", flush=True) + + model, model_create_time, backend, tokenizer = load_model(server) + + if not (model and backend and tokenizer and model_create_time): + raise Exception("Unknown Model") + + print(f"\n\nModel: {model.name} Loaded with Backend: {backend}\n\n", flush=True) + + if backend == "vllm": + create_inference_request = create_vllm_inference_request + elif backend == "tensorrtllm": + create_inference_request = create_trtllm_inference_request + + uvicorn.run( + app, + host=args.host, + port=args.port, + log_level=args.uvicorn_log_level, + timeout_keep_alive=TIMEOUT_KEEP_ALIVE, + ) From 7228d9e919c6349a2cbda5764c136e6abc40a540 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 8 May 2024 20:48:16 -0700 Subject: [PATCH 41/62] update to list both triton id and source id --- .../examples/fastapi/fastapi-codegen/main.py | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index c6c5f7ad..102a57ba 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -41,6 +41,7 @@ server: tritonserver.Server model: tritonserver.Model +model_source_name: str model_create_time: int backend: str tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast] @@ -51,6 +52,7 @@ def load_model(server): model = None backends = [] tokenizer = None + model_source_name = None for model_name, version in server.models().keys(): if version != -1: continue @@ -58,12 +60,13 @@ def load_model(server): backends.append(current_model.config()["backend"]) if model_name in KNOWN_MODELS.keys(): model = current_model - tokenizer = get_tokenizer(KNOWN_MODELS[model_name].replace("hf:", "")) + model_source_name = KNOWN_MODELS[model_name].replace("hf:", "") + tokenizer = get_tokenizer(model_source_name) if model and tokenizer: for backend in backends: if backend in SUPPORTED_BACKENDS: - return model, int(time.time()), backend, tokenizer - return None, None, None, None + return model, int(time.time()), backend, tokenizer, model_source_name + return None, None, None, None, None app = FastAPI( @@ -197,7 +200,7 @@ def create_chat_completion( add_generation_prompt_default = True default_role = "assistant" - if request.model != model.name: + if request.model != model.name and request.model != model_source_name: raise HTTPException(status_code=404, detail=f"Unknown model: {request.model}") if request.n and request.n > 1: @@ -286,7 +289,7 @@ def create_completion( if request.suffix is not None: raise HTTPException(status_code=400, detail="suffix is not currently supported") - if request.model != model.name: + if request.model != model.name and request.model != model_source_name: raise HTTPException(status_code=404, detail=f"Unknown model: {request.model}") if request.prompt is None: @@ -350,7 +353,13 @@ def list_models() -> ListModelsResponse: created=model_create_time, object=ObjectType.model, owned_by=owned_by, - ) + ), + Model( + id=model_source_name, + created=model_create_time, + object=ObjectType.model, + owned_by=owned_by, + ), ] return ListModelsResponse(object=ObjectType.list, data=model_list) @@ -370,6 +379,14 @@ def retrieve_model(model_name: str) -> Model: owned_by=owned_by, ) + if model_name == model_source_name: + return Model( + id=model_source_name, + created=model_create_time, + object=ObjectType.model, + owned_by=owned_by, + ) + raise HTTPException(status_code=404, detail=f"Unknown model: {model_name}") @@ -420,7 +437,7 @@ def parse_args(): print("Loading Model...\n\n", flush=True) - model, model_create_time, backend, tokenizer = load_model(server) + model, model_create_time, backend, tokenizer, model_source_name = load_model(server) if not (model and backend and tokenizer and model_create_time): raise Exception("Unknown Model") From f7181a099c7e85a5efe5f949a10777724c72ba96 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 8 May 2024 22:05:11 -0700 Subject: [PATCH 42/62] matched version in route --- .../examples/fastapi/fastapi-codegen/main.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index 102a57ba..4595a446 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -185,7 +185,7 @@ def create_trtllm_inference_request( @app.post( - "/chat/completions", response_model=CreateChatCompletionResponse, tags=["Chat"] + "/v1/chat/completions", response_model=CreateChatCompletionResponse, tags=["Chat"] ) def create_chat_completion( request: CreateChatCompletionRequest, @@ -275,7 +275,9 @@ def streaming_completion_response(request_id, created, model, responses): yield "data: [DONE]\n\n" -@app.post("/completions", response_model=CreateCompletionResponse, tags=["Completions"]) +@app.post( + "/v1/completions", response_model=CreateCompletionResponse, tags=["Completions"] +) def create_completion( request: CreateCompletionRequest, raw_request: Request ) -> CreateCompletionResponse | StreamingResponse: @@ -341,7 +343,7 @@ def metrics() -> str: return server.metrics() -@app.get("/models", response_model=ListModelsResponse, tags=["Models"]) +@app.get("/v1/models", response_model=ListModelsResponse, tags=["Models"]) def list_models() -> ListModelsResponse: """ Lists the currently available models, and provides basic information about each one such as the owner and availability. @@ -365,7 +367,7 @@ def list_models() -> ListModelsResponse: return ListModelsResponse(object=ObjectType.list, data=model_list) -@app.get("/models/{model_name}", response_model=Model, tags=["Models"]) +@app.get("/v1/models/{model_name}", response_model=Model, tags=["Models"]) def retrieve_model(model_name: str) -> Model: """ Retrieves a model instance, providing basic information about the model such as the owner and permissioning. From e0109e68c0f06fb45c6a43b3d2957494e82baf33 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Thu, 9 May 2024 03:04:10 -0700 Subject: [PATCH 43/62] renaming application --- Triton_Inference_Server_Python_API/docker/Dockerfile | 4 ++-- .../fastapi-codegen/{main.py => openai-tritonserver.py} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/{main.py => openai-tritonserver.py} (100%) diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index 5aebb0bb..afb5d00b 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -55,11 +55,11 @@ RUN find /opt/tritonserver/python -maxdepth 1 -type f -name \ RUN pip3 install --force-reinstall --upgrade /tmp/tritonserver-2.46.0.dev0-py3-none-any.whl[all] -ARG TRITON_CLI_TAG="0.0.7" +ARG TRITON_CLI_TAG="nnshah1-default-echo" RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${TRITON_CLI_TAG} -ARG GENAI_PERF_TAG=r24.04 +ARG GENAI_PERF_TAG=tgerdes-request-count-for-izzy RUN pip install "git+https://github.com/triton-inference-server/client.git@${GENAI_PERF_TAG}#subdirectory=src/c++/perf_analyzer/genai-perf" diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py rename to Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py From 20e14a9f5a35c7ce905bac52b3c95bfbd91fc997 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Thu, 9 May 2024 14:54:52 -0700 Subject: [PATCH 44/62] adding developer tool --- Triton_Inference_Server_Python_API/docker/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index afb5d00b..24acb643 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -83,3 +83,6 @@ RUN if [[ "$RUN_TESTS" == "TRUE" ]] ; then cd /tmp && git clone -b r24.04 https: RUN if [[ "$RUN_TESTS" == "TRUE" ]] ; then pytest /workspace/deps ; fi +ARG INCLUDE_EMACS + +RUN if [[ "$INCLUDE_EMACS" == "TRUE" ]] ; apt-get install -y emacs ; fi From 7237a79590f94fbde61ca5cd7085b8573554e01c Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Thu, 9 May 2024 15:06:25 -0700 Subject: [PATCH 45/62] update typo --- Triton_Inference_Server_Python_API/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index 24acb643..2a9ad18d 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -85,4 +85,4 @@ RUN if [[ "$RUN_TESTS" == "TRUE" ]] ; then pytest /workspace/deps ; fi ARG INCLUDE_EMACS -RUN if [[ "$INCLUDE_EMACS" == "TRUE" ]] ; apt-get install -y emacs ; fi +RUN if [[ "$INCLUDE_EMACS" == "TRUE" ]] ; then apt-get install -y emacs ; fi From 31a576529676cb9761c5adf17ca93e278411a927 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Thu, 9 May 2024 15:11:03 -0700 Subject: [PATCH 46/62] updated --- Triton_Inference_Server_Python_API/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index 2a9ad18d..144c0197 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -85,4 +85,4 @@ RUN if [[ "$RUN_TESTS" == "TRUE" ]] ; then pytest /workspace/deps ; fi ARG INCLUDE_EMACS -RUN if [[ "$INCLUDE_EMACS" == "TRUE" ]] ; then apt-get install -y emacs ; fi +RUN if [[ "$INCLUDE_EMACS" == "TRUE" ]] ; then apt-get install -y emacs && stty erase ^H ; fi From afec8515be7dd43c929370b00f2db3d503c97847 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Thu, 9 May 2024 15:17:31 -0700 Subject: [PATCH 47/62] updated --- Triton_Inference_Server_Python_API/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index 144c0197..2a9ad18d 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -85,4 +85,4 @@ RUN if [[ "$RUN_TESTS" == "TRUE" ]] ; then pytest /workspace/deps ; fi ARG INCLUDE_EMACS -RUN if [[ "$INCLUDE_EMACS" == "TRUE" ]] ; then apt-get install -y emacs && stty erase ^H ; fi +RUN if [[ "$INCLUDE_EMACS" == "TRUE" ]] ; then apt-get install -y emacs ; fi From cc72ee123e0d9833a82c93ff3107b106406585bb Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 14 May 2024 04:13:01 -0700 Subject: [PATCH 48/62] updated tag for genaiperf --- Triton_Inference_Server_Python_API/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index afb5d00b..dc9fe417 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -59,7 +59,7 @@ ARG TRITON_CLI_TAG="nnshah1-default-echo" RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${TRITON_CLI_TAG} -ARG GENAI_PERF_TAG=tgerdes-request-count-for-izzy +ARG GENAI_PERF_TAG="24.05" RUN pip install "git+https://github.com/triton-inference-server/client.git@${GENAI_PERF_TAG}#subdirectory=src/c++/perf_analyzer/genai-perf" From f1ed67f0abbee71a06a250d23750a9c0ba0dbcc8 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 14 May 2024 07:07:05 -0700 Subject: [PATCH 49/62] updated readme with known issues / limitations --- .../docker/Dockerfile | 2 +- .../examples/fastapi/README.md | 181 ++++++++++++------ .../fastapi-codegen/openai-tritonserver.py | 4 +- 3 files changed, 127 insertions(+), 60 deletions(-) diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index 243a28b4..28f0dc76 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -59,7 +59,7 @@ ARG TRITON_CLI_TAG="nnshah1-default-echo" RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${TRITON_CLI_TAG} -ARG GENAI_PERF_TAG="24.05" +ARG GENAI_PERF_TAG="r24.05" RUN pip install "git+https://github.com/triton-inference-server/client.git@${GENAI_PERF_TAG}#subdirectory=src/c++/perf_analyzer/genai-perf" diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/README.md b/Triton_Inference_Server_Python_API/examples/fastapi/README.md index e78e0750..824ec836 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/README.md +++ b/Triton_Inference_Server_Python_API/examples/fastapi/README.md @@ -26,107 +26,174 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> -# Triton Inference Server Fast API / Open API / Open AI Example +# Triton Inference Server Open AI Compatible Server -## Build Image -``` -../../build.sh --framework vllm --build-arg TRITON_CLI_TAG=rmccormick-trtllm-0.9 -``` +Using the Triton In-Process Python API you can integrat triton server +based models into any Python framework including FastAPI with an +OpenAI compatible interface. -## Import Model -``` -triton remove -m all --model-repository llm-models -triton import -m llama-3-8b-instruct --backend vllm --model-repository llm-models -``` +This directory contains a FastAPI based Triton Inference Server +supporing `llama-3-8b-instruct` with both the vLLM and TRT-LLM +backends. -## Open AI API Specification +The front end application was generated using a trimmed version of the +OpenAI OpenAPI [specification](api-spec/openai_trimmed.yml) and the +tool [`fastapi-codegen`](scripts/openai_trimmed.yml). -We use -https://raw.githubusercontent.com/openai/openai-openapi/25d9dacc86a94df1db98725fe87494564317cafa/openapi.yaml -as the base specification. +## Installation -As this tutorial only covers LLM applications we use a trimmed specficiation (api-spec/openai_trimmed.yml). - -## Generating the Fast API server using fastapi-codegen +The following instructions assume you have a huggingface token set in +the environment variable `HF_TOKEN`. +### Clone Repository ``` -./scripts/fastapi-codegen.sh "-i api-spec/openai_trimmed.yml -o fastapi-codegen --model-file openai_protocol_types" +git clone https://github.com/triton-inference-server/tutorials.git -b nnshah1-meetup-04-2024 +cd tutorials/Triton_Inference_Server_Python_API/examples/fastapi ``` +## Triton + vLLM -### Modifications +### Build and Run Image +``` +export HF_TOKEN= +../../build.sh --framework vllm +../../run.sh --framework vllm +cd examples/fastapi +``` -1. Remove relative import +### Import Model -Before: +Note: Model import only has to be done the first time running the server. ``` -from .openapi_protocol_types +triton remove -m all --model-repository llama-3-8b-instruct-vllm +triton import -m llama-3-8b-instruct --backend vllm --model-repository llama-3-8b-instruct-vllm ``` -After: +### Run Server + ``` -from openapi_protocol_types +python3 fastapi-codegen/openai-tritonserver.py --model-repository llama-3-8b-instruct-vllm ``` +## Triton + TRT-LLM -## Generating the Fast API server using openapi-code-generator - - -## curl examples +### Build and Run Image +``` +export HF_TOKEN= +../../build.sh --framework trt_llm +../../run.sh --framework trt_llm +cd examples/fastapi +``` -### Models +### Import Model -#### List +Note: Model import only has to be done the first time running the server. ``` -curl -s http://localhost:8000/models | jq . +triton remove -m all --model-repository llama-3-8b-instruct-trt-llm +triton import -m llama-3-8b-instruct --backend tensorrtllm --model-repository llama-3-8b-instruct-trt-llm ``` +### Run Server + ``` -{ - "object": "list", - "data": [ - { - "id": "llama-3-8b-instruct", - "created": 1714952401, - "object": "model", - "owned_by": "ACME" - } - ] +python3 fastapi-codegen/openai-tritonserver.py --model-repository llama-3-8b-instruct-trt-llm ``` -#### Retrieve Model Info + +## Send OpenAI API Requests + +#### Completions `/v1/completions` ``` -curl -s http://localhost:8000/models/llama-3-8b-instruct | jq . +curl -X 'POST' \ + 'http://0.0.0.0:8000/v1/completions' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "llama-3-8b-instruct", + "prompt": "Once upon a time", + "max_tokens": 16, + "top_p": 1, + "n": 1, + "stream": false, + "stop": "string", + "frequency_penalty": 0.0 + }' | jq . ``` +#### Chat Completions `/v1/chat/completions` + ``` -{ - "id": "llama-3-8b-instruct", - "created": 1714953302, - "object": "model", - "owned_by": "ACME" -} +curl -X 'POST' \ +'http://0.0.0.0:8000/v1/chat/completions' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "llama-3-8b-instruct", + "messages": [ + { + "role":"user", + "content":"Hello there how are you?" + }, + { + "role":"assistant", + "content":"Good and you?" + }, + { + "role":"user", + "content":"Whats your name?" + } + ], + "max_tokens": 16, + "top_p": 1, + "n": 1, + "stream": false, + "stop": "string", + "frequency_penalty": 0.0 + }' | jq . ``` +#### Model List -#### Completion +``` +curl -s http://localhost:8000/v1/models | jq . +``` -### Comparison +#### Model Info -curl http://localhost:8000/v1/completions -H "Content-Type: application/json" -d '{"model":"meta-llama/Meta-Llama-3-8B-Instruct","prompt":"say this is a test, but is it?","seed":800}' +``` +curl -s http://localhost:8000/v1/models/llama-3-8b-instruct | jq . +``` +## Comparison to vllm +The vLLM container can also be used to run the vLLM FastAPI Server -#### chat completion +### Run the vLLM Open AI Server +``` +python3 -m vllm.entrypoints.openai.api_server --model "meta-llama/Meta-Llama-3-8B-Instruct" --disable-log-requests +``` -curl http://localhost:8000/chat/completions -H "Content-Type: application/json" -d '{"model":"llama-3-8b-instruct","messages":[{"role":"system","content":"you are a helpful assistant."},{"role":"user","content":"Hello!"}]}' +``` +curl http://localhost:8000/v1/chat/completions -H "Content-Type: application/json" -d '{"model":"meta-llama/Meta-Llama-3-8B-Instruct","messages":[{"role":"system","content":"you are a helpful assistant."},{"role":"user","content":"Hello!"}]}' | jq . +``` +## Running GenAI Perf -## Comparison +Note: the following command requires the 24.05 pre-release version of genai-perf. -python3 -m vllm.entrypoints.openai.api_server --model "meta-llama/Meta-Llama-3-8B-Instruct" +Preliminary results show performance is on par with vLLM with concurrency 2 -curl http://localhost:8000/v1/chat/completions -H "Content-Type: application/json" -d '{"model":"meta-llama/Meta-Llama-3-8B-Instruct","messages":[{"role":"system","content":"you are a helpful assistant."},{"role":"user","content":"Hello!"}]}' +``` +genai-perf -m meta-llama/Meta-Llama-3-8B-Instruct --endpoint v1/chat/completions --endpoint-type chat --service-kind openai -u http://localhost:8000 --num-prompts 100 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 50 --concurrency 2 --measurement-interval 40000 --extra-inputs max_tokens:512 --extra-input ignore_eos:true -- -v --max-threads=256 +erval 40000 --extra-inputs max_tokens:512 --extra-input ignore_eos:true -- -v --max-threads=256 +``` +## Known Limitations +* Concurrency leads to data corruption +* Max tokens is not processed by trt-llm backend correctly +* Usage information is not populated +* `finish_reason` is currently always set to `stop` +* Limited performance testing has been done +* Using genai-perf to test streaming requires changes to genai-perf SSE handling diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py index 4595a446..58d2ed66 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py @@ -169,11 +169,11 @@ def create_trtllm_inference_request( if request.stop: if isinstance(request.stop, str): request.stop = [request.stop] - inputs["stop_words"] = request.stop + inputs["stop_words"] = [request.stop] if request.top_p: inputs["top_p"] = [[numpy.float32(request.top_p)]] if request.frequency_penalty: - inputs["frequence_penalty"] = [[numpy.int32(request.frequency_penalty)]] + inputs["frequency_penalty"] = [[numpy.float32(request.frequency_penalty)]] if request.presence_penalty: inputs["presence_penalty":] = [[numpy.int32(request.presence_penalty)]] if request.seed: From be5bb9da29e7e531965b147dc841d06ea59890da Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 19 May 2024 08:36:00 -0700 Subject: [PATCH 50/62] updating with pytorch base example --- Triton_Inference_Server_Python_API/build.sh | 21 +++- .../docker/Dockerfile.dockerignore | 8 +- .../docker/Dockerfile.pytorch | 103 ++++++++++++++++++ .../docker/Dockerfile.pytorch.dockerignore | 19 ++++ Triton_Inference_Server_Python_API/run.sh | 6 +- .../pytorch/replace_duplicate_libraries.py | 84 ++++++++++++++ 6 files changed, 236 insertions(+), 5 deletions(-) create mode 100644 Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch create mode 100644 Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch.dockerignore create mode 100644 Triton_Inference_Server_Python_API/scripts/pytorch/replace_duplicate_libraries.py diff --git a/Triton_Inference_Server_Python_API/build.sh b/Triton_Inference_Server_Python_API/build.sh index ad507515..cd6b9ad2 100755 --- a/Triton_Inference_Server_Python_API/build.sh +++ b/Triton_Inference_Server_Python_API/build.sh @@ -30,7 +30,7 @@ RUN_PREFIX= BUILD_MODELS= # Frameworks -declare -A FRAMEWORKS=(["DIFFUSION"]=1 ["TRT_LLM"]=2 ["IDENTITY"]=3 ["VLLM"]=4) +declare -A FRAMEWORKS=(["DIFFUSION"]=1 ["TRT_LLM"]=2 ["IDENTITY"]=3 ["VLLM"]=4 ["PYTORCH"]=5) DEFAULT_FRAMEWORK=IDENTITY SOURCE_DIR=$(dirname "$(readlink -f "$0")") @@ -38,12 +38,13 @@ DOCKERFILE=${SOURCE_DIR}/docker/Dockerfile # Base Images -BASE_IMAGE=nvcr.io/nvidia/tritonserver +BASE_IMAGE_DEFAULT=nvcr.io/nvidia/tritonserver BASE_IMAGE_TAG_IDENTITY=24.04-py3 BASE_IMAGE_TAG_DIFFUSION=24.04-py3 BASE_IMAGE_TAG_TRT_LLM=24.04-trtllm-python-py3 BASE_IMAGE_TAG_VLLM=24.04-vllm-python-py3 - +BASE_IMAGE_PYTORCH=nvcr.io/nvidia/pytorch +BASE_IMAGE_TAG_PYTORCH=24.04-py3 get_options() { while :; do @@ -137,6 +138,16 @@ get_options() { BASE_IMAGE_TAG=BASE_IMAGE_TAG_${FRAMEWORK} BASE_IMAGE_TAG=${!BASE_IMAGE_TAG} fi + + if [ -z $BASE_IMAGE ]; then + BASE_IMAGE=BASE_IMAGE_${FRAMEWORK} + BASE_IMAGE=${!BASE_IMAGE} + fi + + if [ -z $BASE_IMAGE ]; then + BASE_IMAGE=${BASE_IMAGE_DEFAULT} + fi + fi if [ -z "$TAG" ]; then @@ -154,6 +165,10 @@ get_options() { TAG+="-vllm" fi + if [[ $FRAMEWORK == "PYTORCH" ]]; then + TAG+="-pytorch" + DOCKERFILE=${SOURCE_DIR}/docker/Dockerfile.pytorch + fi fi diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile.dockerignore b/Triton_Inference_Server_Python_API/docker/Dockerfile.dockerignore index 09b6c427..37d21ab4 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile.dockerignore +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile.dockerignore @@ -10,4 +10,10 @@ **/*onnx* **/*engine* **/*pytorch_model* -**/*.pth* \ No newline at end of file +**/*.pth* +**/*.pt +**/*.models/* +**/*.model-store/* +**/*.model.*/* +**/*.cache/* +**/*.libtorch_model_store/* \ No newline at end of file diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch b/Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch new file mode 100644 index 00000000..00bee50f --- /dev/null +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch @@ -0,0 +1,103 @@ +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ARG BASE_IMAGE=nvcr.io/nvidia/pytorch +ARG BASE_IMAGE_TAG=24.04-py3 +ARG FRAMEWORK=PYTORCH +ARG GENAI_PERF_TAG=r24.04 +ARG TRITON_CLI_TAG="0.0.6" + +FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} as triton-python-api + +ARG TRITONSERVER_BASE_IMAGE=nvcr.io/nvidia/tritonserver +ARG TRITONSERVER_BASE_IMAGE_TAG=24.04-py3 + +COPY --from=nvcr.io/nvidia/tritonserver:24.04-py3 /opt/tritonserver /opt/tritonserver + +RUN distribution=$(. /etc/os-release;echo $ID$VERSION_ID | sed -e 's/\.//g') && \ + wget https://developer.download.nvidia.com/compute/cuda/repos/$distribution/x86_64/cuda-keyring_1.1-1_all.deb && \ + dpkg -i cuda-keyring_1.1-1_all.deb && \ + rm -rf cuda-keyring_1.1-1_all.deb + +RUN apt-get update; apt-get install -y gdb datacenter-gpu-manager libxml2 libb64-dev + +COPY ./scripts/pytorch/replace_duplicate_libraries.py /tmp/replace_duplicate_libraries.py + +RUN python3 /tmp/replace_duplicate_libraries.py > /opt/tritonserver/backends/pytorch/sym_linked.txt + +COPY ./deps/requirements.txt /tmp/requirements.txt + +COPY ./deps/requirements_trt_llm.txt /tmp/requirements_trt_llm.txt + +COPY ./deps/requirements_vllm.txt /tmp/requirements_vllm.txt + +COPY ./deps/requirements_python_repl.txt /tmp/requirements_python_repl.txt + +COPY ./deps/tritonserver-2.46.0.dev0-py3-none-any.whl /tmp/tritonserver-2.46.0.dev0-py3-none-any.whl + +RUN pip install --timeout=2000 -r /tmp/requirements.txt + +# Finish pyright install + +RUN pyright --help + +RUN find /opt/tritonserver/python -maxdepth 1 -type f -name \ + "tritonserver-*.whl" | xargs -I {} pip3 install --force-reinstall --upgrade {}[all] + +RUN pip3 install --force-reinstall --upgrade /tmp/tritonserver-2.46.0.dev0-py3-none-any.whl[all] + +ARG TRITON_CLI_TAG="nnshah1-default-echo" + +RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${TRITON_CLI_TAG} + +ARG GENAI_PERF_TAG=tgerdes-request-count-for-izzy + +RUN pip install "git+https://github.com/triton-inference-server/client.git@${GENAI_PERF_TAG}#subdirectory=src/c++/perf_analyzer/genai-perf" + +ARG INCLUDE_PYTHON_REPL + +RUN if [[ "$INCLUDE_PYTHON_REPL" != "" ]] ; then pip install --timeout=2000 -r /tmp/requirements_python_repl.txt ; fi + +ARG FRAMEWORK=DIFFUSION + +RUN if [[ "$FRAMEWORK" == "TRT_LLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_trt_llm.txt ; fi + +RUN if [[ "$FRAMEWORK" == "VLLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_vllm.txt ; fi + +RUN ln -sf /bin/bash /bin/sh + +COPY . /workspace + +ARG RUN_TESTS=FALSE + +RUN if [[ "$RUN_TESTS" == "TRUE" ]] ; then cd /tmp && git clone -b r24.04 https://github.com/triton-inference-server/core.git && cp -rf /tmp/core/python/test /workspace/deps/ ; fi + +RUN if [[ "$RUN_TESTS" == "TRUE" ]] ; then pytest /workspace/deps ; fi + +ARG INCLUDE_EMACS + +RUN if [[ "$INCLUDE_EMACS" == "TRUE" ]] ; then apt-get install -y emacs ; fi + diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch.dockerignore b/Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch.dockerignore new file mode 100644 index 00000000..37d21ab4 --- /dev/null +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch.dockerignore @@ -0,0 +1,19 @@ +**/*.onnx +**/*.plan +**/diffuser-models/* +**/identity-models/* +**/scripts/stable_diffusion/models/*/*/*.onnx +**/scripts/stable_diffusion/models/*/*/*.plan +**/*.onnx +**/*.plan +**/.cache/* +**/*onnx* +**/*engine* +**/*pytorch_model* +**/*.pth* +**/*.pt +**/*.models/* +**/*.model-store/* +**/*.model.*/* +**/*.cache/* +**/*.libtorch_model_store/* \ No newline at end of file diff --git a/Triton_Inference_Server_Python_API/run.sh b/Triton_Inference_Server_Python_API/run.sh index efa95797..9d27f38a 100755 --- a/Triton_Inference_Server_Python_API/run.sh +++ b/Triton_Inference_Server_Python_API/run.sh @@ -29,7 +29,7 @@ TAG= RUN_PREFIX= # Frameworks -declare -A FRAMEWORKS=(["DIFFUSION"]=1 ["TRT_LLM"]=2 ["IDENTITY"]=3 ["VLLM"]=4) +declare -A FRAMEWORKS=(["DIFFUSION"]=1 ["TRT_LLM"]=2 ["IDENTITY"]=3 ["VLLM"]=4 ["PYTORCH"]=5) DEFAULT_FRAMEWORK=IDENTITY SOURCE_DIR=$(dirname "$(readlink -f "$0")") @@ -39,6 +39,7 @@ IMAGE= IMAGE_TAG_DIFFUSERS=diffusion IMAGE_TAG_TRT_LLM=trt-llm IMAGE_TAG_VLLM=vllm +IMAGE_TAG_PYTORCH=pytorch get_options() { while :; do @@ -115,6 +116,9 @@ get_options() { IMAGE+="-vllm" fi + if [[ $FRAMEWORK == "PYTORCH" ]]; then + IMAGE+="-pytorch" + fi fi } diff --git a/Triton_Inference_Server_Python_API/scripts/pytorch/replace_duplicate_libraries.py b/Triton_Inference_Server_Python_API/scripts/pytorch/replace_duplicate_libraries.py new file mode 100644 index 00000000..0d0fba5e --- /dev/null +++ b/Triton_Inference_Server_Python_API/scripts/pytorch/replace_duplicate_libraries.py @@ -0,0 +1,84 @@ +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import hashlib +import os +import pathlib +import shutil + +backends = ["pytorch"] +exclude = ["model.py"] + +source_path = pathlib.Path("/usr") + + +def hash_file(file_path): + hash_md5 = hashlib.md5() + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + +def get_candidate(file_path): + candidates = list(source_path.rglob(file_path.name)) + if not candidates: + return None + if len(candidates) == 1: + return candidates[0] + file_hash = hash_file(file_path) + for candidate in candidates: + candidate_file_hash = hash_file(candidate) + if file_hash == candidate_file_hash: + return candidate + + +if __name__ == "__main__": + for backend in backends: + backend_path = pathlib.Path(f"/opt/tritonserver/backends/{backend}") + + for file_path in backend_path.glob("*"): + if file_path.name in exclude: + continue + candidate = get_candidate(file_path) + + if candidate: + print(f"replacing {file_path} with symlink to {candidate}") + file_path.unlink() + file_path = pathlib.Path(file_path) + file_path.symlink_to(candidate) + + +# print(file_path.name) +# print(file_path.stat()) +# print(hash_file(file_path)) + +# files = os.listdir("/opt/tritonserver/backends/pytorch") +# print(files) + +# print(list(pathlib.Path("/opt/tritonserver/backends/pytorch").glob("*"))) + +# print(list(pathlib.Path("/usr").rglob(files[1]))) From cc4d8860584ce15905959b8b1ff666b6336a8812 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 19 May 2024 16:57:11 -0700 Subject: [PATCH 51/62] updating genai perf tag --- Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch b/Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch index 00bee50f..560cf2dd 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch @@ -73,7 +73,7 @@ ARG TRITON_CLI_TAG="nnshah1-default-echo" RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${TRITON_CLI_TAG} -ARG GENAI_PERF_TAG=tgerdes-request-count-for-izzy +ARG GENAI_PERF_TAG="r24.05" RUN pip install "git+https://github.com/triton-inference-server/client.git@${GENAI_PERF_TAG}#subdirectory=src/c++/perf_analyzer/genai-perf" From 41ef1406a20c101a4e05d39484e32cbf246741de Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 22 May 2024 06:25:39 -0700 Subject: [PATCH 52/62] updated to build pytorch without torchvision --- .../docker/Dockerfile.pytorch | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch b/Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch index 560cf2dd..425e8520 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile.pytorch @@ -37,6 +37,15 @@ ARG TRITONSERVER_BASE_IMAGE_TAG=24.04-py3 COPY --from=nvcr.io/nvidia/tritonserver:24.04-py3 /opt/tritonserver /opt/tritonserver +# Build pytorch without torchvision support +RUN cd /tmp && \ + git clone https://github.com/triton-inference-server/pytorch_backend.git -b r24.04 && \ + cd pytorch_backend && mkdir build && cd build && \ + cmake -DCMAKE_INSTALL_PREFIX:PATH=$PWD/install -DTRITON_PYTORCH_INCLUDE_PATHS="/opt/pytorch/pytorch;/opt/pytorch/pytorch/torch/torch/csrc/api/include;/opt/pytorch/torchvision;/usr/local/lib/python3.10/dist-packages/torch/include" -DTRITON_PYTORCH_LIB_PATHS="/usr/local/lib/python3.10/dist-packages/torch/lib" -DTRITON_PYTORCH_ENABLE_TORCHVISION=OFF .. && \ + make install + +RUN cp /tmp/pytorch_backend/build/install/backends/pytorch/libtriton_pytorch.* /opt/tritonserver/backends/pytorch + RUN distribution=$(. /etc/os-release;echo $ID$VERSION_ID | sed -e 's/\.//g') && \ wget https://developer.download.nvidia.com/compute/cuda/repos/$distribution/x86_64/cuda-keyring_1.1-1_all.deb && \ dpkg -i cuda-keyring_1.1-1_all.deb && \ From 3573aaa5b5b491bf7be7ca2b2edb3130f0abd360 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Mon, 3 Jun 2024 22:04:01 -0700 Subject: [PATCH 53/62] upgrade to 24.05 --- Triton_Inference_Server_Python_API/build.sh | 14 +++++++------- .../tritonserver-2.46.0.dev0-py3-none-any.whl | Bin 258285 -> 258314 bytes .../docker/Dockerfile | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Triton_Inference_Server_Python_API/build.sh b/Triton_Inference_Server_Python_API/build.sh index cd6b9ad2..ae3e2e27 100755 --- a/Triton_Inference_Server_Python_API/build.sh +++ b/Triton_Inference_Server_Python_API/build.sh @@ -39,12 +39,12 @@ DOCKERFILE=${SOURCE_DIR}/docker/Dockerfile # Base Images BASE_IMAGE_DEFAULT=nvcr.io/nvidia/tritonserver -BASE_IMAGE_TAG_IDENTITY=24.04-py3 -BASE_IMAGE_TAG_DIFFUSION=24.04-py3 -BASE_IMAGE_TAG_TRT_LLM=24.04-trtllm-python-py3 -BASE_IMAGE_TAG_VLLM=24.04-vllm-python-py3 +BASE_IMAGE_TAG_IDENTITY=24.05-py3 +BASE_IMAGE_TAG_DIFFUSION=24.05-py3 +BASE_IMAGE_TAG_TRT_LLM=24.05-trtllm-python-py3 +BASE_IMAGE_TAG_VLLM=24.05-vllm-python-py3 BASE_IMAGE_PYTORCH=nvcr.io/nvidia/pytorch -BASE_IMAGE_TAG_PYTORCH=24.04-py3 +BASE_IMAGE_TAG_PYTORCH=24.05-py3 get_options() { while :; do @@ -151,7 +151,7 @@ get_options() { fi if [ -z "$TAG" ]; then - TAG="triton-python-api:r24.04" + TAG="triton-python-api:r24.05" if [[ $FRAMEWORK == "TRT_LLM" ]]; then TAG+="-trt-llm" @@ -208,7 +208,7 @@ get_options "$@" if [[ $FRAMEWORK == DIFFUSION ]]; then BASE_IMAGE="tritonserver" - BASE_IMAGE_TAG="r24.04-diffusion" + BASE_IMAGE_TAG="r24.05-diffusion" fi # BUILD RUN TIME IMAGE diff --git a/Triton_Inference_Server_Python_API/deps/tritonserver-2.46.0.dev0-py3-none-any.whl b/Triton_Inference_Server_Python_API/deps/tritonserver-2.46.0.dev0-py3-none-any.whl index 1f2fdd490698182bbdc44c4b6779f19cf4377b4e..6cd844097c4f75581d0f33ce760fdd51fbf9a934 100644 GIT binary patch delta 6600 zcmZu$1x#H{x4m$2hvHToiWPU4;_j}+DeiLd;uO8bp*R$$I23nxmy0_TxAJI{|NH)z z_a>P$Gi&XAcFrWTlC$Py6Fm}wj;bsNguw;?00@8^ik@mzO(5`fthE42!T+69A+4fd zdQ|f$;3_x0WkU@5T$k(&k_#w%Z-CsBc8Uo{R^5-nPh%7c73f; zXaT3x>7E*;;InEQ_d=AK)a8ro+^|swU)l?JyzQyCMEri43CgJDez|{heRq8i@F0p{ix8 ztsbiUCWGXDG1Xntd+s*Dji}!Z;X#_5x6c$ZQx67uH?KC!M3S^exI=n2iR?e?vX6k` zI!M$2UPtvn&u`iOr%gu#XX0Er1m}cX}J# zPOZ3cTftYZxp}TE*y8@;!zpyd;ym9>!ZR@Uvib|#W7WWQ6@WTVdtc@GO?*64%uvzu zGated>C4-VM`qnm5B&|{pMz#oK;1>>8r4hVV0$q29`?9-^-Eb z1ho|QHg3V4mIGc`$=Gv39~)E`HB+ubZ2cd3zQbB();EjprmJOg#>eWJ`L7OwnsLfb z)bj-KvW;ooxqCw!dAgavk;KM_Z1pvX;O=j*yxq--WE{VUNW(+hv<#W>P2RQSpihNC z?9sV}(OrGOK0NCaMJ)q9@({YaOprz4GO)tx)Ut%*Ce(lFLqSM=cj~lW-f=g_50M+T z_2j0(?u4V_<(w(RB80Z!I&Vi%o52M&efF_oKXZtnN4mv#eH?9JW7S5wY+|-?8Q$9Z zH5~4sJnS0*Hwz+e=eJeaN@%vuQZ;XP-(0KPOTMtuH)uQqXGh9p?lD9BTlrewyC>OZ zB@e_o(f283)m{}rQ{C))9eABR3zBTe#DEyo(w^~y3OM+kBSODMVl|V1@{(*qO=s?# z%ua*iBtD?9HZOWiZFofgb{#9w6n8MD~jh|@gSHLwj)s#B~jY>&>TpYhOhiGrZr}tr8YPh-06yvJ4-msBd(2&?q&$CwVKJs}zv|cam}OPm@SpFyWWMWDY=#otp!swa@R>_wL>NQXy#Wn-$L^izRQ-xQ5urJDRhz9vpyg5DW=j4O(2k#4tdJuU6yuS#Ax%#AJe zfuR@Ze`WBG*jXAL)V{AJri?}a&DBUPJSn7uC`!{X8VbFfT-3yUbkP@#}$b6DtGvK`T_DbtIAOL+Do&@hjc*4>w zgz~PoV~sBoO%cYl@E2nGyLIyFME~%*VhYmHMH}yGod&(^VO#2blxUhHffSLXl?XLY zAIu_Xw4j5HQDLxw&YD+y0WCcEFmYZPE_gW7lozGi{HmKP@n9TQ7$aZH0F8Xpm1m)H zJ~+kjYMP>igE|QUG!RZ}PrRjo=7qypq<9{!btkMBWCNT*3TX;56Scpw-7633l}Sd3 z3o|5DZKNz7Z1vS~5`*>~OlMUs4i6wQr4qusXQUP;7kWDmx4o6xL%ePS$I15-JhdNx z{iWg)%mUNL-`G#`{*5c#kZAO@Gqup zJro|yHr8xbQ4fz4hA^0LNKaQ(^Xxe@%Hc=IUPaZjvySLi5Rc{`b{u4Zr91D%QEaUya1S- z6pvY7e_usXM^{=}>;i8njfI)kxByd)ELq9RAHAE)s)H>ac(z$7@TV7^B_~gK|DFag z_z;L)+r)f}gX-~mFv>5b4%UzWfQ-a833-sM!}0*mbA=JBV?c$zW7DE*JUTsbQi@f2 zF{8B%eT4}=ib9O3`@4jlZXM3mmo=9?QPmmBxNmT#i5WiQE4OXzUcG%f?M#rkaU7q^ zhfyQ{o=PJWE}t`b;;5LtQ`1x3cJ}OQVb-I=p;W#U2OpZ+gb>7v1f16GUSy-xU@cIv z*2qp-TSQ*CRDzPosxOWa)M| zm0NUF2*IJSmb~w|oHQpgnbeG0v$PuRt&8IY*>ig$alKx`xHSZBoev=)~t&2WHnHd$X5kT zCmQ>ZJ3Ud^kZc$oGY#E>k-1sgNGKf5?;~CkDzIZ;zm?~B%6>FYE>MRd`yI2z>fpb# zu2V63-s-rg+?`c5?8CS;Vc~SfH_t89lBz^oe^ibBmJvNk_^IRjBFxJvl-Y~f=2BMw zF0;!gJT>?yfNOc`*+N!l#-b;vIfJLxHrRmKk1PGr?r?=qXyBkGXIUM(^DySg`$Io~F@>ewl6}eo;<;6_v$FIiQ542*`1tl_ z#Bwc$GxMe{eSOpp?-$TaAp5=oD7rUsGZj`mw=Mnp>xmu~@y&S(IP2_AZrx)3 z@?PTWO6}_H*uu^4p+XlWZTo{oxSMk(@lqHJd{mg6c75+ymTdM*`*{OqI@R&bHGXY) zXIT=mStfzCI$!0$iQe3|4u;r)3L{36(!0+aNBJ1=XSNS;N-q^*l%iZWjct^b6y3Va#J9cDZ3t1|*8Lb`arsl3btL?8f@=Y9GicS~x z6@=*+hosHX&W+<^yNr;JG6mtu(!8ZaPR+ZpN-EJvzbn(g(mdOsytr-AHpeu{I9-v> zPbzYWSR%oJ13hHg^BM-{WX2aX?za6cH5IK6XzCNFbDvOh=K+c4j2nc#e&dSer+ zYcE7#M{;RSjaX*SoS)30*&LecvP3PPM3;=AQ&-xB(j_q0(-{^bBo<;+oU<>Xo`l`a zv~J!zH? zz>?e5>Syp}r}23M4GA2C`-sSb1A~Zynl?=p1s{KDDpHk3{?=tBNnY`Ve>m&+V9B1U zQ4A94l!zu6hLSD2yc>a_8fWEeTUM=xHQP(7gQQ2%CcUD|6-0n>;)T^{KtlKn)^k)8pc612`j=jF4H!(%*oEu?`PzFjSIy5dB zzO5BNUKX*+${f2=m`hldKle=fkg0tlu1+i?MMK&mxVn}OSJUQ05wMR&@fxd#A>eT8 z8o(_~h+#`IC(~=8Gn8JQVvT|p)`a3OLCqLKNb9Z*q9uoRm%yA6YyWg!(ZklunTkm| z>dY(W55+YU8ARLlOK_9iBtj{$rAK}^#mn_JYl-6l z6sYrw4{?NqOp3A*qD(3+C@vo?u!}r&*=&zJw@({?@8PBp++MfPKIALneou0FexgWD z%gW~D=D5Hl=)pl3qjyD`d9UCjl}-O+&xZox(y04sp>Ypm)y}bXJL$5h>&=QuNg@;x zNXY*2^bDcATZ9`}Vovs9^Er)PD4jGcF62daJS;0$;ljIaE>gopMf64A+*%WSOu$Kv z$qDl7n4u={TJIPWGBMk9{EGF*+G?P%91?)_ zZdFIRcXs4j23b#G)~c$V-k*74NjW9kQBSsb zoTdFhF5H#e+}_j)jkDY<%m716atfT)Kc8)~e)RC-(c2ULiP|%~NEtBYo#-%HMb~w5 zzPV|6dA+sOM391=DBeVK_%!Z7y)-5-Iq&M@4C6o+fe@9u^x&I+5{{dpNr(#s)$%o> zaDIFDMqKXQ`WWf>&KlL)))pdr;(Ck4ZyFOKBRO#c-&0+iD^&LcgDQ;&(1f>?ytw9_ii}}2uyWw&gT*X3{HX*|D+q@Y7KU! z0WY{>j)?%pjFHc!FtCT}j-pq4ptkD#s34<&FOW%zAYo^!d!(LcXkTxE{Zjw@&zV*x z2B)j0t0qfj1{0PI@*XBZsw?$h^h0F^(`7*&bb%vrlew1%P;@w1 z8>K;g;(>hqf&MSQzl~Pc-kPXh*19Y6sP&iK5YOs7tq4q_a}Z|=t)){XfQapN2ct*6!hckDYJa>|>x%1C-u6dDnk4DE6BnYl>}X;{_!W%=AjW~|PM|$fWcsG1 z#;aS0**b*vO~~C*8J*|DRADwob>|gBmIOhkk-W4d8T#XDjnq5}6X8;0q&jTfJc)T? zSP+~~?)evLB4}BzH_!fRXdwQpu6XD6SUJCFQY6DnprFlz3bz4XWEYBrLJCA~_mYG( zR$Dlbg6Dx^P$N-M)48>!(W;_)WY}l@<_6Pf@THmVD5#do(1pH`!!#nRR%(^UxdBPw z%QM|Y0Sv#LH~8&{ds68vGp=MF$SGS6rOdf{B?z-sRD19xV{^w* z$+`Lf(?^#4S1FHd&QQflKg!I&bektXS@%7&KDUm0>HNXOQ;fr7{Gu~y>{y$+kUrDf z9(;1MGp7YA?G_U)h&VmB*_~pIdx$k{e`0~fyxyv*iK}gBbov-P#>7hm=ZZZXIRT8p zULQTPR0n^ zddW3AVx}OJ!}#Yje%C2KhK50Y?N6l{R3O=_IhJAoCHTi23roR$Xl+MyfL(tz8n zKsSn>hQJXZHO3!09+d%XbOyw(md68PzOJ<^Hwi>Bp#cCWSO9?G)j}irLnAhDZ2&2u z|0IC*A;6r!lNSh37vWEewGhD^*klXH4U(x$m&O2W`1m999CLgoW-%VIs1*`^3|bgb ztR;Fn>X*5q7^m|lLPW7B|MOU5s6}3Jh@tzgN?EJbvhat$- zF@Ll%df*Ez%&2eTyMg{G#DR}EbcMbF-zAp}j-1qyB>F&?Q7c&tB{6)H7e$sHP(D?dAFN8bBdh~ZO8MUqOVi1HJ zu45ooT`}0TeCqq506|YQl^nD|u3G1VI&6{k6SLNkreUZaP7}f}_ii3A-<9V_?=*~L zx4K-1eH88#)##+dHxU+CEOd=ybOkwf$=jATaY&n1aw~5aOszq!;=z$tVNO%?d3|t< z&b^p_KD#Yqt<1VFSj=AewGkXFoRpD>%x&ht;ObkC9fcJ+ut+2Z66v*i8mq;>hxeU@ zQ@J;fLc;sePM(v$jYRo9Ji$Y|rqE}0R`Iia`i}fuc_=0i0E+kmKDE*~Z5O2QVBRmK zhGqsCrAT~kQJK;Dgtf8&3te@MW31zF+$Er4;mk(~BN;e?M4!=jTvSL(&k-w<>b@MgRmoAdJ|e6n67iK>Nh8>5z7+tuP#q9hohQsp+q$uTdTw z+fbmK_v}nCYqk3}UJd_7_x)q;ONPLrxW)d1PwI|UZb1vRi&{&dKID6bLWR@OpoN-l ze2ZTanOF_gjjH&gb({=)hb2E<-1hxseP+{irMnVv&PhFU_jKyjJU*ABVrw zGe8p&Mi)_~-jj3TtpGj6AB#APp*korv08(i6?KrcCK)C2AeV(Q(U*)zleL~Wq_S*dY@(g-kuRR67F7{rH zZ}7C2?XZ!dbOwQihlJ;ElgQgClf7WXIg`rR90^n(CZ3>-ci`T zPqJINOPA6`Pz)G%uQuwlelPO?FH$KZYSq|icmKnumZKcC;SjeW#y(&;uQQ?!-nQ|s zjhAf2^5vp(KCbGz%-o-2uSj1Z%Tl}RkTtH{-Dwp%YGCJhJ*7JP)HS^tQ`prhfj#Tj z!0b;`I~krOiXdOidt_xfC}?cpKVCgJcMquZS1q3RfYyH|4c7ZWIVi+`+x;c(R|XhA0tCNyS@i#BABliXE5JmR6rw+hOh*2H#QHxU`a0$ZVr~2riU{s` z75Kl^m(N~n*g^h_`T_yM{-0v3=dWStTmKd2atpWk-kf>Q%z0+c6SaYTypD~jrU*bJfP;fWg+oX#s>aj-0A683Xq^ zwm=u{Oi7I%9S&}c84m7!QZwuL>2ZRc53K%}h~Hii7h!EXHX!;Q?>JJ2gEPQ6fFpXb z@s#mhghx5*T81m34)IC!OD+(0h;^GVasd{+-FJ;H`OvICGPB4Ba?0V%QtRhooYg;(F|gGRIAb_-!!j4uRu0mnIkXJFt1vxS}F5i z3&AE$ZWQxWEn#)duOBX&+TmRD%D*Kp7lJiswMzBSSZmVtN4#~u%n;ZTF{t!TmN3>C z>min=k9&LJ6x;9|3T68^$vWw~u2&SME3*JKS`_eCpV55^!pqYxL_3>#vnf(HD}J5C z-7X;4F1UiWEG-uZ_w`m4XYDEGX!qztw@9KRH5ZMs_Vz)CjLe`r#)U}yaK~Co8}JC1 z1cHYW{8q&x)>@?hL(^_0eY>V!J`HAGa^siq@T^RR(YHVo>K_p^y$V~!{>enB9esWn z4=KRF)#ig*{m`y=*GCJm)0w;Ru70CK&>D@wv&WF;lDo%*JAltHXRB)?5=+bj%}Z#V z@_@$5=LsBe+HBRdPrXVg9i}rp7=5FDa2pK5(TjA ztT`R(JLq8QO{vySbRg-ziZo#~g%%+$v{;i-?O0{t>%YR+1Eax^#U-F*Xns(|7%Q4+3*%5FXuRLilZNQV*sIPor0^(6T#h`d zL2T(sbo|)pWZm~&&I#QqcVzyW$Dgz`2kgeYrkujbdEy6a@~?OcSe7b|u{GFJ(+IzZ zd_U|kVk=hWI|M5*Wztru)AsO! z_lG6aIF0$s=`{PLe!pM-;zH#Elmq^$s?(Zvgs{HiC7o+*E#4}|ZCRX$^MkXtdG$&! z9Xg5L5)uio(a9XNg!%m4!rj)}Tekov{y!B;);DITDU(P23`Cz;6u^6+7>!cGDD~al zxF&RQJW}rNz0a%i!K>U4Q$i4Z%+|ozc_#gae*I<;8t&cWw`TuRhN>Z~^pVcS<{ zYj*M)&TnK^euexQe(l-6!B#P^W#(wwEm0+`6e_$hk_JfK3D!y6cE0mK)whC~e~_^t z#bD#ola@NA64Z-`)C04Z2vXC2wUAg%iZuH|ZAbQQ_`}SMdQRd-TCo;%!J)oZ%(KgQ zE{*W!NrLfc5gtvl{3P5VKIYdc{;zwv$4^z&D|o|1a(lV0YB%E8K3U67nNtV}^M{G-Uz*2RO$0jjP_n(_9vjG;OLOV(Yy zrumpx6kzpt+s0>a+H=SQiDHIls|xpilD>9~f411m zTGnUu|0BXkdsf}Ggxg9UC>ebWD%hds2Q4AX&S7(5=loCBt3z_Bu~Bfe7Rl9OkN=-5 zgYK7K*!(4D`}?DNh(PsFey&j)@#41gTK#~%tcMpxi+#}Ixe*nLQ^p!7q$rv`0LP>J zmX_OyBJf?=5{Tw|G`*?S*TuN5nJZL^EzAqQw}xWCpGHYbsV&Vm2iF&T;Ls`FrIdWm zG5m8hr61_7Km8c8p@IecWF8($^iKytt~DgKmwN_?&{~S@P_LrrKV!_J^@9;E`6At_ zB7B;DZbegNxTPR!jt(-^H^^J%dt>k(&RnQem%v>PmIRp%?P^d#XA?aq>nGHm*~d;ZomJR2AzS7W}2g{j-T+U)#LoPx|^JSOZJyVAg=3 z#2ddn2qrBwQd3I{t~(?WUUuN+=kmdJE%pFm_IBCJ)RI|bv=F$u{A4;MnE5bth5`>jn%}|5P7!t1XlIDYxO-1p}w)<4OZl~ zB-1K_RRk=XJkJ*9Kx`-I!dx!1_#$clT^te0XaPcpn>c3tQ{BU31O8c=WB43 zO>0hxZVA{N6v-)883An8ax9e}Nif*Odydo!!lK1&(w|pcSEV!x%$YcBObIw{vzs~N zFT-MMMg-#9MU#(Ui?`rMQi*mlyN3Z%5hC)DrTgJJUrPZC@WgNED$w#x4Lv=MKDI zxC-=|sr6+z<{U{}#+OS=Rmk0SncrptV%3pm{&1#qS?3`})mYT+Sd$)D!GhkJduBo1 zbuUD*lJcCJA_&&&W7SfWz1ZgW+{r41tZSKKC(CJZ)gy+aN)fIZKBo}lv{~F{YM_Lf z#}wgayMPDr-atncMhoH-z__*25Aq3wD+*Nvze|1dDAcTbqA*P~bOL$5+YK@2Xr$@A zr^SSi0PlMaXJkFjrP%TsjxCc8|L8k9LguvBw_zwG-v5*) z=@KwuX%2M-Sv2P>F6mnq%RI~Ui+VgU%Zr2qqes~1JY=FHw-Lq4D=qtgvW z9^6GrVltjp>y&%efnID+F^G)8>&0n`>R@E9dnv?D(>aL1O5>K^I(M|;dWS20yE1a5 zQvvo(uZ}<+C*x?S80GIPe;MFhoFz5AL!2yP9D$$x@{iwABr5AcpEY8$YWbyg(_BZ( z&zZuLKWz0Qj;VXm8w&o^iW5W$&d<+R$0I_jqtnUVDt7GNkJ=S|&pUWeO~|R{R1B7- z?g{o^pXL-JgBNO+;#5Y1!$i)*3pUmMsy!MU&~vDq0mVp)Aa7P^(CGC}qf@#>1MO>1@!$x}Qg=E~H)#?j z;w%^|N?ux7p4;%;J;jk7_C#ZuYzOY#K25h3CB2m8vj;azYI1^3u%&V(2~op>&U%)` zHwu|g0T}pb1{GyV;$iX>diTT=JIA8Wl5pl#s_qAQctPC0CWlp(Chj<746aZJam-=a?b&i=CVZAG%jkp%mF;}@-C(>XViZ{Xm~4%4%E z;X;%Rh&!b#4-SI5j+ruZ3L=}awY)X;cbG6gL(-JrXw4(oIX#1q93q5mT~dk`ggY~& zvyP|NO1LziLse7EmG=Cr;gi%<5J3_7@RV|@S(uq@!eN-N+;=sMY#>qh50c`Jwy1Wk zrX}v;rni@v??XOv;LquM#(+yS@bh(@yeU|+%S+w7F7AI%-?N#*2hD_}G!`VlUFyp^ zKf1!0D)4Wz<(hl&ACinCrt&Zb?43aMF`Um6vHXhwa)i=Rp{SEvpO`ghL$_~=fP;l%Lib;UcXRlHG z_4~MsaV|y2P+-kw0&PBWfY2uOd75tThKtX9antu-HFp?lk;^HwEy2JxI^Z6!Y8f4~KT5i@K(jADF$n@W-9_P~xdZLFlcNasLH|xfe2nd##ScmCxq4D^D>MOi53r zD=ppJ2P7ig8J@-;1@@tKT$GqfJLP*v)0tL$c<>TEdwM`OHtGa;ASBf7u78qjlWK~> z0Qd1dKNg=UyQ;ZYe!EaQr`s*Q1v-D^q86XV3Z6arM%&s|B3NS#qog|lT?T!in7fV#Lwe=vPzwz0<=jMX499k zy`P+zkd>mmk3;K(5j#mBdLZN#VT^`lM!}k{5IuDFJ_Y=Gv=@A0FLWo{W2TA4=%!k9 z{Jv4W|M5`n4(#Y=zlStW(14Or?#U|3vPXTvHc}cSmYeoE?|0Qr<4+oaUHIvLwkST!>bp;xW>5?de*d+uOJq+iOgL z!`ICC!8QnHDMtsgZ1D|+vQ_{;$(KHgKD#D$4>c;)CxZ7j`{J*gHNO+R$K6_2j|`XX z`)-(w@Sb1PVeE14EY+oaiK;^2R2*?*XAkSGb=`{H2-jHG@!8Ey*8R=N$vg)7;J`QE z28Eg*zPFNj$84Hcu@yzKXH@ACwInPPeM}4Ssjb!}AFG+m@Q2js(`&eDQ!Y;SPILzu zuTM_0oq54Mt;ajm8uf8?Zqxg7^+hWFzf@h&1Z1AEbxu<&K3qsb6|#$=1;5#nM-=9N zqv9U^Z0yltx*-zvYqq4fJ;(b36h5NaW49reu8?lVE|gH!)#Ai+`2x>k&y$PacksW|P9=^Jn{C2iypbfB3_2z5JFEhy>18?TeI8eioRCOw9 ziwrbBM#qI3q*WYzL`NyR?iv}SA$8$80Hd(aKKL| zgq@%TM)tu-QgBCk^X9iwOo}_V-9)r>N|~YIUjBmGnN5R7@q4j;X$ftD&oqN3rJQN; z9-hFZM~tYoz0%jK>9wsx${Sa;f3!YjFW=SA$@}bRmrz(Xf^PSy!Sa&p9@@4 zZZWu^-cFx34Ohm_62^PCe^gm>owPx_v^RVZ2(}Z@4Suacqig+$c~VCB0K!SRpcm&1 z6B`K~2mDD^uu~HVN7X7+6#AU|-S%;Fl!`LWV|}3~h=I#W|2MScUSt)-3tbmY&4LX} zf0$27S*R{Z5QXRS#w4FuURq*>hutE*%Jmcz<-Z|5i`=?K=$`hMluFjVef4Zi%3>F4 z1zzwoohSpP-4Oaw_3G;1_$eP33BfAj2<&;;x4)g1>8cmhth%{WSKZx%rr+5V9k%&a z&B3x%mR(Gs-^3s9?dHO63b+=6r37MU0XXB&Kyto^16H8%c%678E#h?56Ln012xEH=_gIdDuQG2wFtVT zI*p0N^dfoX4NLasP{YABeMzFGmw>{I8btqUV!;jR;SK5VFAJfeAsw*A4e&@PXviK0 zFyQ zf&Lsr@3mz0LXWRfQ**Oz5-2z41R+9M0o#ED;CVw?pGFK@`n6@v(~`HKj5V7`z7bfY z?UOlU0h}LaW&zm>Jp8H6)AZ1NAUW5`{Fnsen16nSHH6pmF}`v++P>s=z85Sas37Pr zM8v3E5lhCLrw>##MRZ|85Z}?a8kdcE4NUP}ef9_MK`*|p-t||^uAH|BV7H1kiHj`n z9^2LOI4jx>d^X3A;n5=5zFG4M`U7aQ>umI_zE`hL5r5a8IGcSPP=p7yO)s>1Cw(*$ zSrfsLtGdU>CytDGfX!o)Hw0!Vhdl38yTR(?JJr90>O%Ne6T?m9OeqkVbL=G zDrf83W8l*~Nu`k>!pPcIo0=La-@V?d*7=FRws%XmN-I_T2E9ud&B~#L#8plbakHXw z7m&a@Y&T=bMWHoXlOFyjbZy*q@{YBP<7hsKiXmb`RQXAAORQYv$|k%gPF#GUUZlRa zDNsaQ4^nEspQ*Xl0JeXdkBvhZkt}D@^(K*_sp06(Bda1+Rn~SUW)prA_oq#Y^Zaf_ zsN>nmJh_MsI-115WM+8$N$16>_U-D{k$hJ!0h7_K*+2{|A@%0#%+7vf#xQ2aaH(as z-bgcgu(KoyGtMb8%#})7FH4lA#h`m+si?TuI`J3NFC>a!2v|={>8_B9h|;?f3rLm& z4cqUn{2|IsIvyx6?CYQn^Rg{uVg&{nfybjux`0tAj^Jc)>Qj!|#FJCW*v~6jO$0s{Q%1C5l>>AN|5wZznM#m=pP*3)SA z(`4hCI+cPo;4=MNEO{J1G;Exg>Umy9dE3%zn^zV*>Co*P?j{%kmruFQpm!Pk+l zj~^Wclwa@T?WbzUReEcx=~8Q3p;hXHzsiC2wHvSIGrn2Bvl``WBJ*9X-Tu8xWpKRZ zC4@UQ#oee3b8mzy5B7bUW8~WOJ^oS#?g5szxDsGI;$9G|DZ(QV0RC5thqUbi^!~90 z*&e|9FK7tc11Q43`d>93vak=Zq4{66>P4KNr-Or&dJ*&g^A!ZXqKBOC14JRY2LK*~ z434A@KM@G_A%G77!ut;g90CYv{{Ook4({E5>R90LRq_)`9ZF(J$+0L*{G z37x!jh)w?&p*{uRL9$N(tbZ95GW<8 Date: Mon, 3 Jun 2024 22:06:41 -0700 Subject: [PATCH 54/62] update run to 24.05 --- Triton_Inference_Server_Python_API/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Triton_Inference_Server_Python_API/run.sh b/Triton_Inference_Server_Python_API/run.sh index 9d27f38a..459b32c6 100755 --- a/Triton_Inference_Server_Python_API/run.sh +++ b/Triton_Inference_Server_Python_API/run.sh @@ -102,7 +102,7 @@ get_options() { fi if [ -z "$IMAGE" ]; then - IMAGE="triton-python-api:r24.04" + IMAGE="triton-python-api:r24.05" if [[ $FRAMEWORK == "TRT_LLM" ]]; then IMAGE+="-trt-llm" From 246a2371ab9836f36052a0ce1d6eaa4e6b3d2459 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 4 Jun 2024 15:17:50 -0700 Subject: [PATCH 55/62] update to exclude none in settng vllm request --- .../examples/fastapi/fastapi-codegen/openai-tritonserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py index 58d2ed66..3735b780 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py @@ -144,8 +144,8 @@ def create_vllm_inference_request( ): inputs = {} sampling_parameters = request.copy( - exclude={"model", "stream", "messages", "prompt", "echo"} - ).dict() + exclude={"model", "stream", "messages", "prompt", "echo"}, + ).model_dump(exclude_none=True) inputs["text_input"] = [prompt] inputs["stream"] = [request.stream] exclude_input_in_output = True From b1653d79701a2d53bdf0117de7c41788fcdb0480 Mon Sep 17 00:00:00 2001 From: Tanmay Verma Date: Thu, 25 Jul 2024 12:44:58 -0700 Subject: [PATCH 56/62] Fix the parameter to tensor conversion in TRTLLM FastAPI implementation (#98) * Fix the parameter to tensor conversion in TRTLLM FastAPI implementation * Fix format --- .../examples/fastapi/README.md | 12 ++++++------ .../fastapi/fastapi-codegen/openai-tritonserver.py | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/README.md b/Triton_Inference_Server_Python_API/examples/fastapi/README.md index 824ec836..694580c8 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/README.md +++ b/Triton_Inference_Server_Python_API/examples/fastapi/README.md @@ -26,7 +26,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> -# Triton Inference Server Open AI Compatible Server +# Triton Inference Server Open AI Compatible Server Using the Triton In-Process Python API you can integrat triton server based models into any Python framework including FastAPI with an @@ -34,7 +34,7 @@ OpenAI compatible interface. This directory contains a FastAPI based Triton Inference Server supporing `llama-3-8b-instruct` with both the vLLM and TRT-LLM -backends. +backends. The front end application was generated using a trimmed version of the OpenAI OpenAPI [specification](api-spec/openai_trimmed.yml) and the @@ -118,7 +118,7 @@ curl -X 'POST' \ "stream": false, "stop": "string", "frequency_penalty": 0.0 - }' | jq . + }' | jq . ``` #### Chat Completions `/v1/chat/completions` @@ -165,7 +165,7 @@ curl -s http://localhost:8000/v1/models | jq . curl -s http://localhost:8000/v1/models/llama-3-8b-instruct | jq . ``` -## Comparison to vllm +## Comparison to vllm The vLLM container can also be used to run the vLLM FastAPI Server @@ -185,7 +185,7 @@ Note: the following command requires the 24.05 pre-release version of genai-perf Preliminary results show performance is on par with vLLM with concurrency 2 ``` -genai-perf -m meta-llama/Meta-Llama-3-8B-Instruct --endpoint v1/chat/completions --endpoint-type chat --service-kind openai -u http://localhost:8000 --num-prompts 100 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 50 --concurrency 2 --measurement-interval 40000 --extra-inputs max_tokens:512 --extra-input ignore_eos:true -- -v --max-threads=256 +genai-perf -m meta-llama/Meta-Llama-3-8B-Instruct --endpoint v1/chat/completions --endpoint-type chat --service-kind openai -u http://localhost:8000 --num-prompts 100 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 50 --concurrency 2 --measurement-interval 40000 --extra-inputs max_tokens:512 --extra-input ignore_eos:true -- -v --max-threads=256 erval 40000 --extra-inputs max_tokens:512 --extra-input ignore_eos:true -- -v --max-threads=256 ``` @@ -195,5 +195,5 @@ erval 40000 --extra-inputs max_tokens:512 --extra-input ignore_eos:true -- -v -- * Max tokens is not processed by trt-llm backend correctly * Usage information is not populated * `finish_reason` is currently always set to `stop` -* Limited performance testing has been done +* Limited performance testing has been done * Using genai-perf to test streaming requires changes to genai-perf SSE handling diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py index 3735b780..253d5c51 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py @@ -165,21 +165,21 @@ def create_trtllm_inference_request( inputs["text_input"] = [[prompt]] inputs["stream"] = [[request.stream]] if request.max_tokens: - inputs["max_tokens"] = [[numpy.int32(request.max_tokens)]] + inputs["max_tokens"] = numpy.int32([[request.max_tokens]]) if request.stop: if isinstance(request.stop, str): request.stop = [request.stop] inputs["stop_words"] = [request.stop] if request.top_p: - inputs["top_p"] = [[numpy.float32(request.top_p)]] + inputs["top_p"] = numpy.float32([[request.top_p]]) if request.frequency_penalty: - inputs["frequency_penalty"] = [[numpy.float32(request.frequency_penalty)]] + inputs["frequency_penalty"] = numpy.float32([[request.frequency_penalty]]) if request.presence_penalty: - inputs["presence_penalty":] = [[numpy.int32(request.presence_penalty)]] + inputs["presence_penalty":] = numpy.int32([[request.presence_penalty]]) if request.seed: - inputs["random_seed"] = [[numpy.uint64(request.seed)]] + inputs["random_seed"] = numpy.uint64([[request.seed]]) if request.temperature: - inputs["temperature"] = [[numpy.float32(request.temperature)]] + inputs["temperature"] = numpy.float32([[request.temperature]]) return model.create_request(inputs=inputs) From 0eb2bd46e99fa61032feb14a771481efab59df8b Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 30 Jul 2024 13:34:34 -0700 Subject: [PATCH 57/62] moving to 24.06 as base --- Triton_Inference_Server_Python_API/build.sh | 14 +++++++------- .../docker/Dockerfile | 6 ++---- Triton_Inference_Server_Python_API/run.sh | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Triton_Inference_Server_Python_API/build.sh b/Triton_Inference_Server_Python_API/build.sh index ae3e2e27..f592ba82 100755 --- a/Triton_Inference_Server_Python_API/build.sh +++ b/Triton_Inference_Server_Python_API/build.sh @@ -39,12 +39,12 @@ DOCKERFILE=${SOURCE_DIR}/docker/Dockerfile # Base Images BASE_IMAGE_DEFAULT=nvcr.io/nvidia/tritonserver -BASE_IMAGE_TAG_IDENTITY=24.05-py3 -BASE_IMAGE_TAG_DIFFUSION=24.05-py3 -BASE_IMAGE_TAG_TRT_LLM=24.05-trtllm-python-py3 -BASE_IMAGE_TAG_VLLM=24.05-vllm-python-py3 +BASE_IMAGE_TAG_IDENTITY=24.06-py3 +BASE_IMAGE_TAG_DIFFUSION=24.06-py3 +BASE_IMAGE_TAG_TRT_LLM=24.06-trtllm-python-py3 +BASE_IMAGE_TAG_VLLM=24.06-vllm-python-py3 BASE_IMAGE_PYTORCH=nvcr.io/nvidia/pytorch -BASE_IMAGE_TAG_PYTORCH=24.05-py3 +BASE_IMAGE_TAG_PYTORCH=24.06-py3 get_options() { while :; do @@ -151,7 +151,7 @@ get_options() { fi if [ -z "$TAG" ]; then - TAG="triton-python-api:r24.05" + TAG="triton-python-api:r24.06" if [[ $FRAMEWORK == "TRT_LLM" ]]; then TAG+="-trt-llm" @@ -208,7 +208,7 @@ get_options "$@" if [[ $FRAMEWORK == DIFFUSION ]]; then BASE_IMAGE="tritonserver" - BASE_IMAGE_TAG="r24.05-diffusion" + BASE_IMAGE_TAG="r24.06-diffusion" fi # BUILD RUN TIME IMAGE diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index 4ae0f5eb..dd28b017 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -25,10 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver -ARG BASE_IMAGE_TAG=24.05-py3 +ARG BASE_IMAGE_TAG=24.06-py3 ARG FRAMEWORK=DIFFUSION -ARG GENAI_PERF_TAG=r24.05 -ARG TRITON_CLI_TAG="0.0.6" FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} as triton-python-api @@ -55,7 +53,7 @@ RUN find /opt/tritonserver/python -maxdepth 1 -type f -name \ RUN pip3 install --force-reinstall --upgrade /tmp/tritonserver-2.46.0.dev0-py3-none-any.whl[all] -ARG TRITON_CLI_TAG="nnshah1-default-echo" +ARG TRITON_CLI_TAG="0.0.8" RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${TRITON_CLI_TAG} diff --git a/Triton_Inference_Server_Python_API/run.sh b/Triton_Inference_Server_Python_API/run.sh index 459b32c6..ecc7c754 100755 --- a/Triton_Inference_Server_Python_API/run.sh +++ b/Triton_Inference_Server_Python_API/run.sh @@ -102,7 +102,7 @@ get_options() { fi if [ -z "$IMAGE" ]; then - IMAGE="triton-python-api:r24.05" + IMAGE="triton-python-api:r24.06" if [[ $FRAMEWORK == "TRT_LLM" ]]; then IMAGE+="-trt-llm" From db86092793608533dbc609014f0baac76762ba2f Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Fri, 2 Aug 2024 15:58:18 -0700 Subject: [PATCH 58/62] updating to remove unnecessary parts --- Triton_Inference_Server_Python_API/build.sh | 11 ++--------- Triton_Inference_Server_Python_API/run.sh | 6 +----- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/Triton_Inference_Server_Python_API/build.sh b/Triton_Inference_Server_Python_API/build.sh index f592ba82..f815c3b7 100755 --- a/Triton_Inference_Server_Python_API/build.sh +++ b/Triton_Inference_Server_Python_API/build.sh @@ -29,8 +29,8 @@ TAG= RUN_PREFIX= BUILD_MODELS= -# Frameworks -declare -A FRAMEWORKS=(["DIFFUSION"]=1 ["TRT_LLM"]=2 ["IDENTITY"]=3 ["VLLM"]=4 ["PYTORCH"]=5) +# Frameworksx +declare -A FRAMEWORKS=(["DIFFUSION"]=1 ["TRT_LLM"]=2 ["IDENTITY"]=3 ["VLLM"]=4) DEFAULT_FRAMEWORK=IDENTITY SOURCE_DIR=$(dirname "$(readlink -f "$0")") @@ -43,8 +43,6 @@ BASE_IMAGE_TAG_IDENTITY=24.06-py3 BASE_IMAGE_TAG_DIFFUSION=24.06-py3 BASE_IMAGE_TAG_TRT_LLM=24.06-trtllm-python-py3 BASE_IMAGE_TAG_VLLM=24.06-vllm-python-py3 -BASE_IMAGE_PYTORCH=nvcr.io/nvidia/pytorch -BASE_IMAGE_TAG_PYTORCH=24.06-py3 get_options() { while :; do @@ -165,11 +163,6 @@ get_options() { TAG+="-vllm" fi - if [[ $FRAMEWORK == "PYTORCH" ]]; then - TAG+="-pytorch" - DOCKERFILE=${SOURCE_DIR}/docker/Dockerfile.pytorch - fi - fi } diff --git a/Triton_Inference_Server_Python_API/run.sh b/Triton_Inference_Server_Python_API/run.sh index ecc7c754..7822e6a0 100755 --- a/Triton_Inference_Server_Python_API/run.sh +++ b/Triton_Inference_Server_Python_API/run.sh @@ -29,7 +29,7 @@ TAG= RUN_PREFIX= # Frameworks -declare -A FRAMEWORKS=(["DIFFUSION"]=1 ["TRT_LLM"]=2 ["IDENTITY"]=3 ["VLLM"]=4 ["PYTORCH"]=5) +declare -A FRAMEWORKS=(["DIFFUSION"]=1 ["TRT_LLM"]=2 ["IDENTITY"]=3 ["VLLM"]=4) DEFAULT_FRAMEWORK=IDENTITY SOURCE_DIR=$(dirname "$(readlink -f "$0")") @@ -39,7 +39,6 @@ IMAGE= IMAGE_TAG_DIFFUSERS=diffusion IMAGE_TAG_TRT_LLM=trt-llm IMAGE_TAG_VLLM=vllm -IMAGE_TAG_PYTORCH=pytorch get_options() { while :; do @@ -116,9 +115,6 @@ get_options() { IMAGE+="-vllm" fi - if [[ $FRAMEWORK == "PYTORCH" ]]; then - IMAGE+="-pytorch" - fi fi } From 45bf9a65b0c95a5e279059fe61eeec99c6c59442 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Fri, 2 Aug 2024 16:22:26 -0700 Subject: [PATCH 59/62] updated to not rely on triton cli internals --- .../examples/fastapi/fastapi-codegen/openai-tritonserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py index 253d5c51..5ec3c968 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py @@ -33,7 +33,7 @@ ) from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast from transformers_utils.tokenizer import get_tokenizer -from triton_cli.constants import SUPPORTED_BACKENDS +SUPPORTED_BACKENDS = ["vllm", "tensorrtllm"] from triton_cli.parser import KNOWN_MODEL_SOURCES as KNOWN_MODELS TIMEOUT_KEEP_ALIVE = 5 # seconds From 1cb3932a1b9c05b58a3b77a0427e50b129c8d73c Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Fri, 2 Aug 2024 16:24:45 -0700 Subject: [PATCH 60/62] typo --- Triton_Inference_Server_Python_API/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Triton_Inference_Server_Python_API/build.sh b/Triton_Inference_Server_Python_API/build.sh index f815c3b7..612b3f3b 100755 --- a/Triton_Inference_Server_Python_API/build.sh +++ b/Triton_Inference_Server_Python_API/build.sh @@ -29,7 +29,7 @@ TAG= RUN_PREFIX= BUILD_MODELS= -# Frameworksx +# Frameworks declare -A FRAMEWORKS=(["DIFFUSION"]=1 ["TRT_LLM"]=2 ["IDENTITY"]=3 ["VLLM"]=4) DEFAULT_FRAMEWORK=IDENTITY From 840298951a11d59a327fb3dc5b48ca8f7b5b0b71 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Fri, 2 Aug 2024 16:37:07 -0700 Subject: [PATCH 61/62] updated to remove pip install --- Triton_Inference_Server_Python_API/docker/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index dd28b017..b07b655c 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -40,8 +40,6 @@ COPY ./deps/requirements_vllm.txt /tmp/requirements_vllm.txt COPY ./deps/requirements_python_repl.txt /tmp/requirements_python_repl.txt -COPY ./deps/tritonserver-2.46.0.dev0-py3-none-any.whl /tmp/tritonserver-2.46.0.dev0-py3-none-any.whl - RUN pip install --timeout=2000 -r /tmp/requirements.txt # Finish pyright install @@ -51,8 +49,6 @@ RUN pyright --help RUN find /opt/tritonserver/python -maxdepth 1 -type f -name \ "tritonserver-*.whl" | xargs -I {} pip3 install --force-reinstall --upgrade {}[all] -RUN pip3 install --force-reinstall --upgrade /tmp/tritonserver-2.46.0.dev0-py3-none-any.whl[all] - ARG TRITON_CLI_TAG="0.0.8" RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${TRITON_CLI_TAG} From c00bf15cb5e33cc6ef1cb6977c23ef1c98bab9df Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Fri, 2 Aug 2024 16:40:20 -0700 Subject: [PATCH 62/62] updated for style --- .../examples/fastapi/fastapi-codegen/openai-tritonserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py index 5ec3c968..df8f2e36 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai-tritonserver.py @@ -33,6 +33,7 @@ ) from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast from transformers_utils.tokenizer import get_tokenizer + SUPPORTED_BACKENDS = ["vllm", "tensorrtllm"] from triton_cli.parser import KNOWN_MODEL_SOURCES as KNOWN_MODELS

S@Y*v^^fup*x<0IAN-YZmAm;|PPx;n`Hhh)rwU(4BG{clP z=Q!x3fa@`CJ$CZVRwSMhZMGb&$*#TcV+>Gt6$bnQjoHd1`dKqc`A2?NF)=smKULve z{kdnkwRWrfn;8+X*D+Pns?PDi*0F63@5;LR)xX2~J5MORUK(0?T_k^N9i139aNrrI z`&T|)tozAe09`Nnjw8W8J23*m_LQCi{54ZK^pWjaEu%1HbU@i6ib?<;ls2_yC3Qj z@DUS{Ratj-f7&3u8nJrg8mQ|>%;{dJsBN`$r}U&q20C)PeHe6deYul%O%czeenhKV zoUHJMI(%lq_KW`6v-}N{^lyGFXtEyfsOf8ML*p3B%$0`6O}bvK?dk&6wO^XE&t?n0 z+h1<$N_fxG>VbQ%{@f_1sg1gYqHTK*>s(;l8@(eEEG0}D4l>2@`_uX!L;$3zd|PT= z#>PTg1f@9F7`+>SqFZgQ2(km8hZ&gT*^eIF@2AvKw8W_JoguonA+I>B#e9ET)C=vb zZ?iatw*~BLyPR(-QXxw zd*!MjG{87YW;FkYoFO&D&8{yAcCAZ6PX4<={sQkeY*M1G0?=z;m2d3bTA}$1>UV6- zabhBceoy-~;1U~8Q0U=SzE7H5XONif#Ak0BMh4qHpt~Cu-hk2V%cBQ{8yBa#&ox;5XT+Z5jiLy7@`o9OL%PA!54uMgb zVGC-dV?ukem;Oyo7fFs$8e=kV72ofubA2c4TR;LmHlV%kUBJOL)zYT|108`bJAY13 zy;HlSg(!&9duJ7IZ}2*r)AeG8`rd*O_?H}8pL|MSlcoYlwA8)nHgnBda!dQvcC9+< zu!ExBBXawV#pIwy%qobAQ*-7Tgb?6IayfUze0<6cRdPw&&)uhPDsFyOI;>f>ccjAl zuID1mU;1ju+P&W4_ZDkAe{Fvm9V`FOMK^Hc(_<^Ls+ERvL4F2pje4R^1UVMycI5yII!b9aD+dks%^M^~*$HQc+E<~}fdCW0@s|Pn*mq?a5 z&=V!4PeqA8u^B$YGQwQr&+G0mqQjUsxW{JYhH@`6ky>jioZCn0)oVIxohPTy zjF`j3mXFdj&uRq9;Oauxpcb;^CT8kFC{=u0s7kBss7gg?XA~1X@{WR?TNyk9}jy?^pOvZH-hOe-xFFU7o_RTkJYH{ z`3;)Og{B`(NZVQv;zbk?^>@XRTYGO~K5WSggc&ZY2rUa1V071OlUi}V!hp-t1q$@7 zkZhmFdy-PYXHA)Y(><4fOVda*to~RILl#K6-12p+s&ofdb_J+1hHCn-^-v(E%eNZ& z#}QRm;HF;?l$1mJdCtV-#q^h)hx=YpTgt;UodR02DBU`ytfgaAPJT#aqi*0W){^l(c7+HYFzYD-UUb7uiYr@;CP zW%1{nLa6jIXJzYL#f}9m%?PHnyuq0qAr|+-&93aZkiRFw?#psdHOA)st9UG@#VU&$lP{QNy7;^J1akQ3(YN>7@F%p8+q(@TM^_jK zWb>zkKRSNOAeY@gyr*m%r@Itq7M3>@W_TB$QoX?hJ;tZR^L{rAD6_x4v0rK@{6nO= zc)@>)8DW&VB8qjW>}4dI&~6i#Q|ryPKu&f=xN;2N@Atl2;+rxhzWC(yoj3TVzU!N` zGje%b|MGX91un4+B=c4#vR{ec(g>%e&o3n0asWq#`k;!Vkf-u&K3VMivX{B*PsB%M zsP%0qlCu_1?xxyilo7`n#J-K#w73&de~)eupouLWv0p!{aL&N_^KwAu8BU|keb}$d zI~$p`anMnyX1#SkWF9iZ#`{YVe2aS~I&##*$yGvianE&}fZt8b6F1lDalV3ir;xaZ z6_WY}*v7B^(S;nITRe6XMV@M}Wz--AV|-+@ifyr{vLX8O+5bj)l37tE+A)n=I%e#5 zZshUZ05=aZFv`T5Vs?0URl%7nj>mfM&dcONti$w_v84jyU1q5aAPx)NE!TNUcRc(G zLI;fruy{xCP7)-Vc#vfe*Gn`VG)-k(b0bIZp66N7d7C|zOV58bVhTCG72>hm;+sMz zE8zU~r7rUuu}C%%E2c*;nmMO!O3yJcz`DExftML`<#yo-;A(=jh5?rRPPIC~(2-t} z8E|8+PtlgWP}iPkcbc}5$`Hh1`_?z+-N|N=CPBa~_zSGbB*evmIKW?gBzO%PVKNW+ z)V3`QEs9lK;lrybXpO`Mz1bsNX91#M{&uK>xJ6GB{zuV2qgE3rW{gGhwxadSlKIg$ zYfkCnuqeXXBOZT=jl9BhY>e-1BKR3Oi?JhQCJ7G35rj84MdNQ_8r<~B?d~6)pZ;jY zs0puqhLbYN8ijd})17xp{5f>Dr0uMS2)_SK-Tf*No^tzuiOtDBlY!M*)GGNO8`SeO zXI;p{)$!Nqj2kxtpwa@s?hOBUq0My`0%kFuq*=d_7aI}V{>^0mpRe1chBmYZdCk$? zf8U|!gvm0bbn7iS>{qxIYS6}nq{=G}XMk(i@jS0Fsw3X%T$Qk8!$N2f(K(HHy&>p@ zb5qo`?VY$sL~otd+fs@az^b+Ru_?Q)WfK^92)g+5?GMW2vj~ebNZSo4Pc{l(xp(vN zR6xvYKlcO^4d0+d^kH%6gKMu7q8j-ELm!+2?6kIcGQ$$4nR%0)cZgPtJeeMoE87zh z1I^$YBkD3~x6=ioAXD|CTmP?VrfYxRL*rpw>k zs`-D_p46%_v|?+6bUkbQq>36)G17vKuKLP%+zt>hApBLPH?^)d=6VP={#1!bbv(j! zvZ!vqwEB5laK|%NYb3oi0PnIKS*Bn67>%D1eGTtkcALY!t*IC>qYg(~nBV?)ufy$a z=2~!;I&9>qf}d1U&uCcJGG_o@d#~1chV>d9g>OzUzzz`N-Qp;3! zbXgbf^&hlgV1EuYXZiJ=?3Y)OQ+DVs%HK!>|3$~Q8#Zi$MRWs(@>P!Roxo@beCf-~ty!r=H$l+ByCvW%k3@2t)oLp`6IAfZ{ypRGMiY6pUzh%9 z@!A#GQ}6em;|5rf%IL0qla>vI$;L_Zae=0SMGTP zH6d0>)>qBnfLJg7RnegJ$e4hXmgC5ZMPb*HfW|l(sP#&II&e|vkMtMM9HQ6qB2U2U zSNG8sv(hF95CF@tb$+cDfNJvKuF2rajo{BB=F$Ff-qeWP$*`#AzU|WYKfZ==>s^16 zBKi^njx6p_&eD5CYYgw}ecoVN?zd7sJ@=WMXq8iTF)nuE9!ft6KI1;|wLP4dP0Eug ziCRz-FZ-7IY-x{Vc9*UORL_$qdqe<>-Khz7=qCkeXZNE zHC%)Cm}@`G2TcSp`U=6xO}ee3+4cu4&$h*}4GJpRVv&T>_O_qH&}*>l?M+8= zdoAvFF63-Nk%UV2seEFO0^^05+v1>}%PzhuKq$en_`imQR9w6fy}Y~a&+wBS`qoRm zO~8zdayEV9g7+!t^F1T1IrTx)y(H^H|*JRdSf@cqMSux@F9NHL^P8fxCUQL*CwGcUR? z{tSul2mve{VX27tnWu2XJLhZW7v95cQRDfbi!kJ4n^^kZz$!Y$&+vWD>bPGUb)^9{ z@?TYz`gevVvC3AutId%pIpbz1dpg=IPuT{&(jb|ip!DZNOAIMiUv|IY!R%8fqOZU` z8RA!2W(m=EgJ`u$KvZ5r#@4?4z5!O zh*QVdO5NyH>9wG)X++&_O1*nqz5v^^9lh(YhM`R(ZXYbR7IR18@rb7uiX#^dTV)>m zcQE_dQY(svJ8J5Q8rPze!Mj$NV|Vb%K!#Kh^SzF)NvcR1KNn;o+AKDvySaMFj=CVV z{GPde0=S4Rf=|HX@!z@^ZsxqG&=jV^2JY-q>TZoS#b>bAZKy?>VbGv5O}lB%adVap z-Yr{Gf1}4fU(?#Br6>sdsyVb$l_nQx?jhh`)Yk-qaCAB3l1aSZtOQ}D!6+}mQx?TK zCoLu!YJYHgABhi`T1hRLtBz(>oVtdA8g>B`PQ6nSh(*oYswXrIjXKnfk~LLCm^an? z5`Y`5_Ze!rwh{$A4}cQ?BPN)*BTp7pL|9jnp~DTH!|Sgw9ovRyVDy=f{x zDZIKxpf-&1U^YpaA|6PwUG+_zRmjWk$$FU* za_Vo2SB(q|J|ba{OCmjB{}w^H4Fg$kHtx3RyB_5sU6<-Y_kL!SXUD#n|1lq{jhEdM z{e$_7<)FmCVx-Y!VtNLcbQS8^s!f}n`bKTp<^%uAZmtP>SxlGZ$oScm)vzGS?6~h} zV;0n>Z9JW=N)Ijtg9E7uhli_h^_^@E7yo*`-721t^1Eq91uACV!Fu&nXsN|2=i+5!La_oB`h5;e!11O|f0H%XJeZw9bWTeCNv|84osvRlr$;%QZV zdAWax@OgW5P$MVqYsFD2rtlHNr=Dt}2i_{0__9@+yY15`dMsLneCDy{V(XX1+?=Xp zU>I9)r|YKf4s>3B=Q{w>I`49)#v6=WC8~85^NRmuDeOMQ>hEdHGNQXR5L_$IV-(Wc zfyQ<3hr|rEwwV$eMQRbx+!Hkg8-T2HeLNi9iC-ZebAL}bTmQ?QoMKBvxZ3nj3ydDG zFcbUPct|<96%EH|zX8~5y?Zq=#gBAiF>Un&gKO<(4km^XYumT$nl|j%Q3S`m{YrP5 z?|(LK9!SH6z!QjU{>X&F>4ZRyEKlV%X#pam15eJcJ8IsZmmPX#+sC7A@K#Bn2qL=0 zmcyGFN@do!d3zw&%Dqwi*)-APCE)|00>uN~!d1G%Dc(9KO~Z)$syZa?x&u0mF?Q8C z>Qm4-v?@Vx#q<8t^DYvz5>Nl3>Xd=^_cgl_c>v@=>wB!@Eh^l*jB|qB6IrF7;W^LD z%+|ErP@K4U9GD*#`YdBktJvlw^@y2&YR#fw$d4347Ir&Ba+=dSEHziYtavL{UiRW} zfOkugaI2!YEo;EqJL4u#wYykDz65jundT@c(|*6ke$a4nh*?z9&4=jV&+sQ5H75&V zT_eis9#itm8htKi^yzfoAlTabYx$=2y5%6-BTj-9CqA}tYUa=H$-iGD9~uJNS$T`WW+0jrY*(q>q{+mi?+(yGV>cLe!I`Vg ziX^V=EvEkv3n!1$dBLJw?D0hLU#dxqpRVY%=kz|W_{YqNbcA|qtGeHR<-(+GtdviW zwHP8|x2N8Qorqq7tc2~fwfV1l#99jx@J$SlaR-G@;iD$pr`e$tJERj$`OK&9jUpUo{EIY$r*hNhLXs zZ$OqRvv^7`ezeW^U=9l+sv+|+J$p8=d{6Y zYwY!DMD}jVY91FQIi;3}z4l-fK@(!oI>9)nTI$v>?&7RA+KOvqzn+eViW0`+{qbXX;JJ~ zO*!tAgq(y7Ol(8CC{rWQ(Qs$=`~W;?tJyPJ^3{fFB`eTbZNi$xlZ8=Y59PgZ^cEPE ztu4c;_RY{f96LFSstm?pSSTI4oHSxH5PxrA^3l{~QD!lIJlN%EfXR==P)!&`c@`mo zhB7fM(bowwXejto4(4-1lrrUtKA0>8^|^TwEAaRwO?nmZ6E=&JLKAVEi)QxS&0Yy#vhG%~z_3$~daRU~<5`gE zc`AhVNBgpLs2YPkAh_4X4QHu@x;rXP-VK&q-~I;Im5|zhh@@w%4{-N+rYC%5R;l@K z%yKv^H*ZM&Y52zN4EDBL{-XC~hD{6`cj!taDer-rvL!r)=2J*PDqezzvvL(rIVu`u z1RoeF$+X)ca_EO|2Gq7@E1`U0IP33hFnDWERe^C`iK=h1IBCyM0b9yQJ>!c|;_yq* z;jLeb^Na9d-o<&5j5p%!U)ug*Tu^%&vT7|;lWmA|aeR0GMgV$&=a$zBSUhKqES-Ch z7nAI4dE0&{_tzk3Da-AczLu5x<|gZtbFw!e({&bXG1e`=AufB+^zZM(_}AL2+jUQN z1!KZC|3!qwmLdL+KR?KJ!WD)^Qm6RTod91nUi+mnMDt85t9R0k1tHGHqj3{ zMEZ$~Wht)l(V@EXWZCY#*ao+0O`;MYbfR8(vY8)aK~B64sh;Py7E7?_V@eH9D)7_{Hih_Sr`oz(+auU-THTilkIJ1ffkbb6p5^UEZ3x)(sFK?UAg+TF9 zsQi>b%9~Yb%6Gl{!QjBzfi#m%FL>F>Xgh6Fd?!t(TQDoeOuym^D#Y(`$|p+hc6goT zeu?L?X8hZX@ke&i77xaVad8&+ww^3PDnI$3=RCoN2_M#H0_X|vY%1P{Brdgazvc*- z2Q`cJ_v~hIpC2BnWFppAW8x3iT9-=)_bvv?yr^BAvCOWhQziRV$OnVQEj-Nw2uXg# z#lOeSLxt4GhxNK{dm7(?$I?}sp1T7z=8%z97b~>ydTv?f(ln2`UzoPkH1g!`%!_)e zMA7uk=qv^4I*AR%^b5}f^R~Md>-3!GC#8?p-J$tR{gX>GT;N_|U=@s*NvB7Wcvser zG~&-b+jNi#+D^u{>A0@bAE)$@p3lmYT;op|38UgS~Pv!qSog+O57Y6dwwcWjqvkCvMH_-e%ia2<^u{lSO#A z{k?F?Hj?vEL7I~w@S7t{R)d^l9|_HA_8iIHFR0+(Z~n4#f~lT|+%0I|O-K#(o&RSj zgqPS5G#zRa90t8ITGUQ>BUy>wT-F?y$$96P%)MS6E<$-UD+E9j-$)`WE4E>u#lx~o zp~(0g?X-qWWs2l#a}Oy7H@t1Pb9Z#x?!G-O*r81U^r|URQ`D^K_>Ym2uY=A9 z`oBw3wuy|!8^Bryyy{ifUv7_ekf@q9|CJ9KbmZ*?4d!K{E+ITb?&iU3pZNTlIN)R2 zb_Vv7-VJ0p_9JxcpW7*aiNrA1YEeQ9VU^-9n7bD8snvqwBt)~ZT~r!zj8RTFd`!Tal72mfnf z-0@UVfl;n*V!Lc(?nlq3a5T@8R?qWyaFv;?R%rqE_Zv|dbx~1^l^RYyi;r9<74<*-@c}sUNzHe zJ4t%nTXhmiI16mUR%xy5S&8}f%xzE2WKDbmS;q(6R$QAKULv^nt`9URA9gyo4Uxk0Nwknhi?2OdVQVhM|r?*<4lQ)>;vDrXac9+}M-C+t+`gWF5KV9~lb9aQxEk zj*Vq5A{tVTqLg&)tr;f=f*sU~`Z%)vJQ*CO0y^8AB0{{Ni{7kES_&AuQ<6Op_+xJ>#@_6;5wUfFL`R+ek$KpF?Q6GPgwN(hVjj<+ zJ~fh(;JoPj@C{&q`RTPVZiMxl3hqxH+75LS>DZkZ8US|X1w!6^EEq=3d?mFpjs_BV{XF{|Mz3lotelhGWo(8J5uD^;>uhjN?kx%p@TSpQ;^*6Fl z2v6>Si(jJagA?L&g0y_oY?~*?p|n@vy5OTqd5aYaj4Lth>}edpG`vW>`u;{(GymD zOYaR_ZLm`%%k{Mrcms4E2o~`zwrftCgep}j@a&rZ%(fYBjL*D9 zz455_(E1E^>5ITqIyvLvPgtpEg0ecBhH*`&;WeFbyRBxG4r%8NUOTSoucoT|tei7I zaPN`!Z$29gXWBqlj?0FR7?5c&>!(anJ50b)?RNCqXl4C7+U$Mmj^`bDW_BP&byZ23 z;+an~SJH^hSEeMd9Uh}6RVa!>tkiStFi~cFAVp#|n)uey>YOH1Z&s)-n2N62I95fo zYgW69&xF>!dRQ3@u4IY6L{|o@CFEd|8jh+^JpOltjuT~G9%2Qa!L)TsdeR_H?ZJby z+y!0HLF9|n}Sac8vGH=fAR0pGLSp;XK=f?j~nqbG8&j!GU7U%YI z1{2JV&e6_nXnWDzMs^G1o#NQWu=_Z9iw=*b9*HvN0x4D`Akl-OpHEx6;scLihv+j& zeqD$E-!sHo6v0$!K5~{a&IwJ11X}bpcri}m?Qg2`%+b~HD_(k zX?0AL*~eQtyz1imY*5^3*-Q(Pv^k7dV9(?1Lp}_w$Qg)}qYB zK+3~ax(bDxM^m}N@Q-1K1x+4KOIFfkY6qDZ@+yv#O>RMv)vUmmYPG@4L$NLEweS9! zZ5hPs1*Xg57Xf9)ZMjXIfUNyUH zP{9AcB2g~>Pmvrk!T3tn|I;H}Yz`)_!CIL@UTt0FvS<)0xIN|xaf6O@{*{Rv2&1s=-i;gOum;BFH1ROR6^ z{3eb44Yj`#Y@5LpkN+)G8;ty%g9(2hr9z1abY#EoY7=Ef46z%>bArUn=hM{Z%L!!^z$fvmUpJK z2%K+fZSD0wj-~+HG@+JO75p=;-zlN~H@4OWj2NVTjk}qY+puq}qn=#m5$PCS%G{QI zA)=97pI{nLvtDX1`b;>Qu`(G&sQ4TWd|Ao1^>k2V-wF`E#c7+2`;S>FO4bX>@Hz40 zmJO%*svh|S4$GyE`&;`8G0Pwtt5`q0{skly^Uv6k4RJU;cruk8CNsjfC(o-xxnXo> z?ZD+7mEDCLWK;gc@i4THc)dh$P$JB$g3z4EEJbJ@P|SQv1U1hDaa;GBqJ*DI9;FJC zVacziK67ukBXA$dhE4Jr95=;xa)(y(LY`*hM4+Q(9coUR;Xlx7+G;__7d4v_O6rdN zyV(&f!N_p`1Kkiwa^n(hV)y^ zyKLCwgvNwlEt}6}560OK9T>-g2iVqRuMJ%6bc<@1wISH%&J;?NDW%W6xhjsPA1vz9 z*u(EA%dn|OQ^m^}K|%V=WB;=^$K5D*7X-$S%Z62}3atZJmb zqbNN6pyKnT;kw7$@Q$(9tkY+SvEHkfe5FZe1)?>U5Xr})&3>mE+_x94|9X4&WM!ey zS?gF8?=KAxyu`yhDt@Nw-nTzjU;aOIXD)SB_0xd#(v3 zv)IYmHT$=7DF}BVP$zh~%5rkpP&06o3YbuHkI#OaY8;#W{8d>bF@y>mdm2UeZKaz= z(OQ~5HSN7046>i1DaT#Z@5$+{I-TDF78px@D%`Cmy$`b&n|nLnpVOCi~^E zB45;-pZY{Ccp=5DQP4hQ*@cR!p>@5PS4@tTS!b6?%qLVb_YS-S1el>uGxE# z{H#4ytu(I^R~I%z%?Gw2R)1S7*7s@+Y*Pm9l~SWx+g$yCB0XatE4F0^AjCm?hut(g zdzVD}oF;d^j3+8Dx3PH_U)kV#&u^gXdBbymY291a3l5v)Cj)gRSBWRR`V5Ec^P?7R zWB3`|3K|3)JH7Ioz$;zcpYOWd_>^HhIcs~NXQ4B-`jLOd#f-9x{2FDH4dLvCX)cheg@kRsl*(4ESa8Avg>Wv#?BQ$dmLqXbFUTSx5MlmSUf z%Z@mw7asr6_preL>ELniY-<9HbGan&7Se5( z<{3l#$G@dYDW@z#%ZUml3rih;<>tz7OV8N#ht z-AG(WgA7}dk}EUmPxYTgV%%~Kh;MPTO#muOnJVK$Drb^11~+bnI+2^v82M4*Z>>RI z6ikV6s|q?)jnWL(Y-Ja&Xrjcmm%spBj@C@muURsj)ttQxsA^kWYeCdBn+gwVuc4ua z&4E_J-}ljtXg1cIZrVtI4R1x{+lN`OrP6#F2p<7T zN&bbO8qB8SG>|q;xcbzqQ1tKOMeS_18j3Rl1zOPGI61~gEH*i1(Wx~b{A*2U)LqsdUV(R`jA!_xa?C<~N4F(|-}%{a zz{%vmxD1hMyqU@5w>zFaejQ2TTfJ&R-dN}S*VUfuyZL6r&~Z0OcJX%)sc)v~K1w~T z;v!6S3Ltw97Numo1efbvulSd#IV7n7+hMk-v={d-oeFjH5jGoH*0u9=(31~jUS_CN zM6t93T-+CVJGM4y^~+s`zjX%5#jUCJb&IwYnfk#t`p&P}0X$p7*j;IT9=f;pWmVqk(-5u(9pK~k!Jdz;I8GY8%{^#;! zf$qOvl!uuCM2W8Gb6fh>)RQTS2N$6(&Hq`fkw8|ed$9e9UtOQ9)T>NUf8}@B)5nQd zbWRR6-VI1~^8DDAZ*=Q=6`Fj{^D8*rSV_uXC62Wwkxe2udBmNoe<;g((#1^e5=oTs z16+8CC*@l}dN!z&Z^7Z9WCN94Rjr*LmkIZ*e?l9Z^Pf!Yq4I3Re4R`gp|^nY?B@L| zjHq(C3g>?(jtv{kxg+%kviM1&789yZi-@A?-%Z!0%G%Vv<#Ej?d(E|MG`3$$#teio zy9Q#k>b9M=$aa1e4=e(?Mz+xSams^F9)=Aj(()lRRlnxqXWKelCL;9uWo4>n!Gh3z z&r2%se{D~BQ2z2*D=?S(Lbzv3cwgI~7tefOdv*Bcu>zMbsals*GHXZw@`xxXJbhp3 zN3w6|8(W*~*ag{<)WlSsOWyx@I13b1Tp7@~RAB`m0=^J#+fKc&^%UCM^7n!AjWzDsLBo@tkO!J5(u z=y1-demKBSs7L9qjLMd4dm+xR=IwECQ$zi8y}DeZIMZCP=6R9KrZ6PTX~nhYCy8XE z&K=U|J|9bYcDou-`GEwPr^2Hy{H+W_tDVLUf^8N1o=j$Jm|&Ioa_Gv-XdqJ} zo91sERqxr64jR~(iFWT1&z|Sfzw(AuTq_y3y~B8=_T%)DesO?U$|ns}dUc z0cUaGX>!t;ppYfT)4*Mkw5$}IZ5b9E=*?`oJ} z{i7Iusmh#_a8%!{Q)RT%{`={XT68Q(^GZ3GWzg;m_>#IA+ia+MQzfSM>t7zw0y*LP zti;q?m&_d#4ppGyLv7X3;nd{CL(pkrd+@$iet7oxa6jk&lG?&{N{(LrWuWkAEsXl% ztN!H*lOFoRDQne>>ke;-tGp4XN0+(9y8JNtfaU?9`$&0%Ikm))(7a?FvB*b8<-ACR zsa^5W-|B~SwTtTD`5TJJL9q-OM%y$V&yNmMG{*~T<1l!Bm74ww7T=ABMDh(iFUBbI z#8waXNS;DN2W&o+cYXG-1)v#ik`O`9Nnz8sOLfwf_7=ezN()hI3mZ4NzOxDuO$*XDI`amy3yiqD!8S_Cl(_vAXzl9Elr&$5kC~(q%jN>ll60R z&+b0po$wWVhYD47(RtN6r=Ql(L{1Y!?;n01(LVX0|DchotgCgO*{#h&EuY{&wns8plYXM24~>vuZ*3&DY2 zKKY-;oUbiAWoBpq20)#zkgWOnBuEPsQS45Zi8R3o;f&3KnrnO{LD`^+rnwKVhRj3% z+tIsC&zUaIWPfPxeoB8N1#mFTvJvnkLvdJhwLhG?S_buynCN?fcF2f)U&FW?BB4se z9+CzzCvjM#!<$(gc6)bTZ4iqWO7M|7I+FLWf@A1Lz7;}H8^*4+=i41~TgkQ6ZtQ=8 zIfks3vkl6uel@ffS<^n8?L*vtz~`S`n2fau*->GN?nR~J&dv`^LF&p#1Ao@!j-kbe|jTv+AR zck8KBQQQlO;z>=vDtxQC=e$?Cv7FR))Z4hfn~c}`s@tA;9;k=K+|rv_w~9F<^U9=5Y2=3>I@x^6WYAKP(B?YeDO z^yG+1V&PDz%g0D4-^52qg=r~spE`wb2@W$Pzj9hT-RMzO%-Y|W=Wz_*b>-^byCu^y zT;Bzz@I7XA$b}HFIKV+B%llj{!v=?4*sY~hK-^9zFl=?WsSH&d_Luy4cL8LqE%Kf= z9}-_tsHB$UH>mQfI}A;zkO7cgb!wE!W!Vtz>2pDC1Yl5bKnwV%%Q9fGTwee=D^bzZ zeW-mu$1`#vVa>mv1WAKDWd^Zj=!>G{SZyQfQ!NM=hy7?j@Em06FyYLUYv$bv|9|OSp)d@!_?mLmfSf+_vx_ zgViKO$$>g)HdqTiI@Yei2Ur1NUIdWu=!B*+46`7Ii7D%`gltU%Y{k1G8>}e~>#%!O z+ueb^zO>ldm$mPVS2|(azbddx6F4&|RmKhl?OgaRAv*N2wgH<)lYP{d601v&BGz1Hxr` zF}PYE`{;b1&9_}Fsymr~Xc$gpMMChdA8LFwL6V3;Y*VUN9cn6V^8aXu5_r~xs`VHz z6XA=@A)Pp1-)MM+$|`t;Wf37oYb>Kh;FA4oO9b6=lJ0+ZXCCK~PyP~^k!9CrpU0GS zrMH}9{g!D`XWFQ>8>Ut9bvDBIXZ>+ZR^Hk%i3shp`b9$zOzB4SovkSwvz0f^axbcDJ=5%aifM~$_>2l(T%+KM4|e^ z%2WkB&CLv$44J4_kiQKds(KVlE3orX+58Qd?TPSWZWQ0~_pqh7v`odiXUMvr6Rb&~08IYP@&Se^G{BM6? zxO1JC27-T{tl*^?oRRIl;U381_pO2?hXZt6&t*`TXA|iQ}Kkx3affVwH^-Nk952`?f0EQeXbuzzt%JJMIEE>lwPA zu%JEeLQ!}hGVY>wOzjo!kbAerAj|*|nIvHB_{=wXAGMbz&toqcy04%;$(=D{Yku9{ zVwF7J-w6zb`mVU zIo7>K5A7WL*Onc^qo=2@c>?1nB;$gS})BTl_m*z4ASo{ZcjiFPtKZ(J9G?_G9B?C(V_i|?KX zw_bb10ooI`c`_o5kNY@{MV2oC#G@v~4Vw+9wh`l(Zy3xJzw40CZ_^%EcJC1_ti9jz zhO7yWHb2R8CqUiad6hs$DSH*DE|GhxE$z=jUKF!!63`7J<2d6eK;1#5cbwkaTMN#n z+SZpeFVRcA+*adkeI*_jmc3mSuG5BUyqUy@=dKX=$+yHeHCyRRX^rF;^QQJepL`Bd z7^{i5vz-?yuLneEMW=iy#3Du4-T2lCip< z?E4l8UMuC=tk+agOVS;T*d0`QB6@}n$$YHmQwSPT>3=#={UF=g@F9-2INU&=#7iiG(Dee1SI^cc4u zpmr=9T#}`Lo7QGPl4b-Q7jzj|fu+9gsy5hz$E|)1p>~O}rE%#8Ja*cp zu^^JFZT5;h7vBM|RC`bEFknRXBo~P%g_N0Mg7q&(0+W0pGp_Sr zvbutvrV3KR?#}D7ZV~gW7g;$1KF#u25m~7#%w*r#=rtL^iedD>dQxymxc&}{^P+Zra1FvPc88a8w1$oY!0t%xhjQFP4-qPRM z;Qd~=FhBhIdwoC4A{Bdj&X8WQ7`DGyZNNK8m9){uH`yumzW4BWv{MO#JH4~B!5^~g zy*7+M3|&JRKP?<=z2f*guA0E6`tti)Pa@e4fF1tN!B$0`OzrMGZ>#M;7|F)>Zl!<# z;RQW$o%Kgj<|*iwP*AOGvAx)y$`097!`=5xsPvJ0Dp=NsN13QNJvuJc}V^9 z3b(|sxyRf*#7yCqQTtV`t(-MJjxIDEg05}#h=+h?Yu3J(i!lr8MVg+y#j#E&-)TGc zn=tgBDqf=bL8DX+y1Al}Ls^9V4b~Z?13kH3?k#pF7vPtQwb{VaiWG=3u5^% zoVf3=;7RWN6X@2LpN+0B#{9St{t<)QVAt+_&1&#F)4yVv2X(B>z< zDA`omPK$&!rf;UNMm>HUqwU|L-?PJuyH{_B`)SwmRU}62KSYa-u=cmYvClQz14WWRS;bC z@M8h#i1&g4dkdjwCKSgoSYA-|h3R_%*Mth9`W%*nw~iN~B^S?s$tRz)Ij@E%{fEv2U&KfG`qHYNEVZsm;Zt1E5sa3|Z` zQM%uXyO=yhGB@?W2bBDX;MFNjJ}&F?s>JR7(XrOVPPol1BU#kI{^;M?{UdRGYTeKQ zgt*tjv<387`iyzQT}5#kRw>1dyt_rdg|OT`roGHVI(r#tk>Bxc8i zeCb8%l_W}e!z5bxV)&4+P+CJ$Lqu_R#%0vOmYeRrIN6lG=p1R?l*wN)JMhk8La9&) zWO_8hT=I=0dP~)ulp(dUpmMUs;bWLBl?2Wi!?^4K7Zt7TJmVP(^IQV2yH=p^mgh*6usS4#C$&LF(kLpEA$rgj+4Q2aF@VD zzi&2<3AZ$K0%6M#_MuN25LmR01@m#@G6SiTT^M8 zBz#hqkl)N&)^xCM1qs`;0=4vuPB1Qz*MuDKsrsFkO<8jj+50Td;X}EoOj7JePpsLs zi&}R@itR?*dfl8+)ueRKTG6+iB!nxtfr=moWD9ka9d4Ob z1l}G>Oa;+V4KpR;=ML$;`GTz4pnORq^u7we8SEzZsTo%REN_poXv06#WN6uySDf>h*{ zIH+&5_i*@!>l-b?Q%YEEtq5;$h_k2-OLd>KIdJE;-{QNquye>VdX1uzBiC_$T=@)_ zgWthr($42C`>LNlbI8oCy6o1Z-)7>UjCNWxs@_{p`UfDMQUfodylNqM?UI-jat&x* zyIQ~XAg<`I8s0%EVH^MIejilM{Bo8{mhy^lNlq_2`$N2hT&x`;qJcd>N6GI{jli&$ zeAv0r?Jy5Sxh|f z7%y9mv$;D07?!F|X9EEqQ92f0QkRPM@j3|1a1;zazm*bena&S18HPxEDu>|iU?4Ox z5mp~KW6T@WrK+v@-cIRas&dBhpriXkhWf@*R%#6eS?5^FKUyb}&+DxA+Q$`Ywp8_f z?A#{nq%Gq3~XCJe~mCB&3889(g`^m%XIw> z#`Tqqioqy%T{D%A$}BDW@SL}eOW=F9C3MYfdKu-XmtS>?M+*;3no+jmRA`$n(~kUh z@mtE(&#;#wsc~^&Y@yk>xyatRhxUbzTYwqkJoHh6_vylRG+xjy#k#0y(Dp?mThNF^ zbkX^{J#;EY0xVhlSr=f(ck znt~m)#Z~=+ZrbaP(7(;qdH-Xt*z^U%k?LfGnP@2xCKNk0uwbfA56L>JxOk z;=DNSaPo(anKi3lKKhwgEgubUOFK?t@s{V-{(I|blpY+sr~#9Br%ES1Rwj5Dg}z$S zoUfHAk*;w!3U>E5qg}V)-g)(cbRLzWYD&>&MDh)mnqt}|M^gnecd`rtZ@m9jD?dI> zmIm$<6B}voJpGo(!S%AQ{EEJER7NV;fMCvDj!!M=22@&eJG@xxV=q22Yv z-?}Fz?GGb7Wr;Z~KC_x%VEOzWuBn zi17RRi^<=B+`%jY3zE%wLgA2!ZaPR2v26(%%yuIE1Q8>NEya3$z*^#%d2@ktkG?sN z?&hF*@xp?Zjs^%~FMcC3%@e5AR7)D`WK@jidz7w=$fx}-);K~H=5$0@ZupC}qE_4Q zlh}1Pb{G-)!b0bTYGQPq@I?b^SJ6=%Xg3IQ#;SDW>!_5aeLv;7kJO}W$n#u>O<Kwz4JwU&fH@fp5m83%_2u8>#qo7>^v1D%*+5+Z ziaKToCFzwlCn>_5-Y;}ZUMF=zw3*`%q_fT*grqW$QC`h&l7tl=rdouVya4LA$&MvD z(#Iz!Ezk&B&p${_2;yi971Uh7&VtR|n{cR0CzV{)gK$Z5nzKBuH1uz8{K(%J%?iWu z?>k-cK&jWIKvb_76T?UoQpS?}7(m&afoAj^`N&2PBUBzzVzpGrDTXp5*Ar!Ojs8$@ z3?hSP8Q*u9qm!11Mb(z>b27^TV_8oL4~~cDr#bMk*gKn^TlxJAddk0rWHT4%QLToc z=0TDW4<8gb_P$A`$03dyJqRqQQDFQE2jr3pir_9v^{HLAq*8F_Ll&L<$%)hs^BMny zSNfo#MY47zy=4aLUtnd%p%o4?!a7phXRja-lY0W2^6owLAj0SfRvu~8>#+jp{DG%DX{0ey*ZlKW-|2|uQsb{YjGI!Xi)XPW+*6hd6p zA_+9Q6rsS&YwjgWCTF{6Qmr`Lrj#{vHi8*t-&dgqT?Rl>qk16QUQyGsZI&?;8 zHCKWI9n&grFs;?=0RwkKdtgE`D}Az|M-5!Q-Bh9;Cpl@cQXKx-k4}>dqf67< zuU$O=qltM&qnJ^rWXw5g8El^6x~p*#vUBfxxZdjjwZ(t&_Ed0YojTq7bLFeM9@t3_ z2QLoFUNFC|%n~G)x)hWyDu0;Ilf@vi$mrnCUZ80-6J;yzFQQRK z9Q{bTP8j$Y@e{T}aZpxoaX`(kgS?2T1E(od1dpNRtvx68P(Ftm z9=_*|9P!=#QHa|;R-EYCy!I$JnBa>*3ZV^mgx;?Ix>#)lf>McYe!?{;y81&~+MkCa zTh+ky&eq(NGC&++Ni$fgSK}aPj`*P1ux351n62SW`j}1IT3YsWks7vB16qpss@gmo z=&u(BAYUpuCP4$)ZzAOXx#UZc`QQN*3mFuDM1pN47U-oealrakWA6|+!=L8nm1zFJ z7BCG1DYHj6k1?8axJ|?}=ta5H$}jV35~KOUEKR1`InPd5^uS*?&gI{b9OlV_yZTUU z4gasUGC(I#=5Vz(N4av(Gb>{ROAzzH;yAv%mj4)scN`6G7Jn9i$F{M`sf0(`t!-;S z;F*%25m%3Sga9Pb$F%%gH3wGzV4M~aqU45e&X z^ytmFnyzs^M>qH`@Tma(xJL=AA!&J7tFf{qB|q+r;K{L8PHQNb?~Y^{4$9bY$*HTk z5njN4+(pCcd{~UtDN(Rb@ku2kTvb~$p~D}Dx85j%0L7CcP+yv~O@E+yV*2biA2x!! zG)3)ug3-Ibp3$eHza;4~zRp6bWwWyBG`x_I9Y9a^8BMu$WD+yP6d@Za1d@HV46K>l zr%yMh5}+b|MPd@8U#tJ%!RN!59jwzgnAL|8$FwlnIpgWdIbMHew`g1ti|VF@VX3R6 ztr0wbv}__4=_szC@QgBPQWZV^${lat(2>A)yl+r2os=1P0dJjYahSZxKbb!=R7=CV zB8gYu=Vg(CSSEoI5N95wJk;xnPB7T(i{2F-rbJv7M;8WFmjcXQrs>wk12Ax+$I7ytArSIr5iEE$7<%nrG2N+++c>9=wNN9xL}Nv)clGb=6NU@SzF7} zCpVpl95+6`bN(FdmW!I`jb^q@{jfyd zMzFm)=gg7Rva1aIV}^Cxv1#YRYj${Z%HLbTDq|lzSYm=Q1~Ixgr4@lnOrXjy%YDTP zXVK(2onOYjreqjtaB58%0F!vDnpUhy~JvsXz5)y}tc?1>$J>MnkFor_T zVbOc`Jp#|L=+ASp?iJ5rgs2s3CuG*R<2EB?xvligN@q|1C?w^OCFWnK1pl&Hn@o!y zWT!Q(CR|%1_mAsw{bAO>b%JRuW7oQ}icCKM*|;~BmRq|bXJres-3T!)bo%F0bJ@DT z-;VfHHNSJ48^!ImA6&hpsQ%*cH4 zZ`+HQ<|O08nC5&kUv+p(?594>3xNl>QX6cHvd69co#pCuU1p;BPR;05$;phdnn-@2 z7)k~JR_qmqe(jgffq7*c5ko`7A@cg1&5zUausUR|xt;;I^<$Hqdx-IRnhkc#-C7hZ z^5bHgxfJ4(E~c0RCd<+G1%^|r=-nv{Q4`ezN9*{`&9(jRkXxX`8zKMzPM84zsf(s2!rNW;vq zuJCI1Zi?mH!4j6sB$x08JRxBmAlZ@+fLA4&hAy6BIbS~;%wb(o^u>??^L(bS8>qE}kG6#z_UDPv7F=HA8JY~~eWSRA(9f|U6CupO;oeV5>Uf}!}i6+E? z#JVumbNV~8fK%!qFtV0dSf7yO`5=y@PkqF|Ka2L;f4d&8;p9q+LzYTq)#qqW8Jp+0 z9mq2-dg|Q9zOz?8D}6ylOCiw@<_6Cn3iuTGl|91Ksx#me035uzWCe}AOj(rGc>XiU zou07!uEF+>o6Yw?XWNr`kS>)KF&ELv3XKZUT$O!d4 z%cnqT65$2p$Br0SvA}BZU|gj2*(x%)Aw*s8DLMV^&B7?JCa9?19kaPPpl`AalR zO#-n8T>{cqCV&>jXjO7>B05wN(ll3*79MPb=WpJ9Pn>}x8g?+l)0+joGR8l{b8NL3GH|y++|IAE1WqEclSj0vUlQJ^_Gz@d&Ic!P zdAwFgE7ZF?MOA;g+ft;B@>?5GYWjzNjf~fXd#O;Eien7!1k)X;N4@LIAV2O<9~=XM zJI%0ea#R0Io*IJuJ6DGfeC^hCO6_QX)Bef_)qj3$pe zmnUZ2;Zvb#IY6J_jew{TDaJ!M8f-t!Dem`XBoWxK$E2{_xPP*A-9WH!1202fm2t-W zZKt&qK-D#csZ&Q%?K=b-3v$p$pYY$T!d zhNlo8UnGpVs#Ke80sl6N=KmNKOry6&jxM-`9J4>uWFzHFnJ$)ARE%Mbg85=;4d~k@ zcD0Zb{obw^V(r=bBuDI4UohSG_MI3!D;^PenzcJ?9K%Ai$56SGPbL3+3U*= zX<8=(LsHV09-2^8werrDID9WzgZMN0%yA`->*3BmJ~sdh$i2VA;<{d+q2t;1RK>Os zt@^esqWV2$@8~e7)cd>Im;iQ=G27AL&z=Ej%EPbpV;8S4r6$Xb-q&={I>ewsw_>L@^{VROqY6Psh`O(E(w`@Bbg|1G zEp)`EO@_Hx&)lY<4qwi(?m9p$WEUZz7Pos|+A((3ZwguEap}*u`(C z#$R`kxhd3Xil#j9Yqz$>FF&5O?I1bg>SBr})2p%{uf8t}TVe>r}dy0?ym)LI2!mcvjRfIh%}h0+jMwyJR91qc&%_ zFKYs#xE7ub4Y@V6JIYswcu{E<5*wd-X8n?vxuAq}3%MUJ?9Ld7HBbuYMu`0iV_x0tEF zj#}K-(*-u_EyhOgcbx%0emC%xdS8s5S}DMf#O6LApzt%A7rABYW{1J4i*`4Go~!Bi zqR-@0keG)xkm9^k5^Cpkv)6KI?tDS$)i)j$=s#bHojL)dh$Io4R*Wxt56)VvKTP2j z@-irPN}sL*bcD_RDVbrDPjAj6m!)FRG+bGlgH%U%ap1^i48M8saoCCLdKZgKlpqhmbB1D=O}Hj0W_2) z1m}OsSIH{4?2&u+!nS#kcK7QRD`;X;8sSDaH$Q?h^`3O!y@LTU?2`!83oY|{v&f3@ zsPD@(C)KxeELi@Th% zL6*G`?l?YL(mU_^EJ)42V=={hRRdCtr?1Z61zhj9?Oe2oo}!8p6m7v!}v6HHOe5A@|{-A({@ge|6kQ9hRrFSo&LsnMF@?rZ$Pmk31bm%i}snO`q5Q^S< zU1v@<1ARGXnm%p?@i)RTCLmXa%_@2JT#+6go1U=V5$Z<~VgaUXchBZUCckz~h$BJH zvb<3l-gD+!n!FMArn=dB*Wu@E-FphPua~rQO&Sp;sa|AKM!bf86@Aj^V*C$M7qp`> zP(I;bafuVQk&Cj&CZB%~;5CT_)V0{$_I=7G{s8(Dmv`Q?mg5H6H}Ll4oJ2epQHqYdS@E^C|xs_p*60daBUf`!^h^rXTffW!o)*|`CVzmTF zyzt$ydmT{-Xl`bm;ChRRtE2&eYnZ@B_BJWgvW&)@j;O`R`dyD%r8YsTD7|Bid%mkD zt@Vyx#MyU7x9vjklW%(+-zs8xDalQb7sr$D?)JW;-?QSS{0f<1^}zGbM>ZFg71}zq zZE}dz#VMbaKDxlJO@h66Q4i{Yu4t939M44nDtLE`)10y8>!sc+4GIC}vqT4J?+>!n zG?w2ZQk(q5pN|Tm_UWgr=$Y0qV*RAu`l#%04a$(=PXN%m7+j;;N71^Bo6Ea#WU3mu zAw>#bh)LZzxppbMsI27L;kNS4AIwI|RiA~n%bx^yZOb2%%r`wL?6{V=zLhdG(sXK6 zneuEz#uxRV#8`f`5>|m-pcBkq0CMA{V_X}MKITWuJhd4L48t3aY0f4|M*kyNg@hy7$v-?5x3IOh^-7@Jse=$GY?g*tC-^jgSCWv>UPJ$s&7V00D1OKL zOwc-iVu9^Ae^FN8uN?lj``hcNd?Np0H()r0H`VJ_gw)TWufSlE)X?6q{0_)rwFp(n zVWQ-j*dwU*V#r}$RL%UjS`nM1vxCkaq>crZL;6-ef9W8g*kV^g$kzBwoq+&3Fbg#T z#B%=l8zoF&5PfvL-OIR~!~-OQ8!8r_&ujIstr5JdwE7TMtmUwimSO3Xnvp6@;p{>N z-56g5;q`vIU--Y$1irPB}xFX-Gphqyh$}lci2b8)1Ez;Zfn~k>zQIz-9=*DZDu2S@*fVYcOtj@1DinCO>*x zMjGo2;o;te@2;b3)wc}Sr7}z<+}j}3;Z?I z-s31St^9_TSl%H3S_qfUT)L?WqgPWM3RT|Vv9E<*X%I053hjRNLYLJViys9^f_|Ob zA&HC9gM`bxZX>5@Ztu)%)GXB` zb=T{@*dgYXTb|+PO$ZGmSWs)*3j?DG@4ojR`ASbHW2%$$<+L}m+)D6z z!D*iTaH1%7W(cTLthD5C`My&zShupEx7v!egw4FI2Y>0Cv1%kg`Y7mZHZ#Oyt-uonlmmk3hRf}UdIT?R4iC@UT?w6G;soj*F9qxcH|iYv)V8 z&hKqK^O2@#sn_X&UUH9GRYOlDIqjz!+K{-*;{AQ6+`o5!$9Grte9V{M#Y?8F+!*c! zMerP^9x|J}1Scu0W^_lawH*6h${XG=E;|(07HP9hv5RdKh4UG=hJ7B)#He4Dw-5Wn zRx}Ima%eg$GE?cC?l$-pH3gDJb`f;IqDt0``Vh*QHQn@wjo^h87D8( z?fGWEZuu030J3X76j!ZzctHxXM)f9P*AO9&EeDsqJIqAd@X5QfNVhRRZlzk9!>ytK zLXICDkxh~!%;>+oWO?tw1KO9GzIaCe_jH@nfjy!tvqL|i4`KmLP3Y75ML~zo|IW=P zx#1U8;o*AxP@X=rllPI+-eN%V+rpP{wsge&C8oV4CEHG$5qF`4@6mpNPT%F{^&Pga zPLSYK`AEZ6u;(D_Y%W$CKcJu2z1U#z^w86OeWHBlNz3!SsaA4-S24ioDepmObh`6M zXrp@8@5{sKg_6WZe%(VVCrt<2D18-JAy z?I1f9FN4N8L5CC`i7?(+FDKoh3`>vCvgM?8(g;kG>{3H2Expi4i7iV2@Oh5%(xsZy zUn?GpOkn_gZA19?hBv5!hAnoDvb}oA&a#gO_##Pq>t2$djq&0GZu65utJ@MRoRYaST$w;V`@ObZ=K;a zwa$_j`GqM%^yf=&=kG%DVu6`)B5l_W6;h-60Pjx$Al3SSAD`YTC|?(kP>T4!C1Ki$ z(a;O75ETpcO`j>YHvV7`fYP}+-+-09S`C054IB~eOcT&V*go|vh&m}moopB4w6BPI z)k$_(;OaIPAk(fP2BDp5@F~Ya$W}S@QN_T844l=M#jEm|Md|+WsddGvxT9yeOQ{67h2K<#UawOa)W=nV27ZUG$1}%|pu{?N36tcoAJ>fl}AT;PFC7X-fu6hA5ADq<_* zD`F|)DdH#+jNy)9kKvDDjp2og^cwv~0RA|y!I#0v!I8mo!T*Apf=h#~gAaq#f^~zJ zgTDl~1lMRoQ3p}!k?2wAk?B#3kcv=>kc&_)P%MxwQ1_7bPxTWt-l;a&j%?F2vUi2YaXu7zm#I)wzvRK@So=^pj#s~( z3Dc23fXJrNej_%sT9s3+Yq>&hm0q>ZYcv&LJ-ZXL$I)Q7u$LFT)c$;>FuvNcCR})` zkQAXcUa4VZGgG9Lc&9~t$6eev#w4e{L+{uw-~O|X?$a5}wp$;GZ`-FDwrKE_e|9H@3ka=D0 zdgJ=wpqZPRKN-q?S6G4lP}Au*z7=^}u8Vu^_|eBw6!ct}IKG=L@c%Jp|wd zN~SmGZsL6iT9lRXU5&NQ2TPo@cB#<5hAoH;Aay?%1>Ic6LUFP#1Nc7HNX#o-=bZk` zah*Ww_6vZU5jsAnmU(`Z5&p3BkE$#2O|Qmou+eX}@kz_8=c}cc4{39ZKLmbj)gAJP z4Q8t5e)UfP$L26z;?ViO{0+L?COgc6IbL7Av~SZCI%J?Bb+^i>&Z;KIcs;aw3K5S%#Ti1GSlER8AFa~j5`M^H z8UXYd{p&A0{P*3Tk1zr>;RJO+i}%=F^^*RUPj7E9stT}hxG*p|6t={=iy`LWH&Q+e?I_SfgDb5Kg{i{-vu)MpQkvC zO#jclk%_f~nYDwZ3%iq-^{Jtc<7y-Rn^;b;PEeb>Q=ModaCo2tWHMUHDRY&)<1V3Q zRsT&$&s;9?pkSl8E23ayL5Qr%83JQVdffi7{e@QQ$K(_b5@teMNAbaUIlrS(ub7x( z{39TZ2|L~9@%t>D`n1LkChhqC$kznL<>qg{_o*iE@_+WzdQ(p94%9_k$}+@~%nlo# zmy$q{o-kI1;95kZLj25dUdLB%on6HZVqOAaKAl!)D%;i5eS?OHhjkUEFnP6 zNIN}Im?XgXD;RMx2=j(q9U2ixIEhMb7!ZBZ?O`y88}ezek>&rp1Tkx)Ug&0nIMdkq2^v_OFgr3;eP7V2jSQ$40Y_5rReMU>lBk8(aN;hrTx2h zC$(`Np}HEdG-AVYo-EBg_(g#aUoAcj{CN&s2$eu2MgDY222J5i3jEv_h)rt(2X*O| z56t=WH>jAJ*|{b+4hy4dA{Nc1o_PgZ;nCxdlfd0{6Af=Mb$mx$< z7%`+dx_J#V(_mb;R-JL&TZ`Z-ZTj!LIBZG!18`jG8Jr@N+aN4!7gv_!S&xfiSM-kQ z_HDuas8TL_7gdJb!h(zu*M@l4Pk#+0F3^cSJ6c42Hk2Ky6mI)w{INvp%14(s;Zj+T zmHC}!P(v5-Nn%h#UCHo+B=5zY9)fz|aTsfTsRyN2YB9YF^BYfCd{9g@GFQr#Mi8bj zBUdx+p$85=LA=fIo9+i9VQKC|t=12iIZYAaUIn(Z$?RaRJ}UCTqZLj(7xR$==~)Zm zo$!h-?V`VzD)m!n0l^>buvkyJ*44q= zKjjcY`$!e)F|J!ti%3fkBur{&V~PYrSrph<4N%ts(6|Q{XeuYm`L^!ZDya5y<4cO@K#I)H^qfTcpeAzZ?jY zm57Mf-9GE3%(IFT!K@aMT~fy!y7VDcn)>*i(U{8YdkvzYKLyuE*}DV+mu~U&hm;iM*R@2@Q9q3Uy$hxzN72eD_G1uTc^yjamLn3}7`;WPJ@xXO zpVJ|gsj3E)HxoT@7>b62J;6EgftA~=v*LDq!wc@GEiC3dL2cK*dW#YJtPhKzmHW&St;foU1%0VgiNeyq<~I1x zN83C*Q}ch~kN)zsl_yp}F&=u&k65`$U^x@G&^RT{-mJ+0I8H9=Tz>_prM{|bp_izN zY_Tv_+FC!d9N=G^Te^g)-vmljY0#u z{aDJFlF#@vJKA7UtgHEJYHBKS=~kcdrz;JDd01K#nW#oz)n#S00aNl5zv4({DwYT5 z4shToHXf`Lwvl0t1o%-r60a>lg6)UjNY~$cjK;O8#xtPeF&_EJFa~oZ{6m_CX@5=) ziVj`90RzEiypmRT_E7(7yEw-i`UJFK`Glwlgp9wZhd&HRWNfS`ub}jGHOTP)`ewrQ z$va$Uv>na&yj!fLwXrH*q-PVqx=RZ=qFoT zK~YUu*PQc9-&0HeM*U9GOsF}bjQTk=)K;jW^SWW8@bUF7YPpZX@y}fU7v60}ic2Pe zxyEPf$C!o)N5YL6#JP1WP^Z4oVr(#|Mk=cTXyI5&49;L(bN zflECo)@je0^z58l`KSl1efZO`H?Y>L%S&Nma9m>?`uSc$v6Az`;7Ro}%@YfMF;vq; z>$04G$f%*eM)t1n^L`wBGCKP*p?8HgKcO5Gwd%b%D<~Ylb_!EcbECO9kJ;?$0{svl zd2~1%3&*CG$2LqL*bXIsmDv)MX>Z$l5QZQ3jXw@3oiWpV`i=foe7#RqgpZ8?=)`&^YaP#lvoz< zA*7L(SZZlRB$kE+mXH<+X#@mB!k|MzKqQxtknR!zsg-b%1|@u#C*DVS{hs6T`DV|a z+1<0}_nSL&=ia$<|7T{0eyQu3JQDU&h5OmDmmAAULzuZ-x`KG+ye$2C zU;k!3el;8s=o~Pd`eE2{$hb!sCoioaVt+@w>;~C_gf%iuXs1e2aJ?@7mB^`XI%L$> zTD>&q?x!+QMdsFR`C*1mfY+{_cs3y;S^>5ymyv`IG|KLqIu#W;|? z7z{!mz0vAf8yH4Kn&@tSN;XZ+m-qE#JLLf4~E` z&mrVe&;kP6Ju;HA05}qLJe1>_BK9Jo13oXViY2Y#r!HkS(zmC0abQyRqI0F#!-|~z zkWSfzM;@ou*Lfg$?&(j93nxeUF4{W(P&B%3!4AIxXxW9I%?0+LkZ39jI)Su-AKFT7yKN`0hp;2s6M<-Cg z%*-JzHQO~q-%`LdDmHR zhN7Mnx_NaSHiG(u7py^cBAJ)y+gj+lb}$)69{VqKH(AQPOZzC;i@iIymCYQRF2XF& z^T|h0jl#}}a9}zwfZh-AWOgYDp*3Pwz0yggTIC%Y#MkI5cv`M|SeuePtPc8gHGx#i zE?>00Aw9?9xaTp?ZN&h(n^L(;D}xTO^x~0@q-U!srYy?N{?$J5sg{7%%a2Oy_16es zf``uxz|^0NM{1E-R(-;Udmf73IQXO(8$mt7&j*eI0ya_1W>WD32@~&pb!Vk#n=Vm- zS6$>s5Q&r9@9ru*PMF8xJhm+nqqlvO!tww$>zXVa$-q%DQZ{p77(2s`MI~Ne^u%=} zdh)9NvZWx*?4Y)dnr374VPj1drM_bPifJ5h)NrCK+mD7F)jU@-O(Q?nW8zoSiL*1R zlO$!VK*0{W!HlK4nf5Rrq%MPsnxbyF3^v;UUdyG}x9%lcnXn&Q)MCQTPVL8$dTVv^0i$Af13a8QE270#P(K%wM5vADwfyhLI-*-5xl47 z3UD^{ZgoW&MU|Ep-P_TK79xIVW)Cb()Ny5vHdIp?&0C{%2G z){>>p#Ae7pRh006Nq#Q|zz*{S-*|~#q^SOATp3?U!5u64QRqX4Q zl68Ca-fMwiW=8?edtT%$^G`$9DO69%!5s9$mY+?1IjKKX^ypYt$u-Yst($Gh+FX&m zkSgoV*4z37a`zU>PXJ{7%!$lXtj5Ye@X!Ke_K|P;j%54x_R>AP2`x(hvRcJu`y1en z4_aQ&NU1dG(jAB>+3ladzJ_retW9n;z4s~JU7uokYMsHmxAjRFN zL!L&s(**)q6CF%TgiJm@wJW)`mvby+L1lmUG$zRL^KTzy8ATK{V5F5qOwS(5NZ!0x zE)1xbjUBANH4*R?nd)-sf#ce8h2GOZL_TVf$^}DOY4@<9Jj_8h43-#QB1cXw#$1|Jq zU3>{q6~*##xQ!7)*&`d81fl)18)*flxz5d<9wtVg7G2zwBHCzue}=?tl~%QTdwJ5f zcS(eaf(K>jkZQ!fPR^VAKt6iwEOg`3)C$w#MjD;uBc>oeE< zbqQ6%_2cw=@l8fp+6;lZ1hzFd$fdG)M0-W1rnCmCEo{;&8Z%xmG55E8POL+p3Fq7u zCvvpoS-#p8%22}N1}=lF-7f7(sr{wkfMkzNneKl0?T|rvtG0FKV|YVd zej>%tN~qK%oGc@=!7LEnq66bNHQM=kYwRB^XP zaRu&PnO`Hb2u(mrn7rv#u^;w;dTYA3*6LDz@m1@QNhi4*XIs|hXKl|>k^}_LAuh*t zld(jOamgOVejVX?E~=(%HLZ_1a~o1Bc4s|P$z0!scg;(yXIuwFVqgU%?vCx{(Ab|7 zO9H<2FmT>X18pJF0~gE)Aga%)-_VI_Uev9khM0`7!}@r?@PrHSU8-)lPQD_qcAPyC zu^VA9))dfX-bYq_ql+yG1Prn)J^&ar2o3e4^jY~F&=JS9QDExM)4%L7jG=m1NMcm%vY;2#cGc$^4*f;bp8EpcqNd7M^-#jMJBalT zB~W6F0|vdqTA0EkGou%i;(N;rM~8yw}2eoTK1WU8(@}jXTNUeyVlDWIADBB}k%rydueL|rQ$JZ_$U2vZ?jOyUz;S$AV11|c4vuStP^xgEN z!v4S(Jr<#$4@}s8qMu>N3!CA-)OuCPTMXEuBiExb4&~)ru0SY7W5t;0B;x6l#yF>D8??J!Nk zlR>TLvAYrryOb5G9CLq&p}yD13QbcyNiZdq8k#kAIwm2c+KD zrHTHJMlY|{GNGF0D?Oe0C~Ily9R(-AU+My|A2oER=s_Sc3A*I0D$-B!1a_Bwe^%5$GhTlu8{)< z{ey{WCesvIggWSm#hj*)cZ`#mD0{6oClcz-#cDjO>9)1!)4{D3i?!f7t~AG z>A*KrL<0xS_LjPAY!l3ATW)j<+waJ69m%!G8?^Ayy?+^8L9)u6ZV$UVg(%9`f4cTbCt3(l**roy~0D^cT0O%O!9QQ%a(g+vX1J~^ArKyz|_b+gv+Zt1gze}cCF2R zk9HqlsTOG6YNO?|Xc6hl*zJl>h+yn!t5>@-V;5Fyv1#{xstbAcs#ajl%r~p1mteKn> z=wN(X2yjMxfRIMIj;Ie&UHA3U2Nsb{prxU~71Q-ZhMPM@Vs-304@~Apw%8^76oCC% zEHA0p#SAR^Cx_cX>6e+>I-y|@Yn$z`;;q-vH5*9YxAm}nR(?n&sM!{ zcFYpGB~lKjIq;@Ye{lnOdHEt#NjQMquqYb8##%ReQ-g?&(YT4;yS%6{3 zG2O4!6StOWEjKx^r#4kSuG#ekTvMAH^&b0RN??Cl9L1@21X#hLYGa=3b+TNY?K?q` zOsvtr+V#5K6WR~6ryY0OFH=d_e00*;WFgl)Ue>&8->?0W-rVR)dpnfb4E0eQhbD2S zy{YfDSHp-)hl;{nJFDIGW!;kCcH^X5Rbu$tDD=7)1Ct8tj||z5R=#LU@_We*Ijg?S zg#B;6?DwSEH|yZ@*hs~3p$%LBfan4MAo~aGpV%g7!}ZVDKhDKjIJi1^T3DbnaOb(s zfYIxW-;30r^;wl0l7lvozrf|@E8ywt2DSN?D9QzBdc5xKTB@N>8}x4Sh_g~ zTKrft=VCo$5ZrJ9K?DF~psl>hUu@_)H^dVC>n}+52cf02GtAo36XtQ2&-_-%y|`Rs z!c9#b4mA1znp62Z&QRKa!Fi@IUY-ul=PA$n_5Hnr=f9yTC)oi2h3_Z{IQ|W#vzw*0 z(>W2(x^>qhy(j}40Juj00DM1=8R!V`m%G&&W@m>^690fOf+fbfoE!iEq1oiW6EIEq zUl6*$Y|yKP?}zYAK8xXIt30$u^01zlFJI|j(0M@Jy`XT<9}r^K@GIG%mpWBq005fd zza=-MRQq2M!rjhN_U9P6(OUYnSH9yB3jk1?cwS{Z(5u2<9>PB!4>-qn)}@KeCQhhyha&&N!EqfY-E>UR;; pdGzx^(_iRSA>xZa)#g7!C>;$P^ppeuXwWZZwEqbFfSxvh{{ZPfCjtNf literal 0 HcmV?d00001 diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index 1ef5fb58..fc1be5b7 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -38,6 +38,10 @@ COPY ./deps/requirements.txt /tmp/requirements.txt COPY ./deps/requirements_trt_llm.txt /tmp/requirements_trt_llm.txt +COPY ./deps/requirements_trt_llm.txt /tmp/requirements_vllm.txt + +COPY ./deps/tritonserver-2.46.0.dev0-py3-none-any.whl /tmp/tritonserver-2.46.0.dev0-py3-none-any.whl + RUN pip install --timeout=2000 -r /tmp/requirements.txt # Finish pyright install @@ -47,11 +51,13 @@ RUN pyright --help RUN find /opt/tritonserver/python -maxdepth 1 -type f -name \ "tritonserver-*.whl" | xargs -I {} pip3 install --force-reinstall --upgrade {}[all] +RUN pip3 install --force-reinstall --upgrade /tmp/tritonserver-2.46.0.dev0-py3-none-any.whl[all] + ARG TRITON_CLI_TAG="0.0.6" RUN pip install git+https://github.com/triton-inference-server/triton_cli.git@${TRITON_CLI_TAG} -ARG GENAI_PERF_TAG=r24.03 +ARG GENAI_PERF_TAG=r24.04 RUN pip install "git+https://github.com/triton-inference-server/client.git@${GENAI_PERF_TAG}#subdirectory=src/c++/perf_analyzer/genai-perf" @@ -65,13 +71,15 @@ ARG FRAMEWORK=DIFFUSION RUN if [[ "$FRAMEWORK" == "TRT_LLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_trt_llm.txt ; fi +RUN if [[ "$FRAMEWORK" == "VLLM" ]] ; then pip install --timeout=2000 -r /tmp/requirements_vllm.txt ; fi + RUN ln -sf /bin/bash /bin/sh COPY . /workspace ARG RUN_TESTS=FALSE -RUN if [[ "$RUN_TESTS" == "TRUE" ]] ; then cd /tmp && git clone -b r23.12-python-api https://github.com/triton-inference-server/core.git && cp -rf /tmp/core/python/test /workspace/deps/ ; fi +RUN if [[ "$RUN_TESTS" == "TRUE" ]] ; then cd /tmp && git clone -b r24.04 https://github.com/triton-inference-server/core.git && cp -rf /tmp/core/python/test /workspace/deps/ ; fi RUN if [[ "$RUN_TESTS" == "TRUE" ]] ; then pytest /workspace/deps ; fi diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openapi_modified.yaml b/Triton_Inference_Server_Python_API/examples/fastapi/openapi_modified.yaml new file mode 100644 index 00000000..775a264b --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi_modified.yaml @@ -0,0 +1,2326 @@ +openapi: 3.0.0 +info: + title: OpenAI API + description: The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + version: "2.0.0" + termsOfService: https://openai.com/policies/terms-of-use + contact: + name: OpenAI Support + url: https://help.openai.com/ + license: + name: MIT + url: https://github.com/openai/openai-openapi/blob/master/LICENSE +servers: + - url: https://api.openai.com/v1 +tags: + - name: Chat + description: Given a list of messages comprising a conversation, the model will return a response. + - name: Completions + description: Given a prompt, the model will return one or more predicted completions, and can also return the probabilities of alternative tokens at each position. +paths: + # Note: When adding an endpoint, make sure you also add it in the `groups` section, in the end of this file, + # under the appropriate group + /chat/completions: + post: + operationId: createChatCompletion + tags: + - Chat + summary: Creates a model response for the given chat conversation. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateChatCompletionRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/CreateChatCompletionResponse" + + x-oaiMeta: + name: Create chat completion + group: chat + returns: | + Returns a [chat completion](/docs/api-reference/chat/object) object, or a streamed sequence of [chat completion chunk](/docs/api-reference/chat/streaming) objects if the request is streamed. + path: create + examples: + - title: Default + request: + curl: | + curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "VAR_model_id", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Hello!" + } + ] + }' + python: | + from openai import OpenAI + client = OpenAI() + + completion = client.chat.completions.create( + model="VAR_model_id", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"} + ] + ) + + print(completion.choices[0].message) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const completion = await openai.chat.completions.create({ + messages: [{ role: "system", content: "You are a helpful assistant." }], + model: "VAR_model_id", + }); + + console.log(completion.choices[0]); + } + + main(); + response: &chat_completion_example | + { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-3.5-turbo-0125", + "system_fingerprint": "fp_44709d6fcb", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "\n\nHello there, how may I assist you today?", + }, + "logprobs": null, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } + } + - title: Image input + request: + curl: | + curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "gpt-4-turbo", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What'\''s in this image?" + }, + { + "type": "image_url", + "image_url": { + "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + } + } + ] + } + ], + "max_tokens": 300 + }' + python: | + from openai import OpenAI + + client = OpenAI() + + response = client.chat.completions.create( + model="gpt-4-turbo", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", + }, + ], + } + ], + max_tokens=300, + ) + + print(response.choices[0]) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const response = await openai.chat.completions.create({ + model: "gpt-4-turbo", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What's in this image?" }, + { + type: "image_url", + image_url: + "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", + }, + ], + }, + ], + }); + console.log(response.choices[0]); + } + main(); + response: &chat_completion_image_example | + { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-3.5-turbo-0125", + "system_fingerprint": "fp_44709d6fcb", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "\n\nThis image shows a wooden boardwalk extending through a lush green marshland.", + }, + "logprobs": null, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } + } + - title: Streaming + request: + curl: | + curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "VAR_model_id", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Hello!" + } + ], + "stream": true + }' + python: | + from openai import OpenAI + client = OpenAI() + + completion = client.chat.completions.create( + model="VAR_model_id", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"} + ], + stream=True + ) + + for chunk in completion: + print(chunk.choices[0].delta) + + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const completion = await openai.chat.completions.create({ + model: "VAR_model_id", + messages: [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"} + ], + stream: true, + }); + + for await (const chunk of completion) { + console.log(chunk.choices[0].delta.content); + } + } + + main(); + response: &chat_completion_chunk_example | + {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0125", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]} + + {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0125", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}]} + + .... + + {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0125", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}]} + - title: Functions + request: + curl: | + curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "gpt-4-turbo", + "messages": [ + { + "role": "user", + "content": "What'\''s the weather like in Boston today?" + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"] + } + }, + "required": ["location"] + } + } + } + ], + "tool_choice": "auto" + }' + python: | + from openai import OpenAI + client = OpenAI() + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + } + ] + messages = [{"role": "user", "content": "What's the weather like in Boston today?"}] + completion = client.chat.completions.create( + model="VAR_model_id", + messages=messages, + tools=tools, + tool_choice="auto" + ) + + print(completion) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const messages = [{"role": "user", "content": "What's the weather like in Boston today?"}]; + const tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + } + ]; + + const response = await openai.chat.completions.create({ + model: "gpt-4-turbo", + messages: messages, + tools: tools, + tool_choice: "auto", + }); + + console.log(response); + } + + main(); + response: &chat_completion_function_example | + { + "id": "chatcmpl-abc123", + "object": "chat.completion", + "created": 1699896916, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\n\"location\": \"Boston, MA\"\n}" + } + } + ] + }, + "logprobs": null, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 82, + "completion_tokens": 17, + "total_tokens": 99 + } + } + - title: Logprobs + request: + curl: | + curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "VAR_model_id", + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ], + "logprobs": true, + "top_logprobs": 2 + }' + python: | + from openai import OpenAI + client = OpenAI() + + completion = client.chat.completions.create( + model="VAR_model_id", + messages=[ + {"role": "user", "content": "Hello!"} + ], + logprobs=True, + top_logprobs=2 + ) + + print(completion.choices[0].message) + print(completion.choices[0].logprobs) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const completion = await openai.chat.completions.create({ + messages: [{ role: "user", content: "Hello!" }], + model: "VAR_model_id", + logprobs: true, + top_logprobs: 2, + }); + + console.log(completion.choices[0]); + } + + main(); + response: | + { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1702685778, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + }, + "logprobs": { + "content": [ + { + "token": "Hello", + "logprob": -0.31725305, + "bytes": [72, 101, 108, 108, 111], + "top_logprobs": [ + { + "token": "Hello", + "logprob": -0.31725305, + "bytes": [72, 101, 108, 108, 111] + }, + { + "token": "Hi", + "logprob": -1.3190403, + "bytes": [72, 105] + } + ] + }, + { + "token": "!", + "logprob": -0.02380986, + "bytes": [ + 33 + ], + "top_logprobs": [ + { + "token": "!", + "logprob": -0.02380986, + "bytes": [33] + }, + { + "token": " there", + "logprob": -3.787621, + "bytes": [32, 116, 104, 101, 114, 101] + } + ] + }, + { + "token": " How", + "logprob": -0.000054669687, + "bytes": [32, 72, 111, 119], + "top_logprobs": [ + { + "token": " How", + "logprob": -0.000054669687, + "bytes": [32, 72, 111, 119] + }, + { + "token": "<|end|>", + "logprob": -10.953937, + "bytes": null + } + ] + }, + { + "token": " can", + "logprob": -0.015801601, + "bytes": [32, 99, 97, 110], + "top_logprobs": [ + { + "token": " can", + "logprob": -0.015801601, + "bytes": [32, 99, 97, 110] + }, + { + "token": " may", + "logprob": -4.161023, + "bytes": [32, 109, 97, 121] + } + ] + }, + { + "token": " I", + "logprob": -3.7697225e-6, + "bytes": [ + 32, + 73 + ], + "top_logprobs": [ + { + "token": " I", + "logprob": -3.7697225e-6, + "bytes": [32, 73] + }, + { + "token": " assist", + "logprob": -13.596657, + "bytes": [32, 97, 115, 115, 105, 115, 116] + } + ] + }, + { + "token": " assist", + "logprob": -0.04571125, + "bytes": [32, 97, 115, 115, 105, 115, 116], + "top_logprobs": [ + { + "token": " assist", + "logprob": -0.04571125, + "bytes": [32, 97, 115, 115, 105, 115, 116] + }, + { + "token": " help", + "logprob": -3.1089056, + "bytes": [32, 104, 101, 108, 112] + } + ] + }, + { + "token": " you", + "logprob": -5.4385737e-6, + "bytes": [32, 121, 111, 117], + "top_logprobs": [ + { + "token": " you", + "logprob": -5.4385737e-6, + "bytes": [32, 121, 111, 117] + }, + { + "token": " today", + "logprob": -12.807695, + "bytes": [32, 116, 111, 100, 97, 121] + } + ] + }, + { + "token": " today", + "logprob": -0.0040071653, + "bytes": [32, 116, 111, 100, 97, 121], + "top_logprobs": [ + { + "token": " today", + "logprob": -0.0040071653, + "bytes": [32, 116, 111, 100, 97, 121] + }, + { + "token": "?", + "logprob": -5.5247097, + "bytes": [63] + } + ] + }, + { + "token": "?", + "logprob": -0.0008108172, + "bytes": [63], + "top_logprobs": [ + { + "token": "?", + "logprob": -0.0008108172, + "bytes": [63] + }, + { + "token": "?\n", + "logprob": -7.184561, + "bytes": [63, 10] + } + ] + } + ] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 9, + "total_tokens": 18 + }, + "system_fingerprint": null + } + + /completions: + post: + operationId: createCompletion + tags: + - Completions + summary: Creates a completion for the provided prompt and parameters. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateCompletionRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/CreateCompletionResponse" + x-oaiMeta: + name: Create completion + group: completions + returns: | + Returns a [completion](/docs/api-reference/completions/object) object, or a sequence of completion objects if the request is streamed. + legacy: true + examples: + - title: No streaming + request: + curl: | + curl https://api.openai.com/v1/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "VAR_model_id", + "prompt": "Say this is a test", + "max_tokens": 7, + "temperature": 0 + }' + python: | + from openai import OpenAI + client = OpenAI() + + client.completions.create( + model="VAR_model_id", + prompt="Say this is a test", + max_tokens=7, + temperature=0 + ) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const completion = await openai.completions.create({ + model: "VAR_model_id", + prompt: "Say this is a test.", + max_tokens: 7, + temperature: 0, + }); + + console.log(completion); + } + main(); + response: | + { + "id": "cmpl-uqkvlQyYK7bGYrRHQ0eXlWi7", + "object": "text_completion", + "created": 1589478378, + "model": "VAR_model_id", + "system_fingerprint": "fp_44709d6fcb", + "choices": [ + { + "text": "\n\nThis is indeed a test", + "index": 0, + "logprobs": null, + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 7, + "total_tokens": 12 + } + } + - title: Streaming + request: + curl: | + curl https://api.openai.com/v1/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "VAR_model_id", + "prompt": "Say this is a test", + "max_tokens": 7, + "temperature": 0, + "stream": true + }' + python: | + from openai import OpenAI + client = OpenAI() + + for chunk in client.completions.create( + model="VAR_model_id", + prompt="Say this is a test", + max_tokens=7, + temperature=0, + stream=True + ): + print(chunk.choices[0].text) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const stream = await openai.completions.create({ + model: "VAR_model_id", + prompt: "Say this is a test.", + stream: true, + }); + + for await (const chunk of stream) { + console.log(chunk.choices[0].text) + } + } + main(); + response: | + { + "id": "cmpl-7iA7iJjj8V2zOkCGvWF2hAkDWBQZe", + "object": "text_completion", + "created": 1690759702, + "choices": [ + { + "text": "This", + "index": 0, + "logprobs": null, + "finish_reason": null + } + ], + "model": "gpt-3.5-turbo-instruct" + "system_fingerprint": "fp_44709d6fcb", + } + /models: + get: + operationId: listModels + tags: + - Models + summary: Lists the currently available models, and provides basic information about each one such as the owner and availability. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListModelsResponse" + x-oaiMeta: + name: List models + group: models + returns: A list of [model](/docs/api-reference/models/object) objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/models \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.models.list() + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const list = await openai.models.list(); + + for await (const model of list) { + console.log(model); + } + } + main(); + response: | + { + "object": "list", + "data": [ + { + "id": "model-id-0", + "object": "model", + "created": 1686935002, + "owned_by": "organization-owner" + }, + { + "id": "model-id-1", + "object": "model", + "created": 1686935002, + "owned_by": "organization-owner", + }, + { + "id": "model-id-2", + "object": "model", + "created": 1686935002, + "owned_by": "openai" + }, + ], + "object": "list" + } + /models/{model}: + get: + operationId: retrieveModel + tags: + - Models + summary: Retrieves a model instance, providing basic information about the model such as the owner and permissioning. + parameters: + - in: path + name: model + required: true + schema: + type: string + # ideally this will be an actual ID, so this will always work from browser + example: gpt-3.5-turbo + description: The ID of the model to use for this request + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/Model" + x-oaiMeta: + name: Retrieve model + group: models + returns: The [model](/docs/api-reference/models/object) object matching the specified ID. + examples: + request: + curl: | + curl https://api.openai.com/v1/models/VAR_model_id \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.models.retrieve("VAR_model_id") + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const model = await openai.models.retrieve("VAR_model_id"); + + console.log(model); + } + + main(); + response: &retrieve_model_response | + { + "id": "VAR_model_id", + "object": "model", + "created": 1686935002, + "owned_by": "openai" + } + delete: + operationId: deleteModel + tags: + - Models + summary: Delete a fine-tuned model. You must have the Owner role in your organization to delete a model. + parameters: + - in: path + name: model + required: true + schema: + type: string + example: ft:gpt-3.5-turbo:acemeco:suffix:abc123 + description: The model to delete + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/DeleteModelResponse" + x-oaiMeta: + name: Delete a fine-tuned model + group: models + returns: Deletion status. + examples: + request: + curl: | + curl https://api.openai.com/v1/models/ft:gpt-3.5-turbo:acemeco:suffix:abc123 \ + -X DELETE \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.models.delete("ft:gpt-3.5-turbo:acemeco:suffix:abc123") + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const model = await openai.models.del("ft:gpt-3.5-turbo:acemeco:suffix:abc123"); + + console.log(model); + } + main(); + response: | + { + "id": "ft:gpt-3.5-turbo:acemeco:suffix:abc123", + "object": "model", + "deleted": true + } + +components: + securitySchemes: + ApiKeyAuth: + type: http + scheme: "bearer" + + schemas: + Error: + type: object + properties: + code: + type: string + nullable: true + message: + type: string + nullable: false + param: + type: string + nullable: true + type: + type: string + nullable: false + required: + - type + - message + - param + - code + ErrorResponse: + type: object + properties: + error: + $ref: "#/components/schemas/Error" + required: + - error + + ListModelsResponse: + type: object + properties: + object: + type: string + enum: [list] + data: + type: array + items: + $ref: "#/components/schemas/Model" + required: + - object + - data + DeleteModelResponse: + type: object + properties: + id: + type: string + deleted: + type: boolean + object: + type: string + required: + - id + - object + - deleted + + CreateCompletionRequest: + type: object + properties: + model: + description: &model_description | + ID of the model to use. You can use the [List models](/docs/api-reference/models/list) API to see all of your available models, or see our [Model overview](/docs/models/overview) for descriptions of them. + anyOf: + - type: string + - type: string + enum: ["gpt-3.5-turbo-instruct", "davinci-002", "babbage-002"] + x-oaiTypeLabel: string + prompt: + description: &completions_prompt_description | + The prompt(s) to generate completions for, encoded as a string, array of strings, array of tokens, or array of token arrays. + + Note that <|endoftext|> is the document separator that the model sees during training, so if a prompt is not specified the model will generate as if from the beginning of a new document. + default: "<|endoftext|>" + nullable: true + oneOf: + - type: string + default: "" + example: "This is a test." + - type: array + items: + type: string + default: "" + example: "This is a test." + - type: array + minItems: 1 + items: + type: integer + example: "[1212, 318, 257, 1332, 13]" + - type: array + minItems: 1 + items: + type: array + minItems: 1 + items: + type: integer + example: "[[1212, 318, 257, 1332, 13]]" + best_of: + type: integer + default: 1 + minimum: 0 + maximum: 20 + nullable: true + description: &completions_best_of_description | + Generates `best_of` completions server-side and returns the "best" (the one with the highest log probability per token). Results cannot be streamed. + + When used with `n`, `best_of` controls the number of candidate completions and `n` specifies how many to return – `best_of` must be greater than `n`. + + **Note:** Because this parameter generates many completions, it can quickly consume your token quota. Use carefully and ensure that you have reasonable settings for `max_tokens` and `stop`. + echo: + type: boolean + default: false + nullable: true + description: &completions_echo_description > + Echo back the prompt in addition to the completion + frequency_penalty: + type: number + default: 0 + minimum: -2 + maximum: 2 + nullable: true + description: &completions_frequency_penalty_description | + Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. + + [See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details) + logit_bias: &completions_logit_bias + type: object + x-oaiTypeLabel: map + default: null + nullable: true + additionalProperties: + type: integer + description: &completions_logit_bias_description | + Modify the likelihood of specified tokens appearing in the completion. + + Accepts a JSON object that maps tokens (specified by their token ID in the GPT tokenizer) to an associated bias value from -100 to 100. You can use this [tokenizer tool](/tokenizer?view=bpe) to convert text to token IDs. Mathematically, the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should result in a ban or exclusive selection of the relevant token. + + As an example, you can pass `{"50256": -100}` to prevent the <|endoftext|> token from being generated. + logprobs: &completions_logprobs_configuration + type: integer + minimum: 0 + maximum: 5 + default: null + nullable: true + description: &completions_logprobs_description | + Include the log probabilities on the `logprobs` most likely output tokens, as well the chosen tokens. For example, if `logprobs` is 5, the API will return a list of the 5 most likely tokens. The API will always return the `logprob` of the sampled token, so there may be up to `logprobs+1` elements in the response. + + The maximum value for `logprobs` is 5. + max_tokens: + type: integer + minimum: 0 + default: 16 + example: 16 + nullable: true + description: &completions_max_tokens_description | + The maximum number of [tokens](/tokenizer) that can be generated in the completion. + + The token count of your prompt plus `max_tokens` cannot exceed the model's context length. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens. + n: + type: integer + minimum: 1 + maximum: 128 + default: 1 + example: 1 + nullable: true + description: &completions_completions_description | + How many completions to generate for each prompt. + + **Note:** Because this parameter generates many completions, it can quickly consume your token quota. Use carefully and ensure that you have reasonable settings for `max_tokens` and `stop`. + presence_penalty: + type: number + default: 0 + minimum: -2 + maximum: 2 + nullable: true + description: &completions_presence_penalty_description | + Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. + + [See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details) + seed: &completions_seed_param + type: integer + minimum: -9223372036854775808 + maximum: 9223372036854775807 + nullable: true + description: | + If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same `seed` and parameters should return the same result. + + Determinism is not guaranteed, and you should refer to the `system_fingerprint` response parameter to monitor changes in the backend. + stop: + description: &completions_stop_description > + Up to 4 sequences where the API will stop generating further tokens. The returned text will not contain the stop sequence. + default: null + nullable: true + oneOf: + - type: string + default: <|endoftext|> + example: "\n" + nullable: true + - type: array + minItems: 1 + maxItems: 4 + items: + type: string + example: '["\n"]' + stream: + description: > + Whether to stream back partial progress. If set, tokens will be sent as data-only [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) + as they become available, with the stream terminated by a `data: [DONE]` message. [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions). + type: boolean + nullable: true + default: false + suffix: + description: | + The suffix that comes after a completion of inserted text. + + This parameter is only supported for `gpt-3.5-turbo-instruct`. + default: null + nullable: true + type: string + example: "test." + temperature: + type: number + minimum: 0 + maximum: 2 + default: 1 + example: 1 + nullable: true + description: &completions_temperature_description | + What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. + + We generally recommend altering this or `top_p` but not both. + top_p: + type: number + minimum: 0 + maximum: 1 + default: 1 + example: 1 + nullable: true + description: &completions_top_p_description | + An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. + + We generally recommend altering this or `temperature` but not both. + user: &end_user_param_configuration + type: string + example: user-1234 + description: | + A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](/docs/guides/safety-best-practices/end-user-ids). + required: + - model + - prompt + + CreateCompletionResponse: + type: object + description: | + Represents a completion response from the API. Note: both the streamed and non-streamed response objects share the same shape (unlike the chat endpoint). + properties: + id: + type: string + description: A unique identifier for the completion. + choices: + type: array + description: The list of completion choices the model generated for the input prompt. + items: + type: object + required: + - finish_reason + - index + - logprobs + - text + properties: + finish_reason: + type: string + description: &completion_finish_reason_description | + The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, + `length` if the maximum number of tokens specified in the request was reached, + or `content_filter` if content was omitted due to a flag from our content filters. + enum: ["stop", "length", "content_filter"] + index: + type: integer + logprobs: + type: object + nullable: true + properties: + text_offset: + type: array + items: + type: integer + token_logprobs: + type: array + items: + type: number + tokens: + type: array + items: + type: string + top_logprobs: + type: array + items: + type: object + additionalProperties: + type: number + text: + type: string + created: + type: integer + description: The Unix timestamp (in seconds) of when the completion was created. + model: + type: string + description: The model used for completion. + system_fingerprint: + type: string + description: | + This fingerprint represents the backend configuration that the model runs with. + + Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. + object: + type: string + description: The object type, which is always "text_completion" + enum: [text_completion] + usage: + $ref: "#/components/schemas/CompletionUsage" + required: + - id + - object + - created + - model + - choices + x-oaiMeta: + name: The completion object + legacy: true + example: | + { + "id": "cmpl-uqkvlQyYK7bGYrRHQ0eXlWi7", + "object": "text_completion", + "created": 1589478378, + "model": "gpt-4-turbo", + "choices": [ + { + "text": "\n\nThis is indeed a test", + "index": 0, + "logprobs": null, + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 7, + "total_tokens": 12 + } + } + + ChatCompletionRequestMessageContentPart: + oneOf: + - $ref: "#/components/schemas/ChatCompletionRequestMessageContentPartText" + - $ref: "#/components/schemas/ChatCompletionRequestMessageContentPartImage" + x-oaiExpandable: true + + ChatCompletionRequestMessageContentPartImage: + type: object + title: Image content part + properties: + type: + type: string + enum: ["image_url"] + description: The type of the content part. + image_url: + type: object + properties: + url: + type: string + description: Either a URL of the image or the base64 encoded image data. + format: uri + detail: + type: string + description: Specifies the detail level of the image. Learn more in the [Vision guide](/docs/guides/vision/low-or-high-fidelity-image-understanding). + enum: ["auto", "low", "high"] + default: "auto" + required: + - url + required: + - type + - image_url + + ChatCompletionRequestMessageContentPartText: + type: object + title: Text content part + properties: + type: + type: string + enum: ["text"] + description: The type of the content part. + text: + type: string + description: The text content. + required: + - type + - text + + ChatCompletionRequestMessage: + oneOf: + - $ref: "#/components/schemas/ChatCompletionRequestSystemMessage" + - $ref: "#/components/schemas/ChatCompletionRequestUserMessage" + - $ref: "#/components/schemas/ChatCompletionRequestAssistantMessage" + - $ref: "#/components/schemas/ChatCompletionRequestToolMessage" + - $ref: "#/components/schemas/ChatCompletionRequestFunctionMessage" + x-oaiExpandable: true + + ChatCompletionRequestSystemMessage: + type: object + title: System message + properties: + content: + description: The contents of the system message. + type: string + role: + type: string + enum: ["system"] + description: The role of the messages author, in this case `system`. + name: + type: string + description: An optional name for the participant. Provides the model information to differentiate between participants of the same role. + required: + - content + - role + + ChatCompletionRequestUserMessage: + type: object + title: User message + properties: + content: + description: | + The contents of the user message. + oneOf: + - type: string + description: The text contents of the message. + title: Text content + - type: array + description: An array of content parts with a defined type, each can be of type `text` or `image_url` when passing in images. You can pass multiple images by adding multiple `image_url` content parts. Image input is only supported when using the `gpt-4-visual-preview` model. + title: Array of content parts + items: + $ref: "#/components/schemas/ChatCompletionRequestMessageContentPart" + minItems: 1 + x-oaiExpandable: true + role: + type: string + enum: ["user"] + description: The role of the messages author, in this case `user`. + name: + type: string + description: An optional name for the participant. Provides the model information to differentiate between participants of the same role. + required: + - content + - role + + ChatCompletionRequestAssistantMessage: + type: object + title: Assistant message + properties: + content: + nullable: true + type: string + description: | + The contents of the assistant message. Required unless `tool_calls` or `function_call` is specified. + role: + type: string + enum: ["assistant"] + description: The role of the messages author, in this case `assistant`. + name: + type: string + description: An optional name for the participant. Provides the model information to differentiate between participants of the same role. + tool_calls: + $ref: "#/components/schemas/ChatCompletionMessageToolCalls" + function_call: + type: object + deprecated: true + description: "Deprecated and replaced by `tool_calls`. The name and arguments of a function that should be called, as generated by the model." + properties: + arguments: + type: string + description: The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function. + name: + type: string + description: The name of the function to call. + required: + - arguments + - name + required: + - role + + ChatCompletionRequestToolMessage: + type: object + title: Tool message + properties: + role: + type: string + enum: ["tool"] + description: The role of the messages author, in this case `tool`. + content: + type: string + description: The contents of the tool message. + tool_call_id: + type: string + description: Tool call that this message is responding to. + required: + - role + - content + - tool_call_id + + ChatCompletionRequestFunctionMessage: + type: object + title: Function message + deprecated: true + properties: + role: + type: string + enum: ["function"] + description: The role of the messages author, in this case `function`. + content: + nullable: true + type: string + description: The contents of the function message. + name: + type: string + description: The name of the function to call. + required: + - role + - content + - name + + FunctionParameters: + type: object + description: "The parameters the functions accepts, described as a JSON Schema object. See the [guide](/docs/guides/text-generation/function-calling) for examples, and the [JSON Schema reference](https://json-schema.org/understanding-json-schema/) for documentation about the format. \n\nOmitting `parameters` defines a function with an empty parameter list." + additionalProperties: true + + ChatCompletionFunctions: + type: object + deprecated: true + properties: + description: + type: string + description: A description of what the function does, used by the model to choose when and how to call the function. + name: + type: string + description: The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64. + parameters: + $ref: "#/components/schemas/FunctionParameters" + required: + - name + + ChatCompletionFunctionCallOption: + type: object + description: > + Specifying a particular function via `{"name": "my_function"}` forces the model to call that function. + properties: + name: + type: string + description: The name of the function to call. + required: + - name + + ChatCompletionTool: + type: object + properties: + type: + type: string + enum: ["function"] + description: The type of the tool. Currently, only `function` is supported. + function: + $ref: "#/components/schemas/FunctionObject" + required: + - type + - function + + FunctionObject: + type: object + properties: + description: + type: string + description: A description of what the function does, used by the model to choose when and how to call the function. + name: + type: string + description: The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64. + parameters: + $ref: "#/components/schemas/FunctionParameters" + required: + - name + + ChatCompletionToolChoiceOption: + description: | + Controls which (if any) tool is called by the model. + `none` means the model will not call any tool and instead generates a message. + `auto` means the model can pick between generating a message or calling one or more tools. + `required` means the model must call one or more tools. + Specifying a particular tool via `{"type": "function", "function": {"name": "my_function"}}` forces the model to call that tool. + + `none` is the default when no tools are present. `auto` is the default if tools are present. + oneOf: + - type: string + description: > + `none` means the model will not call any tool and instead generates a message. + `auto` means the model can pick between generating a message or calling one or more tools. + `required` means the model must call one or more tools. + enum: [none, auto, required] + - $ref: "#/components/schemas/ChatCompletionNamedToolChoice" + x-oaiExpandable: true + + ChatCompletionNamedToolChoice: + type: object + description: Specifies a tool the model should use. Use to force the model to call a specific function. + properties: + type: + type: string + enum: ["function"] + description: The type of the tool. Currently, only `function` is supported. + function: + type: object + properties: + name: + type: string + description: The name of the function to call. + required: + - name + required: + - type + - function + + ChatCompletionMessageToolCalls: + type: array + description: The tool calls generated by the model, such as function calls. + items: + $ref: "#/components/schemas/ChatCompletionMessageToolCall" + + ChatCompletionMessageToolCall: + type: object + properties: + # TODO: index included when streaming + id: + type: string + description: The ID of the tool call. + type: + type: string + enum: ["function"] + description: The type of the tool. Currently, only `function` is supported. + function: + type: object + description: The function that the model called. + properties: + name: + type: string + description: The name of the function to call. + arguments: + type: string + description: The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function. + required: + - name + - arguments + required: + - id + - type + - function + + ChatCompletionMessageToolCallChunk: + type: object + properties: + index: + type: integer + id: + type: string + description: The ID of the tool call. + type: + type: string + enum: ["function"] + description: The type of the tool. Currently, only `function` is supported. + function: + type: object + properties: + name: + type: string + description: The name of the function to call. + arguments: + type: string + description: The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function. + required: + - index + + # Note, this isn't referenced anywhere, but is kept as a convenience to record all possible roles in one place. + ChatCompletionRole: + type: string + description: The role of the author of a message + enum: + - system + - user + - assistant + - tool + - function + + ChatCompletionResponseMessage: + type: object + description: A chat completion message generated by the model. + properties: + content: + type: string + description: The contents of the message. + nullable: true + tool_calls: + $ref: "#/components/schemas/ChatCompletionMessageToolCalls" + role: + type: string + enum: ["assistant"] + description: The role of the author of this message. + function_call: + type: object + deprecated: true + description: "Deprecated and replaced by `tool_calls`. The name and arguments of a function that should be called, as generated by the model." + properties: + arguments: + type: string + description: The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function. + name: + type: string + description: The name of the function to call. + required: + - name + - arguments + required: + - role + - content + + ChatCompletionStreamResponseDelta: + type: object + description: A chat completion delta generated by streamed model responses. + properties: + content: + type: string + description: The contents of the chunk message. + nullable: true + function_call: + deprecated: true + type: object + description: "Deprecated and replaced by `tool_calls`. The name and arguments of a function that should be called, as generated by the model." + properties: + arguments: + type: string + description: The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function. + name: + type: string + description: The name of the function to call. + tool_calls: + type: array + items: + $ref: "#/components/schemas/ChatCompletionMessageToolCallChunk" + role: + type: string + enum: ["system", "user", "assistant", "tool"] + description: The role of the author of this message. + + CreateChatCompletionRequest: + type: object + properties: + messages: + description: A list of messages comprising the conversation so far. [Example Python code](https://cookbook.openai.com/examples/how_to_format_inputs_to_chatgpt_models). + type: array + minItems: 1 + items: + $ref: "#/components/schemas/ChatCompletionRequestMessage" + model: + description: ID of the model to use. See the [model endpoint compatibility](/docs/models/model-endpoint-compatibility) table for details on which models work with the Chat API. + example: "gpt-4-turbo" + anyOf: + - type: string + - type: string + enum: + [ + "gpt-4-turbo", + "gpt-4-turbo-2024-04-09", + "gpt-4-0125-preview", + "gpt-4-turbo-preview", + "gpt-4-1106-preview", + "gpt-4-vision-preview", + "gpt-4", + "gpt-4-0314", + "gpt-4-0613", + "gpt-4-32k", + "gpt-4-32k-0314", + "gpt-4-32k-0613", + "gpt-3.5-turbo", + "gpt-3.5-turbo-16k", + "gpt-3.5-turbo-0301", + "gpt-3.5-turbo-0613", + "gpt-3.5-turbo-1106", + "gpt-3.5-turbo-0125", + "gpt-3.5-turbo-16k-0613", + ] + x-oaiTypeLabel: string + frequency_penalty: + type: number + default: 0 + minimum: -2 + maximum: 2 + nullable: true + description: *completions_frequency_penalty_description + logit_bias: + type: object + x-oaiTypeLabel: map + default: null + nullable: true + additionalProperties: + type: integer + description: | + Modify the likelihood of specified tokens appearing in the completion. + + Accepts a JSON object that maps tokens (specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically, the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should result in a ban or exclusive selection of the relevant token. + logprobs: + description: Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the `content` of `message`. + type: boolean + default: false + nullable: true + top_logprobs: + description: An integer between 0 and 20 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to `true` if this parameter is used. + type: integer + minimum: 0 + maximum: 20 + nullable: true + max_tokens: + description: | + The maximum number of [tokens](/tokenizer) that can be generated in the chat completion. + + The total length of input tokens and generated tokens is limited by the model's context length. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens. + type: integer + nullable: true + n: + type: integer + minimum: 1 + maximum: 128 + default: 1 + example: 1 + nullable: true + description: How many chat completion choices to generate for each input message. Note that you will be charged based on the number of generated tokens across all of the choices. Keep `n` as `1` to minimize costs. + presence_penalty: + type: number + default: 0 + minimum: -2 + maximum: 2 + nullable: true + description: *completions_presence_penalty_description + response_format: + type: object + description: | + An object specifying the format that the model must output. Compatible with [GPT-4 Turbo](/docs/models/gpt-4-and-gpt-4-turbo) and all GPT-3.5 Turbo models newer than `gpt-3.5-turbo-1106`. + + Setting to `{ "type": "json_object" }` enables JSON mode, which guarantees the message the model generates is valid JSON. + + **Important:** when using JSON mode, you **must** also instruct the model to produce JSON yourself via a system or user message. Without this, the model may generate an unending stream of whitespace until the generation reaches the token limit, resulting in a long-running and seemingly "stuck" request. Also note that the message content may be partially cut off if `finish_reason="length"`, which indicates the generation exceeded `max_tokens` or the conversation exceeded the max context length. + properties: + type: + type: string + enum: ["text", "json_object"] + example: "json_object" + default: "text" + description: Must be one of `text` or `json_object`. + seed: + type: integer + minimum: -9223372036854775808 + maximum: 9223372036854775807 + nullable: true + description: | + This feature is in Beta. + If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same `seed` and parameters should return the same result. + Determinism is not guaranteed, and you should refer to the `system_fingerprint` response parameter to monitor changes in the backend. + x-oaiMeta: + beta: true + stop: + description: | + Up to 4 sequences where the API will stop generating further tokens. + default: null + oneOf: + - type: string + nullable: true + - type: array + minItems: 1 + maxItems: 4 + items: + type: string + stream: + description: > + If set, partial message deltas will be sent, like in ChatGPT. Tokens will be sent as data-only [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) + as they become available, with the stream terminated by a `data: [DONE]` message. [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions). + type: boolean + nullable: true + default: false + temperature: + type: number + minimum: 0 + maximum: 2 + default: 1 + example: 1 + nullable: true + description: *completions_temperature_description + top_p: + type: number + minimum: 0 + maximum: 1 + default: 1 + example: 1 + nullable: true + description: *completions_top_p_description + tools: + type: array + description: > + A list of tools the model may call. Currently, only functions are supported as a tool. + Use this to provide a list of functions the model may generate JSON inputs for. A max of 128 functions are supported. + items: + $ref: "#/components/schemas/ChatCompletionTool" + tool_choice: + $ref: "#/components/schemas/ChatCompletionToolChoiceOption" + user: *end_user_param_configuration + function_call: + deprecated: true + description: | + Deprecated in favor of `tool_choice`. + + Controls which (if any) function is called by the model. + `none` means the model will not call a function and instead generates a message. + `auto` means the model can pick between generating a message or calling a function. + Specifying a particular function via `{"name": "my_function"}` forces the model to call that function. + + `none` is the default when no functions are present. `auto` is the default if functions are present. + oneOf: + - type: string + description: > + `none` means the model will not call a function and instead generates a message. + `auto` means the model can pick between generating a message or calling a function. + enum: [none, auto] + - $ref: "#/components/schemas/ChatCompletionFunctionCallOption" + x-oaiExpandable: true + functions: + deprecated: true + description: | + Deprecated in favor of `tools`. + + A list of functions the model may generate JSON inputs for. + type: array + minItems: 1 + maxItems: 128 + items: + $ref: "#/components/schemas/ChatCompletionFunctions" + + required: + - model + - messages + + CreateChatCompletionResponse: + type: object + description: Represents a chat completion response returned by model, based on the provided input. + properties: + id: + type: string + description: A unique identifier for the chat completion. + choices: + type: array + description: A list of chat completion choices. Can be more than one if `n` is greater than 1. + items: + type: object + required: + - finish_reason + - index + - message + - logprobs + properties: + finish_reason: + type: string + description: &chat_completion_finish_reason_description | + The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, + `length` if the maximum number of tokens specified in the request was reached, + `content_filter` if content was omitted due to a flag from our content filters, + `tool_calls` if the model called a tool, or `function_call` (deprecated) if the model called a function. + enum: + [ + "stop", + "length", + "tool_calls", + "content_filter", + "function_call", + ] + index: + type: integer + description: The index of the choice in the list of choices. + message: + $ref: "#/components/schemas/ChatCompletionResponseMessage" + logprobs: &chat_completion_response_logprobs + description: Log probability information for the choice. + type: object + nullable: true + properties: + content: + description: A list of message content tokens with log probability information. + type: array + items: + $ref: "#/components/schemas/ChatCompletionTokenLogprob" + nullable: true + required: + - content + created: + type: integer + description: The Unix timestamp (in seconds) of when the chat completion was created. + model: + type: string + description: The model used for the chat completion. + system_fingerprint: + type: string + description: | + This fingerprint represents the backend configuration that the model runs with. + + Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. + object: + type: string + description: The object type, which is always `chat.completion`. + enum: [chat.completion] + usage: + $ref: "#/components/schemas/CompletionUsage" + required: + - choices + - created + - id + - model + - object + x-oaiMeta: + name: The chat completion object + group: chat + example: *chat_completion_example + + CreateChatCompletionFunctionResponse: + type: object + description: Represents a chat completion response returned by model, based on the provided input. + properties: + id: + type: string + description: A unique identifier for the chat completion. + choices: + type: array + description: A list of chat completion choices. Can be more than one if `n` is greater than 1. + items: + type: object + required: + - finish_reason + - index + - message + - logprobs + properties: + finish_reason: + type: string + description: + &chat_completion_function_finish_reason_description | + The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, `length` if the maximum number of tokens specified in the request was reached, `content_filter` if content was omitted due to a flag from our content filters, or `function_call` if the model called a function. + enum: ["stop", "length", "function_call", "content_filter"] + index: + type: integer + description: The index of the choice in the list of choices. + message: + $ref: "#/components/schemas/ChatCompletionResponseMessage" + created: + type: integer + description: The Unix timestamp (in seconds) of when the chat completion was created. + model: + type: string + description: The model used for the chat completion. + system_fingerprint: + type: string + description: | + This fingerprint represents the backend configuration that the model runs with. + + Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. + object: + type: string + description: The object type, which is always `chat.completion`. + enum: [chat.completion] + usage: + $ref: "#/components/schemas/CompletionUsage" + required: + - choices + - created + - id + - model + - object + x-oaiMeta: + name: The chat completion object + group: chat + example: *chat_completion_function_example + + ChatCompletionTokenLogprob: + type: object + properties: + token: &chat_completion_response_logprobs_token + description: The token. + type: string + logprob: &chat_completion_response_logprobs_token_logprob + description: The log probability of this token, if it is within the top 20 most likely tokens. Otherwise, the value `-9999.0` is used to signify that the token is very unlikely. + type: number + bytes: &chat_completion_response_logprobs_bytes + description: A list of integers representing the UTF-8 bytes representation of the token. Useful in instances where characters are represented by multiple tokens and their byte representations must be combined to generate the correct text representation. Can be `null` if there is no bytes representation for the token. + type: array + items: + type: integer + nullable: true + top_logprobs: + description: List of the most likely tokens and their log probability, at this token position. In rare cases, there may be fewer than the number of requested `top_logprobs` returned. + type: array + items: + type: object + properties: + token: *chat_completion_response_logprobs_token + logprob: *chat_completion_response_logprobs_token_logprob + bytes: *chat_completion_response_logprobs_bytes + required: + - token + - logprob + - bytes + required: + - token + - logprob + - bytes + - top_logprobs + + CreateChatCompletionStreamResponse: + type: object + description: Represents a streamed chunk of a chat completion response returned by model, based on the provided input. + properties: + id: + type: string + description: A unique identifier for the chat completion. Each chunk has the same ID. + choices: + type: array + description: A list of chat completion choices. Can be more than one if `n` is greater than 1. + items: + type: object + required: + - delta + - finish_reason + - index + properties: + delta: + $ref: "#/components/schemas/ChatCompletionStreamResponseDelta" + logprobs: *chat_completion_response_logprobs + finish_reason: + type: string + description: *chat_completion_finish_reason_description + enum: + [ + "stop", + "length", + "tool_calls", + "content_filter", + "function_call", + ] + nullable: true + index: + type: integer + description: The index of the choice in the list of choices. + created: + type: integer + description: The Unix timestamp (in seconds) of when the chat completion was created. Each chunk has the same timestamp. + model: + type: string + description: The model to generate the completion. + system_fingerprint: + type: string + description: | + This fingerprint represents the backend configuration that the model runs with. + Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. + object: + type: string + description: The object type, which is always `chat.completion.chunk`. + enum: [chat.completion.chunk] + required: + - choices + - created + - id + - model + - object + x-oaiMeta: + name: The chat completion chunk object + group: chat + example: *chat_completion_chunk_example + + CreateChatCompletionImageResponse: + type: object + description: Represents a streamed chunk of a chat completion response returned by model, based on the provided input. + x-oaiMeta: + name: The chat completion chunk object + group: chat + example: *chat_completion_image_example + + Model: + title: Model + description: Describes an OpenAI model offering that can be used with the API. + properties: + id: + type: string + description: The model identifier, which can be referenced in the API endpoints. + created: + type: integer + description: The Unix timestamp (in seconds) when the model was created. + object: + type: string + description: The object type, which is always "model". + enum: [model] + owned_by: + type: string + description: The organization that owns the model. + required: + - id + - object + - created + - owned_by + x-oaiMeta: + name: The model object + example: *retrieve_model_response + + CompletionUsage: + type: object + description: Usage statistics for the completion request. + properties: + completion_tokens: + type: integer + description: Number of tokens in the generated completion. + prompt_tokens: + type: integer + description: Number of tokens in the prompt. + total_tokens: + type: integer + description: Total number of tokens used in the request (prompt + completion). + required: + - prompt_tokens + - completion_tokens + - total_tokens + + ErrorEvent: + type: object + properties: + event: + type: string + enum: ["error"] + data: + $ref: "#/components/schemas/Error" + required: + - event + - data + description: Occurs when an [error](/docs/guides/error-codes/api-errors) occurs. This can happen due to an internal server error or a timeout. + x-oaiMeta: + dataDescription: "`data` is an [error](/docs/guides/error-codes/api-errors)" + + DoneEvent: + type: object + properties: + event: + type: string + enum: ["done"] + data: + type: string + enum: ["[DONE]"] + required: + - event + - data + description: Occurs when a stream ends. + x-oaiMeta: + dataDescription: "`data` is `[DONE]`" + +security: + - ApiKeyAuth: [] + +x-oaiMeta: + navigationGroups: + - id: endpoints + title: Endpoints + - id: legacy + title: Legacy + groups: + # > General Notes + # The `groups` section is used to generate the API reference pages and navigation, in the same + # order listed below. Additionally, each `group` can have a list of `sections`, each of which + # will become a navigation subroute and subsection under the group. Each section has: + # - `type`: Currently, either an `endpoint` or `object`, depending on how the section needs to + # be rendered + # - `key`: The reference key that can be used to lookup the section definition + # - `path`: The path (url) of the section, which is used to generate the navigation link. + # + # > The `object` sections maps to a schema component and the following fields are read for rendering + # - `x-oaiMeta.name`: The name of the object, which will become the section title + # - `x-oaiMeta.example`: The example object, which will be used to generate the example sample (always JSON) + # - `description`: The description of the object, which will be used to generate the section description + # + # > The `endpoint` section maps to an operation path and the following fields are read for rendering: + # - `x-oaiMeta.name`: The name of the endpoint, which will become the section title + # - `x-oaiMeta.examples`: The endpoint examples, which can be an object (meaning a single variation, most + # endpoints, or an array of objects, meaning multiple variations, e.g. the + # chat completion and completion endpoints, with streamed and non-streamed examples. + # - `x-oaiMeta.returns`: text describing what the endpoint returns. + # - `summary`: The summary of the endpoint, which will be used to generate the section description + - id: chat + title: Chat + description: | + Given a list of messages comprising a conversation, the model will return a response. + + Related guide: [Chat Completions](/docs/guides/text-generation) + navigationGroup: endpoints + sections: + - type: endpoint + key: createChatCompletion + path: create + - type: object + key: CreateChatCompletionResponse + path: object + - type: object + key: CreateChatCompletionStreamResponse + path: streaming + - id: models + title: Models + description: | + List and describe the various models available in the API. You can refer to the [Models](/docs/models) documentation to understand what models are available and the differences between them. + navigationGroup: endpoints + sections: + - type: endpoint + key: listModels + path: list + - type: endpoint + key: retrieveModel + path: retrieve + - type: endpoint + key: deleteModel + path: delete + - type: object + key: Model + path: object + - id: completions + title: Completions + legacy: true + navigationGroup: legacy + description: | + Given a prompt, the model will return one or more predicted completions along with the probabilities of alternative tokens at each position. Most developer should use our [Chat Completions API](/docs/guides/text-generation/text-generation-models) to leverage our best and newest models. + sections: + - type: endpoint + key: createCompletion + path: create + - type: object + key: CreateCompletionResponse + path: object diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/simple/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/simple/main.py new file mode 100644 index 00000000..109d84fc --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/simple/main.py @@ -0,0 +1,146 @@ +# generated by fastapi-codegen: +# filename: openapi_modified.yaml +# timestamp: 2024-05-04T14:14:42+00:00 + +from __future__ import annotations + +import copy +import time +import uuid + +import tritonserver +from fastapi import FastAPI + +triton_server = tritonserver.Server( + model_repository="/workspace/llm-models", log_verbose=6, strict_model_config=False +).start() + +from models import ( + ChatCompletionResponseMessage, + Choice, + Choice1, + CreateChatCompletionRequest, + CreateChatCompletionResponse, + CreateCompletionRequest, + CreateCompletionResponse, + DeleteModelResponse, + FinishReason, + FinishReason1, + ListModelsResponse, + Logprobs, + Logprobs2, + Model, + Object1, + Object2, + Role5, +) + +app = FastAPI( + title="OpenAI API", + description="The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details.", + version="2.0.0", + termsOfService="https://openai.com/policies/terms-of-use", + contact={"name": "OpenAI Support", "url": "https://help.openai.com/"}, + license={ + "name": "MIT", + "url": "https://github.com/openai/openai-openapi/blob/master/LICENSE", + }, + servers=[{"url": "https://api.openai.com/v1"}], +) + + +@app.post( + "/chat/completions", response_model=CreateChatCompletionResponse, tags=["Chat"] +) +def create_chat_completion( + body: CreateChatCompletionRequest, +) -> CreateChatCompletionResponse: + """ + Creates a model response for the given chat conversation. + """ + return CreateChatCompletionResponse( + id="foo", + choices=[ + Choice1( + finish_reason=FinishReason1.stop, + index=0, + message=ChatCompletionResponseMessage( + content="hello", role=Role5.assistant, function_call=None + ), + logprobs=Logprobs2(content=[]), + ) + ], + created=0, + model="foo", + system_fingerprint=None, + object=Object2.chat_completion, + ) + + +@app.post("/completions", response_model=CreateCompletionResponse, tags=["Completions"]) +def create_completion(body: CreateCompletionRequest) -> CreateCompletionResponse: + """ + Creates a completion for the provided prompt and parameters. + """ + exclude_input_in_output = True + + if body.echo: + exclude_input_in_output = False + + model = triton_server.model("llama-3-8b-instruct") + parameters = copy.deepcopy(body.dict()) + del parameters["prompt"] + del parameters["stream"] + del parameters["echo"] + del parameters["model"] + + response = list( + model.infer( + inputs={ + "text_input": [body.prompt], + "stream": [False], + "exclude_input_in_output": [exclude_input_in_output], + }, + parameters=parameters, + ) + )[0] + + choice = Choice( + finish_reason=FinishReason.stop, + index=0, + logprobs=Logprobs(), + text=response.outputs["text_output"].to_string_array()[0], + ) + + return CreateCompletionResponse( + id=f"cmpl-{uuid.uuid1()}", + created=int(time.time()), + model=model.name, + choices=[choice], + system_fingerprint=None, + object=Object1.text_completion, + ) + + +@app.get("/models", response_model=ListModelsResponse, tags=["Models"]) +def list_models() -> ListModelsResponse: + """ + Lists the currently available models, and provides basic information about each one such as the owner and availability. + """ + pass + + +@app.get("/models/{model}", response_model=Model, tags=["Models"]) +def retrieve_model(model: str) -> Model: + """ + Retrieves a model instance, providing basic information about the model such as the owner and permissioning. + """ + pass + + +@app.delete("/models/{model}", response_model=DeleteModelResponse, tags=["Models"]) +def delete_model(model: str) -> DeleteModelResponse: + """ + Delete a fine-tuned model. You must have the Owner role in your organization to delete a model. + """ + pass diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/simple/models.py b/Triton_Inference_Server_Python_API/examples/fastapi/simple/models.py new file mode 100644 index 00000000..0051af83 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/simple/models.py @@ -0,0 +1,830 @@ +# generated by fastapi-codegen: +# filename: openai-openapi/openapi_modified.yaml +# timestamp: 2024-05-04T14:14:42+00:00 + +from __future__ import annotations + +from enum import Enum +from typing import Any, Dict, List, Optional, Union + +from pydantic import AnyUrl, BaseModel, Extra, Field, RootModel, confloat, conint + + +class Error(BaseModel): + code: str + message: str + param: str + type: str + + +class ErrorResponse(BaseModel): + error: Error + + +class Object(Enum): + list = "list" + + +class DeleteModelResponse(BaseModel): + id: str + deleted: bool + object: str + + +class Model1(Enum): + gpt_3_5_turbo_instruct = "gpt-3.5-turbo-instruct" + davinci_002 = "davinci-002" + babbage_002 = "babbage-002" + + +class PromptItem(RootModel): + root: List[Any] + + +class CreateCompletionRequest(BaseModel): + model: Union[str, Model1] = Field( + ..., + description="ID of the model to use. You can use the [List models](/docs/api-reference/models/list) API to see all of your available models, or see our [Model overview](/docs/models/overview) for descriptions of them.\n", + ) + prompt: Union[str, List[str], List[int], List[PromptItem]] = Field( + ..., + description="The prompt(s) to generate completions for, encoded as a string, array of strings, array of tokens, or array of token arrays.\n\nNote that <|endoftext|> is the document separator that the model sees during training, so if a prompt is not specified the model will generate as if from the beginning of a new document.\n", + ) + best_of: Optional[conint(ge=0, le=20)] = Field( + 1, + description='Generates `best_of` completions server-side and returns the "best" (the one with the highest log probability per token). Results cannot be streamed.\n\nWhen used with `n`, `best_of` controls the number of candidate completions and `n` specifies how many to return – `best_of` must be greater than `n`.\n\n**Note:** Because this parameter generates many completions, it can quickly consume your token quota. Use carefully and ensure that you have reasonable settings for `max_tokens` and `stop`.\n', + ) + echo: Optional[bool] = Field( + False, description="Echo back the prompt in addition to the completion\n" + ) + frequency_penalty: Optional[confloat(ge=-2.0, le=2.0)] = Field( + 0, + description="Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.\n\n[See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details)\n", + ) + logit_bias: Optional[Dict[str, int]] = Field( + None, + description='Modify the likelihood of specified tokens appearing in the completion.\n\nAccepts a JSON object that maps tokens (specified by their token ID in the GPT tokenizer) to an associated bias value from -100 to 100. You can use this [tokenizer tool](/tokenizer?view=bpe) to convert text to token IDs. Mathematically, the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should result in a ban or exclusive selection of the relevant token.\n\nAs an example, you can pass `{"50256": -100}` to prevent the <|endoftext|> token from being generated.\n', + ) + logprobs: Optional[conint(ge=0, le=5)] = Field( + None, + description="Include the log probabilities on the `logprobs` most likely output tokens, as well the chosen tokens. For example, if `logprobs` is 5, the API will return a list of the 5 most likely tokens. The API will always return the `logprob` of the sampled token, so there may be up to `logprobs+1` elements in the response.\n\nThe maximum value for `logprobs` is 5.\n", + ) + max_tokens: Optional[conint(ge=0)] = Field( + 16, + description="The maximum number of [tokens](/tokenizer) that can be generated in the completion.\n\nThe token count of your prompt plus `max_tokens` cannot exceed the model's context length. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens.\n", + example=16, + ) + n: Optional[conint(ge=1, le=128)] = Field( + 1, + description="How many completions to generate for each prompt.\n\n**Note:** Because this parameter generates many completions, it can quickly consume your token quota. Use carefully and ensure that you have reasonable settings for `max_tokens` and `stop`.\n", + example=1, + ) + presence_penalty: Optional[confloat(ge=-2.0, le=2.0)] = Field( + 0, + description="Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.\n\n[See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details)\n", + ) + seed: Optional[conint(ge=-9223372036854775808, le=9223372036854775807)] = Field( + None, + description="If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same `seed` and parameters should return the same result.\n\nDeterminism is not guaranteed, and you should refer to the `system_fingerprint` response parameter to monitor changes in the backend.\n", + ) + stop: Optional[Union[str, List[str]]] = Field( + None, + description="Up to 4 sequences where the API will stop generating further tokens. The returned text will not contain the stop sequence.\n", + ) + stream: Optional[bool] = Field( + False, + description="Whether to stream back partial progress. If set, tokens will be sent as data-only [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) as they become available, with the stream terminated by a `data: [DONE]` message. [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions).\n", + ) + suffix: Optional[str] = Field( + None, + description="The suffix that comes after a completion of inserted text.\n\nThis parameter is only supported for `gpt-3.5-turbo-instruct`.\n", + example="test.", + ) + temperature: Optional[confloat(ge=0.0, le=2.0)] = Field( + 1, + description="What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or `top_p` but not both.\n", + example=1, + ) + top_p: Optional[confloat(ge=0.0, le=1.0)] = Field( + 1, + description="An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.\n\nWe generally recommend altering this or `temperature` but not both.\n", + example=1, + ) + user: Optional[str] = Field( + None, + description="A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](/docs/guides/safety-best-practices/end-user-ids).\n", + example="user-1234", + ) + + +class FinishReason(Enum): + stop = "stop" + length = "length" + content_filter = "content_filter" + + +class Logprobs(BaseModel): + text_offset: Optional[List[int]] = None + token_logprobs: Optional[List[float]] = None + tokens: Optional[List[str]] = None + top_logprobs: Optional[List[Dict[str, float]]] = None + + +class Choice(BaseModel): + finish_reason: FinishReason = Field( + ..., + description="The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence,\n`length` if the maximum number of tokens specified in the request was reached,\nor `content_filter` if content was omitted due to a flag from our content filters.\n", + ) + index: int + logprobs: Logprobs + text: str + + +class Object1(Enum): + text_completion = "text_completion" + + +class Type(Enum): + image_url = "image_url" + + +class Detail(Enum): + auto = "auto" + low = "low" + high = "high" + + +class ImageUrl(BaseModel): + url: AnyUrl = Field( + ..., description="Either a URL of the image or the base64 encoded image data." + ) + detail: Optional[Detail] = Field( + "auto", + description="Specifies the detail level of the image. Learn more in the [Vision guide](/docs/guides/vision/low-or-high-fidelity-image-understanding).", + ) + + +class ChatCompletionRequestMessageContentPartImage(BaseModel): + type: Type = Field(..., description="The type of the content part.") + image_url: ImageUrl + + +class Type1(Enum): + text = "text" + + +class ChatCompletionRequestMessageContentPartText(BaseModel): + type: Type1 = Field(..., description="The type of the content part.") + text: str = Field(..., description="The text content.") + + +class Role(Enum): + system = "system" + + +class ChatCompletionRequestSystemMessage(BaseModel): + content: str = Field(..., description="The contents of the system message.") + role: Role = Field( + ..., description="The role of the messages author, in this case `system`." + ) + name: Optional[str] = Field( + None, + description="An optional name for the participant. Provides the model information to differentiate between participants of the same role.", + ) + + +class Role1(Enum): + user = "user" + + +class Role2(Enum): + assistant = "assistant" + + +class FunctionCall(BaseModel): + arguments: str = Field( + ..., + description="The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function.", + ) + name: str = Field(..., description="The name of the function to call.") + + +class Role3(Enum): + tool = "tool" + + +class ChatCompletionRequestToolMessage(BaseModel): + role: Role3 = Field( + ..., description="The role of the messages author, in this case `tool`." + ) + content: str = Field(..., description="The contents of the tool message.") + tool_call_id: str = Field( + ..., description="Tool call that this message is responding to." + ) + + +class Role4(Enum): + function = "function" + + +class ChatCompletionRequestFunctionMessage(BaseModel): + role: Role4 = Field( + ..., description="The role of the messages author, in this case `function`." + ) + content: str = Field(..., description="The contents of the function message.") + name: str = Field(..., description="The name of the function to call.") + + +class FunctionParameters(BaseModel): + pass + + class Config: + extra = Extra.allow + + +class ChatCompletionFunctions(BaseModel): + description: Optional[str] = Field( + None, + description="A description of what the function does, used by the model to choose when and how to call the function.", + ) + name: str = Field( + ..., + description="The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64.", + ) + parameters: Optional[FunctionParameters] = None + + +class ChatCompletionFunctionCallOption(BaseModel): + name: str = Field(..., description="The name of the function to call.") + + +class Type2(Enum): + function = "function" + + +class FunctionObject(BaseModel): + description: Optional[str] = Field( + None, + description="A description of what the function does, used by the model to choose when and how to call the function.", + ) + name: str = Field( + ..., + description="The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64.", + ) + parameters: Optional[FunctionParameters] = None + + +class ChatCompletionToolChoiceOption1(Enum): + none = "none" + auto = "auto" + required = "required" + + +class Function(BaseModel): + name: str = Field(..., description="The name of the function to call.") + + +class ChatCompletionNamedToolChoice(BaseModel): + type: Type2 = Field( + ..., + description="The type of the tool. Currently, only `function` is supported.", + ) + function: Function + + +class Function1(BaseModel): + name: str = Field(..., description="The name of the function to call.") + arguments: str = Field( + ..., + description="The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function.", + ) + + +class ChatCompletionMessageToolCall(BaseModel): + id: str = Field(..., description="The ID of the tool call.") + type: Type2 = Field( + ..., + description="The type of the tool. Currently, only `function` is supported.", + ) + function: Function1 = Field(..., description="The function that the model called.") + + +class Function2(BaseModel): + name: Optional[str] = Field(None, description="The name of the function to call.") + arguments: Optional[str] = Field( + None, + description="The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function.", + ) + + +class ChatCompletionMessageToolCallChunk(BaseModel): + index: int + id: Optional[str] = Field(None, description="The ID of the tool call.") + type: Optional[Type2] = Field( + None, + description="The type of the tool. Currently, only `function` is supported.", + ) + function: Optional[Function2] = None + + +class ChatCompletionRole(Enum): + system = "system" + user = "user" + assistant = "assistant" + tool = "tool" + function = "function" + + +class Role5(Enum): + assistant = "assistant" + + +class FunctionCall2(BaseModel): + arguments: Optional[str] = Field( + None, + description="The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function.", + ) + name: Optional[str] = Field(None, description="The name of the function to call.") + + +class Role6(Enum): + system = "system" + user = "user" + assistant = "assistant" + tool = "tool" + + +class ChatCompletionStreamResponseDelta(BaseModel): + content: Optional[str] = Field( + None, description="The contents of the chunk message." + ) + function_call: Optional[FunctionCall2] = Field( + None, + description="Deprecated and replaced by `tool_calls`. The name and arguments of a function that should be called, as generated by the model.", + ) + tool_calls: Optional[List[ChatCompletionMessageToolCallChunk]] = None + role: Optional[Role6] = Field( + None, description="The role of the author of this message." + ) + + +class Model2(Enum): + gpt_4_turbo = "gpt-4-turbo" + gpt_4_turbo_2024_04_09 = "gpt-4-turbo-2024-04-09" + gpt_4_0125_preview = "gpt-4-0125-preview" + gpt_4_turbo_preview = "gpt-4-turbo-preview" + gpt_4_1106_preview = "gpt-4-1106-preview" + gpt_4_vision_preview = "gpt-4-vision-preview" + gpt_4 = "gpt-4" + gpt_4_0314 = "gpt-4-0314" + gpt_4_0613 = "gpt-4-0613" + gpt_4_32k = "gpt-4-32k" + gpt_4_32k_0314 = "gpt-4-32k-0314" + gpt_4_32k_0613 = "gpt-4-32k-0613" + gpt_3_5_turbo = "gpt-3.5-turbo" + gpt_3_5_turbo_16k = "gpt-3.5-turbo-16k" + gpt_3_5_turbo_0301 = "gpt-3.5-turbo-0301" + gpt_3_5_turbo_0613 = "gpt-3.5-turbo-0613" + gpt_3_5_turbo_1106 = "gpt-3.5-turbo-1106" + gpt_3_5_turbo_0125 = "gpt-3.5-turbo-0125" + gpt_3_5_turbo_16k_0613 = "gpt-3.5-turbo-16k-0613" + + +class Type6(Enum): + text = "text" + json_object = "json_object" + + +class ResponseFormat(BaseModel): + type: Optional[Type6] = Field( + "text", + description="Must be one of `text` or `json_object`.", + example="json_object", + ) + + +class FunctionCall3(Enum): + none = "none" + auto = "auto" + + +class FinishReason1(Enum): + stop = "stop" + length = "length" + tool_calls = "tool_calls" + content_filter = "content_filter" + function_call = "function_call" + + +class Object2(Enum): + chat_completion = "chat.completion" + + +class FinishReason2(Enum): + stop = "stop" + length = "length" + function_call = "function_call" + content_filter = "content_filter" + + +class TopLogprob(BaseModel): + token: str = Field(..., description="The token.") + logprob: float = Field( + ..., + description="The log probability of this token, if it is within the top 20 most likely tokens. Otherwise, the value `-9999.0` is used to signify that the token is very unlikely.", + ) + bytes: List[int] = Field( + ..., + description="A list of integers representing the UTF-8 bytes representation of the token. Useful in instances where characters are represented by multiple tokens and their byte representations must be combined to generate the correct text representation. Can be `null` if there is no bytes representation for the token.", + ) + + +class ChatCompletionTokenLogprob(BaseModel): + token: str = Field(..., description="The token.") + logprob: float = Field( + ..., + description="The log probability of this token, if it is within the top 20 most likely tokens. Otherwise, the value `-9999.0` is used to signify that the token is very unlikely.", + ) + bytes: List[int] = Field( + ..., + description="A list of integers representing the UTF-8 bytes representation of the token. Useful in instances where characters are represented by multiple tokens and their byte representations must be combined to generate the correct text representation. Can be `null` if there is no bytes representation for the token.", + ) + top_logprobs: List[TopLogprob] = Field( + ..., + description="List of the most likely tokens and their log probability, at this token position. In rare cases, there may be fewer than the number of requested `top_logprobs` returned.", + ) + + +class Logprobs2(BaseModel): + content: List[ChatCompletionTokenLogprob] = Field( + ..., + description="A list of message content tokens with log probability information.", + ) + + +class FinishReason3(Enum): + stop = "stop" + length = "length" + tool_calls = "tool_calls" + content_filter = "content_filter" + function_call = "function_call" + + +class Choice3(BaseModel): + delta: ChatCompletionStreamResponseDelta + logprobs: Optional[Logprobs2] = Field( + None, description="Log probability information for the choice." + ) + finish_reason: FinishReason3 = Field( + ..., + description="The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence,\n`length` if the maximum number of tokens specified in the request was reached,\n`content_filter` if content was omitted due to a flag from our content filters,\n`tool_calls` if the model called a tool, or `function_call` (deprecated) if the model called a function.\n", + ) + index: int = Field( + ..., description="The index of the choice in the list of choices." + ) + + +class Object4(Enum): + chat_completion_chunk = "chat.completion.chunk" + + +class CreateChatCompletionStreamResponse(BaseModel): + id: str = Field( + ..., + description="A unique identifier for the chat completion. Each chunk has the same ID.", + ) + choices: List[Choice3] = Field( + ..., + description="A list of chat completion choices. Can be more than one if `n` is greater than 1.", + ) + created: int = Field( + ..., + description="The Unix timestamp (in seconds) of when the chat completion was created. Each chunk has the same timestamp.", + ) + model: str = Field(..., description="The model to generate the completion.") + system_fingerprint: Optional[str] = Field( + None, + description="This fingerprint represents the backend configuration that the model runs with.\nCan be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism.\n", + ) + object: Object4 = Field( + ..., description="The object type, which is always `chat.completion.chunk`." + ) + + +class CreateChatCompletionImageResponse(BaseModel): + pass + + +class Object5(Enum): + model = "model" + + +class Model(BaseModel): + id: str = Field( + ..., + description="The model identifier, which can be referenced in the API endpoints.", + ) + created: int = Field( + ..., description="The Unix timestamp (in seconds) when the model was created." + ) + object: Object5 = Field( + ..., description='The object type, which is always "model".' + ) + owned_by: str = Field(..., description="The organization that owns the model.") + + +class CompletionUsage(BaseModel): + completion_tokens: int = Field( + ..., description="Number of tokens in the generated completion." + ) + prompt_tokens: int = Field(..., description="Number of tokens in the prompt.") + total_tokens: int = Field( + ..., + description="Total number of tokens used in the request (prompt + completion).", + ) + + +class Event(Enum): + error = "error" + + +class ErrorEvent(BaseModel): + event: Event + data: Error + + +class Event1(Enum): + done = "done" + + +class Data(Enum): + field_DONE_ = "[DONE]" + + +class DoneEvent(BaseModel): + event: Event1 + data: Data + + +class ListModelsResponse(BaseModel): + object: Object + data: List[Model] + + +class CreateCompletionResponse(BaseModel): + id: str = Field(..., description="A unique identifier for the completion.") + choices: List[Choice] = Field( + ..., + description="The list of completion choices the model generated for the input prompt.", + ) + created: int = Field( + ..., + description="The Unix timestamp (in seconds) of when the completion was created.", + ) + model: str = Field(..., description="The model used for completion.") + system_fingerprint: Optional[str] = Field( + None, + description="This fingerprint represents the backend configuration that the model runs with.\n\nCan be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism.\n", + ) + object: Object1 = Field( + ..., description='The object type, which is always "text_completion"' + ) + usage: Optional[CompletionUsage] = None + + +class ChatCompletionRequestMessageContentPart(RootModel): + root: Union[ + ChatCompletionRequestMessageContentPartText, + ChatCompletionRequestMessageContentPartImage, + ] + + +class ChatCompletionRequestUserMessage(BaseModel): + content: Union[str, List[ChatCompletionRequestMessageContentPart]] = Field( + ..., description="The contents of the user message.\n" + ) + role: Role1 = Field( + ..., description="The role of the messages author, in this case `user`." + ) + name: Optional[str] = Field( + None, + description="An optional name for the participant. Provides the model information to differentiate between participants of the same role.", + ) + + +class ChatCompletionTool(BaseModel): + type: Type2 = Field( + ..., + description="The type of the tool. Currently, only `function` is supported.", + ) + function: FunctionObject + + +class ChatCompletionToolChoiceOption(RootModel): + root: Union[ChatCompletionToolChoiceOption1, ChatCompletionNamedToolChoice] = Field( + ..., + description='Controls which (if any) tool is called by the model.\n`none` means the model will not call any tool and instead generates a message.\n`auto` means the model can pick between generating a message or calling one or more tools.\n`required` means the model must call one or more tools.\nSpecifying a particular tool via `{"type": "function", "function": {"name": "my_function"}}` forces the model to call that tool.\n\n`none` is the default when no tools are present. `auto` is the default if tools are present.\n', + ) + + +class ChatCompletionMessageToolCalls(RootModel): + root: List[ChatCompletionMessageToolCall] = Field( + ..., + description="The tool calls generated by the model, such as function calls.", + ) + + +class ChatCompletionResponseMessage(BaseModel): + content: str = Field(..., description="The contents of the message.") + tool_calls: Optional[ChatCompletionMessageToolCalls] = None + role: Role5 = Field(..., description="The role of the author of this message.") + function_call: Optional[FunctionCall] = Field( + None, + description="Deprecated and replaced by `tool_calls`. The name and arguments of a function that should be called, as generated by the model.", + ) + + +class Choice1(BaseModel): + finish_reason: FinishReason1 = Field( + ..., + description="The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence,\n`length` if the maximum number of tokens specified in the request was reached,\n`content_filter` if content was omitted due to a flag from our content filters,\n`tool_calls` if the model called a tool, or `function_call` (deprecated) if the model called a function.\n", + ) + index: int = Field( + ..., description="The index of the choice in the list of choices." + ) + message: ChatCompletionResponseMessage + logprobs: Logprobs2 = Field( + ..., description="Log probability information for the choice." + ) + + +class CreateChatCompletionResponse(BaseModel): + id: str = Field(..., description="A unique identifier for the chat completion.") + choices: List[Choice1] = Field( + ..., + description="A list of chat completion choices. Can be more than one if `n` is greater than 1.", + ) + created: int = Field( + ..., + description="The Unix timestamp (in seconds) of when the chat completion was created.", + ) + model: str = Field(..., description="The model used for the chat completion.") + system_fingerprint: Optional[str] = Field( + None, + description="This fingerprint represents the backend configuration that the model runs with.\n\nCan be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism.\n", + ) + object: Object2 = Field( + ..., description="The object type, which is always `chat.completion`." + ) + usage: Optional[CompletionUsage] = None + + +class Choice2(BaseModel): + finish_reason: FinishReason2 = Field( + ..., + description="The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, `length` if the maximum number of tokens specified in the request was reached, `content_filter` if content was omitted due to a flag from our content filters, or `function_call` if the model called a function.\n", + ) + index: int = Field( + ..., description="The index of the choice in the list of choices." + ) + message: ChatCompletionResponseMessage + + +class CreateChatCompletionFunctionResponse(BaseModel): + id: str = Field(..., description="A unique identifier for the chat completion.") + choices: List[Choice2] = Field( + ..., + description="A list of chat completion choices. Can be more than one if `n` is greater than 1.", + ) + created: int = Field( + ..., + description="The Unix timestamp (in seconds) of when the chat completion was created.", + ) + model: str = Field(..., description="The model used for the chat completion.") + system_fingerprint: Optional[str] = Field( + None, + description="This fingerprint represents the backend configuration that the model runs with.\n\nCan be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism.\n", + ) + object: Object2 = Field( + ..., description="The object type, which is always `chat.completion`." + ) + usage: Optional[CompletionUsage] = None + + +class ChatCompletionRequestAssistantMessage(BaseModel): + content: Optional[str] = Field( + None, + description="The contents of the assistant message. Required unless `tool_calls` or `function_call` is specified.\n", + ) + role: Role2 = Field( + ..., description="The role of the messages author, in this case `assistant`." + ) + name: Optional[str] = Field( + None, + description="An optional name for the participant. Provides the model information to differentiate between participants of the same role.", + ) + tool_calls: Optional[ChatCompletionMessageToolCalls] = None + function_call: Optional[FunctionCall] = Field( + None, + description="Deprecated and replaced by `tool_calls`. The name and arguments of a function that should be called, as generated by the model.", + ) + + +class ChatCompletionRequestMessage(RootModel): + root: Union[ + ChatCompletionRequestSystemMessage, + ChatCompletionRequestUserMessage, + ChatCompletionRequestAssistantMessage, + ChatCompletionRequestToolMessage, + ChatCompletionRequestFunctionMessage, + ] + + +class CreateChatCompletionRequest(BaseModel): + messages: List[ChatCompletionRequestMessage] = Field( + ..., + description="A list of messages comprising the conversation so far. [Example Python code](https://cookbook.openai.com/examples/how_to_format_inputs_to_chatgpt_models).", + min_items=1, + ) + model: Union[str, Model2] = Field( + ..., + description="ID of the model to use. See the [model endpoint compatibility](/docs/models/model-endpoint-compatibility) table for details on which models work with the Chat API.", + example="gpt-4-turbo", + ) + frequency_penalty: Optional[confloat(ge=-2.0, le=2.0)] = Field( + 0, + description="Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.\n\n[See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details)\n", + ) + logit_bias: Optional[Dict[str, int]] = Field( + None, + description="Modify the likelihood of specified tokens appearing in the completion.\n\nAccepts a JSON object that maps tokens (specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically, the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should result in a ban or exclusive selection of the relevant token.\n", + ) + logprobs: Optional[bool] = Field( + False, + description="Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the `content` of `message`.", + ) + top_logprobs: Optional[conint(ge=0, le=20)] = Field( + None, + description="An integer between 0 and 20 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to `true` if this parameter is used.", + ) + max_tokens: Optional[int] = Field( + None, + description="The maximum number of [tokens](/tokenizer) that can be generated in the chat completion.\n\nThe total length of input tokens and generated tokens is limited by the model's context length. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens.\n", + ) + n: Optional[conint(ge=1, le=128)] = Field( + 1, + description="How many chat completion choices to generate for each input message. Note that you will be charged based on the number of generated tokens across all of the choices. Keep `n` as `1` to minimize costs.", + example=1, + ) + presence_penalty: Optional[confloat(ge=-2.0, le=2.0)] = Field( + 0, + description="Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.\n\n[See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details)\n", + ) + response_format: Optional[ResponseFormat] = Field( + None, + description='An object specifying the format that the model must output. Compatible with [GPT-4 Turbo](/docs/models/gpt-4-and-gpt-4-turbo) and all GPT-3.5 Turbo models newer than `gpt-3.5-turbo-1106`.\n\nSetting to `{ "type": "json_object" }` enables JSON mode, which guarantees the message the model generates is valid JSON.\n\n**Important:** when using JSON mode, you **must** also instruct the model to produce JSON yourself via a system or user message. Without this, the model may generate an unending stream of whitespace until the generation reaches the token limit, resulting in a long-running and seemingly "stuck" request. Also note that the message content may be partially cut off if `finish_reason="length"`, which indicates the generation exceeded `max_tokens` or the conversation exceeded the max context length.\n', + ) + seed: Optional[conint(ge=-9223372036854775808, le=9223372036854775807)] = Field( + None, + description="This feature is in Beta.\nIf specified, our system will make a best effort to sample deterministically, such that repeated requests with the same `seed` and parameters should return the same result.\nDeterminism is not guaranteed, and you should refer to the `system_fingerprint` response parameter to monitor changes in the backend.\n", + ) + stop: Optional[Union[str, List[str]]] = Field( + None, + description="Up to 4 sequences where the API will stop generating further tokens.\n", + ) + stream: Optional[bool] = Field( + False, + description="If set, partial message deltas will be sent, like in ChatGPT. Tokens will be sent as data-only [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) as they become available, with the stream terminated by a `data: [DONE]` message. [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions).\n", + ) + temperature: Optional[confloat(ge=0.0, le=2.0)] = Field( + 1, + description="What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or `top_p` but not both.\n", + example=1, + ) + top_p: Optional[confloat(ge=0.0, le=1.0)] = Field( + 1, + description="An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.\n\nWe generally recommend altering this or `temperature` but not both.\n", + example=1, + ) + tools: Optional[List[ChatCompletionTool]] = Field( + None, + description="A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of functions the model may generate JSON inputs for. A max of 128 functions are supported.\n", + ) + tool_choice: Optional[ChatCompletionToolChoiceOption] = None + user: Optional[str] = Field( + None, + description="A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](/docs/guides/safety-best-practices/end-user-ids).\n", + example="user-1234", + ) + function_call: Optional[ + Union[FunctionCall3, ChatCompletionFunctionCallOption] + ] = Field( + None, + description='Deprecated in favor of `tool_choice`.\n\nControls which (if any) function is called by the model.\n`none` means the model will not call a function and instead generates a message.\n`auto` means the model can pick between generating a message or calling a function.\nSpecifying a particular function via `{"name": "my_function"}` forces the model to call that function.\n\n`none` is the default when no functions are present. `auto` is the default if functions are present.\n', + ) + functions: Optional[List[ChatCompletionFunctions]] = Field( + None, + description="Deprecated in favor of `tools`.\n\nA list of functions the model may generate JSON inputs for.\n", + max_items=128, + min_items=1, + ) diff --git a/Triton_Inference_Server_Python_API/run.sh b/Triton_Inference_Server_Python_API/run.sh index a705b21a..f48b91aa 100755 --- a/Triton_Inference_Server_Python_API/run.sh +++ b/Triton_Inference_Server_Python_API/run.sh @@ -142,7 +142,7 @@ fi $RUN_PREFIX mkdir -p backend/diffusion -$RUN_PREFIX docker run --gpus all -it --rm --network host --shm-size=10G --ulimit memlock=-1 --ulimit stack=67108864 -eHF_TOKEN -eGITHUB_TOKEN -eAWS_DEFAULT_REGION -eAWS_ACCESS_KEY_ID -eAWS_SECRET_ACCESS_KEY -eS3_BUCKET_URL -v ${SOURCE_DIR}:/workspace -v${SOURCE_DIR}/.cache/huggingface:/root/.cache/huggingface -w /workspace -v${SOURCE_DIR}/backend/diffusion:/opt/tritonserver/backends/diffusion -v/tmp:/tmp $IMAGE +$RUN_PREFIX docker run --gpus all -it --rm --network host --shm-size=10G --ulimit memlock=-1 --ulimit stack=67108864 -eHF_TOKEN -eGITHUB_TOKEN -eAWS_DEFAULT_REGION -eAWS_ACCESS_KEY_ID -eAWS_SECRET_ACCESS_KEY -eS3_BUCKET_URL -v ${SOURCE_DIR}:/workspace -v${SOURCE_DIR}/.cache/huggingface:/root/.cache/huggingface -w /workspace -v${SOURCE_DIR}/backend/diffusion:/opt/tritonserver/backends/diffusion -v/tmp:/tmp -v${SOURCE_DIR}/examples/litellm/triton.py:/usr/local/lib/python3.10/dist-packages/litellm/llms/triton.py $IMAGE { set +x; } 2>/dev/null From 2b27882378a04730f3d2c801e42ff8e72bc2193c Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sat, 4 May 2024 13:36:52 -0700 Subject: [PATCH 12/62] more elaborate openai server --- .../examples/fastapi/openai-server/.flake8 | 3 + .../examples/fastapi/openai-server/.gitignore | 138 + .../openai-server/.openapi-generator-ignore | 23 + .../openai-server/.openapi-generator/FILES | 82 + .../openai-server/.openapi-generator/VERSION | 1 + .../examples/fastapi/openai-server/Dockerfile | 30 + .../examples/fastapi/openai-server/README.md | 39 + .../fastapi/openai-server/docker-compose.yaml | 9 + .../fastapi/openai-server/openapi.yaml | 3148 +++++++++++++++++ .../fastapi/openai-server/pyproject.toml | 30 + .../fastapi/openai-server/requirements.txt | 36 + .../examples/fastapi/openai-server/setup.cfg | 20 + .../src/openapi_server/apis/__init__.py | 0 .../src/openapi_server/apis/chat_api.py | 54 + .../src/openapi_server/apis/chat_api_base.py | 25 + .../openapi_server/apis/completions_api.py | 101 + .../apis/completions_api_base.py | 22 + .../src/openapi_server/apis/models_api.py | 80 + .../openapi_server/apis/models_api_base.py | 33 + .../src/openapi_server/impl/__init__.py | 0 .../openai-server/src/openapi_server/main.py | 29 + .../src/openapi_server/models/__init__.py | 0 .../chat_completion_function_call_option.py | 85 + .../models/chat_completion_functions.py | 101 + .../chat_completion_message_tool_call.py | 113 + ...chat_completion_message_tool_call_chunk.py | 121 + ...letion_message_tool_call_chunk_function.py | 93 + ...t_completion_message_tool_call_function.py | 90 + .../chat_completion_named_tool_choice.py | 111 + ...t_completion_named_tool_choice_function.py | 85 + ...at_completion_request_assistant_message.py | 152 + ...request_assistant_message_function_call.py | 90 + ...hat_completion_request_function_message.py | 109 + .../models/chat_completion_request_message.py | 261 ++ ...completion_request_message_content_part.py | 196 + ...tion_request_message_content_part_image.py | 109 + ...st_message_content_part_image_image_url.py | 108 + ...etion_request_message_content_part_text.py | 93 + .../chat_completion_request_system_message.py | 105 + .../chat_completion_request_tool_message.py | 104 + .../chat_completion_request_user_message.py | 116 + ...completion_request_user_message_content.py | 192 + .../chat_completion_response_message.py | 141 + .../models/chat_completion_role.py | 45 + .../chat_completion_stream_response_delta.py | 147 + ...ion_stream_response_delta_function_call.py | 93 + .../models/chat_completion_token_logprob.py | 122 + ...letion_token_logprob_top_logprobs_inner.py | 102 + .../models/chat_completion_tool.py | 107 + .../chat_completion_tool_choice_option.py | 179 + .../openapi_server/models/completion_usage.py | 101 + ...reate_chat_completion_function_response.py | 147 + ...pletion_function_response_choices_inner.py | 115 + .../models/create_chat_completion_request.py | 377 ++ ...e_chat_completion_request_function_call.py | 184 + .../create_chat_completion_request_model.py | 161 + ...chat_completion_request_response_format.py | 99 + .../create_chat_completion_request_stop.py | 176 + .../models/create_chat_completion_response.py | 147 + ..._chat_completion_response_choices_inner.py | 143 + ...pletion_response_choices_inner_logprobs.py | 112 + .../create_chat_completion_stream_response.py | 140 + ...ompletion_stream_response_choices_inner.py | 146 + .../models/create_completion_request.py | 320 ++ .../models/create_completion_request_model.py | 161 + .../create_completion_request_prompt.py | 230 ++ .../models/create_completion_request_stop.py | 182 + .../models/create_completion_response.py | 147 + ...reate_completion_response_choices_inner.py | 122 + ...pletion_response_choices_inner_logprobs.py | 100 + .../models/delete_model_response.py | 93 + .../src/openapi_server/models/done_event.py | 100 + .../src/openapi_server/models/error.py | 105 + .../src/openapi_server/models/error_event.py | 105 + .../openapi_server/models/error_response.py | 96 + .../src/openapi_server/models/extra_models.py | 9 + .../openapi_server/models/function_object.py | 101 + .../models/list_models_response.py | 109 + .../src/openapi_server/models/model.py | 106 + .../src/openapi_server/security_api.py | 40 + .../fastapi/openai-server/tests/conftest.py | 17 + .../openai-server/tests/test_chat_api.py | 79 + .../tests/test_completions_api.py | 50 + .../openai-server/tests/test_models_api.py | 69 + 84 files changed, 11662 insertions(+) create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.flake8 create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.gitignore create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator-ignore create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator/FILES create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator/VERSION create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/Dockerfile create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/README.md create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/docker-compose.yaml create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/openapi.yaml create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/pyproject.toml create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/requirements.txt create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/setup.cfg create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/__init__.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/chat_api.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/chat_api_base.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/completions_api.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/completions_api_base.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/models_api.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/models_api_base.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/impl/__init__.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/main.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/__init__.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_function_call_option.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_functions.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk_function.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_function.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_named_tool_choice.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_named_tool_choice_function.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_assistant_message.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_assistant_message_function_call.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_function_message.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image_image_url.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_text.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_system_message.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_tool_message.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_user_message.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_user_message_content.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_response_message.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_role.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_stream_response_delta.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_stream_response_delta_function_call.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_token_logprob.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_token_logprob_top_logprobs_inner.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_tool.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_tool_choice_option.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/completion_usage.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_function_response.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_function_response_choices_inner.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_function_call.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_model.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_response_format.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_stop.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner_logprobs.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_stream_response.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_stream_response_choices_inner.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_model.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_prompt.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_stop.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response_choices_inner.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response_choices_inner_logprobs.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/delete_model_response.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/done_event.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error_event.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error_response.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/extra_models.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/function_object.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/list_models_response.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/model.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/security_api.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/conftest.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_chat_api.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_completions_api.py create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_models_api.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.flake8 b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.flake8 new file mode 100644 index 00000000..9e008c5b --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 88 +exclude = .git,__pycache__,__init__.py,.mypy_cache,.pytest_cache,.venv diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.gitignore b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.gitignore new file mode 100644 index 00000000..a81c8ee1 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.gitignore @@ -0,0 +1,138 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator-ignore b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator-ignore new file mode 100644 index 00000000..7484ee59 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator-ignore @@ -0,0 +1,23 @@ +# OpenAPI Generator Ignore +# Generated by openapi-generator https://github.com/openapitools/openapi-generator + +# Use this file to prevent files from being overwritten by the generator. +# The patterns follow closely to .gitignore or .dockerignore. + +# As an example, the C# client generator defines ApiClient.cs. +# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line: +#ApiClient.cs + +# You can match any string of characters against a directory, file or extension with a single asterisk (*): +#foo/*/qux +# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux + +# You can recursively match patterns against a directory, file or extension with a double asterisk (**): +#foo/**/qux +# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux + +# You can also negate patterns with an exclamation (!). +# For example, you can ignore all files in a docs folder with the file extension .md: +#docs/*.md +# Then explicitly reverse the ignore rule for a single file: +#!docs/README.md diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator/FILES b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator/FILES new file mode 100644 index 00000000..3dab015b --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator/FILES @@ -0,0 +1,82 @@ +.flake8 +.gitignore +.openapi-generator-ignore +Dockerfile +README.md +docker-compose.yaml +openapi.yaml +pyproject.toml +requirements.txt +setup.cfg +src/openapi_server/apis/__init__.py +src/openapi_server/apis/chat_api.py +src/openapi_server/apis/chat_api_base.py +src/openapi_server/apis/completions_api.py +src/openapi_server/apis/completions_api_base.py +src/openapi_server/apis/models_api.py +src/openapi_server/apis/models_api_base.py +src/openapi_server/impl/__init__.py +src/openapi_server/main.py +src/openapi_server/models/__init__.py +src/openapi_server/models/chat_completion_function_call_option.py +src/openapi_server/models/chat_completion_functions.py +src/openapi_server/models/chat_completion_message_tool_call.py +src/openapi_server/models/chat_completion_message_tool_call_chunk.py +src/openapi_server/models/chat_completion_message_tool_call_chunk_function.py +src/openapi_server/models/chat_completion_message_tool_call_function.py +src/openapi_server/models/chat_completion_named_tool_choice.py +src/openapi_server/models/chat_completion_named_tool_choice_function.py +src/openapi_server/models/chat_completion_request_assistant_message.py +src/openapi_server/models/chat_completion_request_assistant_message_function_call.py +src/openapi_server/models/chat_completion_request_function_message.py +src/openapi_server/models/chat_completion_request_message.py +src/openapi_server/models/chat_completion_request_message_content_part.py +src/openapi_server/models/chat_completion_request_message_content_part_image.py +src/openapi_server/models/chat_completion_request_message_content_part_image_image_url.py +src/openapi_server/models/chat_completion_request_message_content_part_text.py +src/openapi_server/models/chat_completion_request_system_message.py +src/openapi_server/models/chat_completion_request_tool_message.py +src/openapi_server/models/chat_completion_request_user_message.py +src/openapi_server/models/chat_completion_request_user_message_content.py +src/openapi_server/models/chat_completion_response_message.py +src/openapi_server/models/chat_completion_role.py +src/openapi_server/models/chat_completion_stream_response_delta.py +src/openapi_server/models/chat_completion_stream_response_delta_function_call.py +src/openapi_server/models/chat_completion_token_logprob.py +src/openapi_server/models/chat_completion_token_logprob_top_logprobs_inner.py +src/openapi_server/models/chat_completion_tool.py +src/openapi_server/models/chat_completion_tool_choice_option.py +src/openapi_server/models/completion_usage.py +src/openapi_server/models/create_chat_completion_function_response.py +src/openapi_server/models/create_chat_completion_function_response_choices_inner.py +src/openapi_server/models/create_chat_completion_request.py +src/openapi_server/models/create_chat_completion_request_function_call.py +src/openapi_server/models/create_chat_completion_request_model.py +src/openapi_server/models/create_chat_completion_request_response_format.py +src/openapi_server/models/create_chat_completion_request_stop.py +src/openapi_server/models/create_chat_completion_response.py +src/openapi_server/models/create_chat_completion_response_choices_inner.py +src/openapi_server/models/create_chat_completion_response_choices_inner_logprobs.py +src/openapi_server/models/create_chat_completion_stream_response.py +src/openapi_server/models/create_chat_completion_stream_response_choices_inner.py +src/openapi_server/models/create_completion_request.py +src/openapi_server/models/create_completion_request_model.py +src/openapi_server/models/create_completion_request_prompt.py +src/openapi_server/models/create_completion_request_stop.py +src/openapi_server/models/create_completion_response.py +src/openapi_server/models/create_completion_response_choices_inner.py +src/openapi_server/models/create_completion_response_choices_inner_logprobs.py +src/openapi_server/models/delete_model_response.py +src/openapi_server/models/done_event.py +src/openapi_server/models/error.py +src/openapi_server/models/error_event.py +src/openapi_server/models/error_response.py +src/openapi_server/models/extra_models.py +src/openapi_server/models/function_object.py +src/openapi_server/models/list_models_response.py +src/openapi_server/models/model.py +src/openapi_server/security_api.py +tests/conftest.py +tests/test_chat_api.py +tests/test_completions_api.py +tests/test_models_api.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator/VERSION b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator/VERSION new file mode 100644 index 00000000..ecb21862 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator/VERSION @@ -0,0 +1 @@ +7.6.0-SNAPSHOT diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/Dockerfile b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/Dockerfile new file mode 100644 index 00000000..b66458eb --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/Dockerfile @@ -0,0 +1,30 @@ +FROM python:3.7 AS builder + +WORKDIR /usr/src/app + +RUN python3 -m venv /venv +ENV PATH="/venv/bin:$PATH" + +RUN pip install --upgrade pip + +COPY . . +RUN pip install --no-cache-dir . + + +FROM python:3.7 AS test_runner +WORKDIR /tmp +COPY --from=builder /venv /venv +COPY --from=builder /usr/src/app/tests tests +ENV PATH=/venv/bin:$PATH + +# install test dependencies +RUN pip install pytest + +# run tests +RUN pytest tests + + +FROM python:3.7 AS service +WORKDIR /root/app/site-packages +COPY --from=test_runner /venv /venv +ENV PATH=/venv/bin:$PATH diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/README.md b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/README.md new file mode 100644 index 00000000..3bd860d7 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/README.md @@ -0,0 +1,39 @@ +# OpenAPI generated FastAPI server + +This Python package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project: + +- API version: 2.0.0 +- Generator version: 7.6.0-SNAPSHOT +- Build package: org.openapitools.codegen.languages.PythonFastAPIServerCodegen + +## Requirements. + +Python >= 3.7 + +## Installation & Usage + +To run the server, please execute the following from the root directory: + +```bash +pip3 install -r requirements.txt +PYTHONPATH=src uvicorn openapi_server.main:app --host 0.0.0.0 --port 8080 +``` + +and open your browser at `http://localhost:8080/docs/` to see the docs. + +## Running with Docker + +To run the server on a Docker container, please execute the following from the root directory: + +```bash +docker-compose up --build +``` + +## Tests + +To run the tests: + +```bash +pip3 install pytest +PYTHONPATH=src pytest tests +``` diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/docker-compose.yaml b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/docker-compose.yaml new file mode 100644 index 00000000..638e4e00 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/docker-compose.yaml @@ -0,0 +1,9 @@ +version: '3.6' +services: + service: + build: + context: . + target: service + ports: + - "8080:8080" + command: uvicorn openapi_server.main:app --host 0.0.0.0 --port 8080 diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/openapi.yaml b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/openapi.yaml new file mode 100644 index 00000000..6a1ff92b --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/openapi.yaml @@ -0,0 +1,3148 @@ +openapi: 3.0.0 +info: + contact: + name: OpenAI Support + url: https://help.openai.com/ + description: The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference + for more details. + license: + name: MIT + url: https://github.com/openai/openai-openapi/blob/master/LICENSE + termsOfService: https://openai.com/policies/terms-of-use + title: OpenAI API + version: 2.0.0 +servers: +- url: https://api.openai.com/v1 +security: +- ApiKeyAuth: [] +tags: +- description: "Given a list of messages comprising a conversation, the model will\ + \ return a response." + name: Chat +- description: "Given a prompt, the model will return one or more predicted completions,\ + \ and can also return the probabilities of alternative tokens at each position." + name: Completions +paths: + /chat/completions: + post: + operationId: createChatCompletion + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CreateChatCompletionRequest' + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/CreateChatCompletionResponse' + description: OK + summary: Creates a model response for the given chat conversation. + tags: + - Chat + x-oaiMeta: + name: Create chat completion + group: chat + returns: | + Returns a [chat completion](/docs/api-reference/chat/object) object, or a streamed sequence of [chat completion chunk](/docs/api-reference/chat/streaming) objects if the request is streamed. + path: create + examples: + - title: Default + request: + curl: | + curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "VAR_model_id", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Hello!" + } + ] + }' + python: | + from openai import OpenAI + client = OpenAI() + + completion = client.chat.completions.create( + model="VAR_model_id", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"} + ] + ) + + print(completion.choices[0].message) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const completion = await openai.chat.completions.create({ + messages: [{ role: "system", content: "You are a helpful assistant." }], + model: "VAR_model_id", + }); + + console.log(completion.choices[0]); + } + + main(); + response: | + { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-3.5-turbo-0125", + "system_fingerprint": "fp_44709d6fcb", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "\n\nHello there, how may I assist you today?", + }, + "logprobs": null, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } + } + - title: Image input + request: + curl: | + curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "gpt-4-turbo", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What'\''s in this image?" + }, + { + "type": "image_url", + "image_url": { + "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + } + } + ] + } + ], + "max_tokens": 300 + }' + python: | + from openai import OpenAI + + client = OpenAI() + + response = client.chat.completions.create( + model="gpt-4-turbo", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", + }, + ], + } + ], + max_tokens=300, + ) + + print(response.choices[0]) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const response = await openai.chat.completions.create({ + model: "gpt-4-turbo", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What's in this image?" }, + { + type: "image_url", + image_url: + "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", + }, + ], + }, + ], + }); + console.log(response.choices[0]); + } + main(); + response: | + { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-3.5-turbo-0125", + "system_fingerprint": "fp_44709d6fcb", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "\n\nThis image shows a wooden boardwalk extending through a lush green marshland.", + }, + "logprobs": null, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } + } + - title: Streaming + request: + curl: | + curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "VAR_model_id", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Hello!" + } + ], + "stream": true + }' + python: | + from openai import OpenAI + client = OpenAI() + + completion = client.chat.completions.create( + model="VAR_model_id", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"} + ], + stream=True + ) + + for chunk in completion: + print(chunk.choices[0].delta) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const completion = await openai.chat.completions.create({ + model: "VAR_model_id", + messages: [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"} + ], + stream: true, + }); + + for await (const chunk of completion) { + console.log(chunk.choices[0].delta.content); + } + } + + main(); + response: | + {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0125", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]} + + {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0125", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}]} + + .... + + {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0125", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}]} + - title: Functions + request: + curl: | + curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "gpt-4-turbo", + "messages": [ + { + "role": "user", + "content": "What'\''s the weather like in Boston today?" + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"] + } + }, + "required": ["location"] + } + } + } + ], + "tool_choice": "auto" + }' + python: | + from openai import OpenAI + client = OpenAI() + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + } + ] + messages = [{"role": "user", "content": "What's the weather like in Boston today?"}] + completion = client.chat.completions.create( + model="VAR_model_id", + messages=messages, + tools=tools, + tool_choice="auto" + ) + + print(completion) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const messages = [{"role": "user", "content": "What's the weather like in Boston today?"}]; + const tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + } + ]; + + const response = await openai.chat.completions.create({ + model: "gpt-4-turbo", + messages: messages, + tools: tools, + tool_choice: "auto", + }); + + console.log(response); + } + + main(); + response: | + { + "id": "chatcmpl-abc123", + "object": "chat.completion", + "created": 1699896916, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\n\"location\": \"Boston, MA\"\n}" + } + } + ] + }, + "logprobs": null, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 82, + "completion_tokens": 17, + "total_tokens": 99 + } + } + - title: Logprobs + request: + curl: | + curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "VAR_model_id", + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ], + "logprobs": true, + "top_logprobs": 2 + }' + python: | + from openai import OpenAI + client = OpenAI() + + completion = client.chat.completions.create( + model="VAR_model_id", + messages=[ + {"role": "user", "content": "Hello!"} + ], + logprobs=True, + top_logprobs=2 + ) + + print(completion.choices[0].message) + print(completion.choices[0].logprobs) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const completion = await openai.chat.completions.create({ + messages: [{ role: "user", content: "Hello!" }], + model: "VAR_model_id", + logprobs: true, + top_logprobs: 2, + }); + + console.log(completion.choices[0]); + } + + main(); + response: | + { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1702685778, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + }, + "logprobs": { + "content": [ + { + "token": "Hello", + "logprob": -0.31725305, + "bytes": [72, 101, 108, 108, 111], + "top_logprobs": [ + { + "token": "Hello", + "logprob": -0.31725305, + "bytes": [72, 101, 108, 108, 111] + }, + { + "token": "Hi", + "logprob": -1.3190403, + "bytes": [72, 105] + } + ] + }, + { + "token": "!", + "logprob": -0.02380986, + "bytes": [ + 33 + ], + "top_logprobs": [ + { + "token": "!", + "logprob": -0.02380986, + "bytes": [33] + }, + { + "token": " there", + "logprob": -3.787621, + "bytes": [32, 116, 104, 101, 114, 101] + } + ] + }, + { + "token": " How", + "logprob": -0.000054669687, + "bytes": [32, 72, 111, 119], + "top_logprobs": [ + { + "token": " How", + "logprob": -0.000054669687, + "bytes": [32, 72, 111, 119] + }, + { + "token": "<|end|>", + "logprob": -10.953937, + "bytes": null + } + ] + }, + { + "token": " can", + "logprob": -0.015801601, + "bytes": [32, 99, 97, 110], + "top_logprobs": [ + { + "token": " can", + "logprob": -0.015801601, + "bytes": [32, 99, 97, 110] + }, + { + "token": " may", + "logprob": -4.161023, + "bytes": [32, 109, 97, 121] + } + ] + }, + { + "token": " I", + "logprob": -3.7697225e-6, + "bytes": [ + 32, + 73 + ], + "top_logprobs": [ + { + "token": " I", + "logprob": -3.7697225e-6, + "bytes": [32, 73] + }, + { + "token": " assist", + "logprob": -13.596657, + "bytes": [32, 97, 115, 115, 105, 115, 116] + } + ] + }, + { + "token": " assist", + "logprob": -0.04571125, + "bytes": [32, 97, 115, 115, 105, 115, 116], + "top_logprobs": [ + { + "token": " assist", + "logprob": -0.04571125, + "bytes": [32, 97, 115, 115, 105, 115, 116] + }, + { + "token": " help", + "logprob": -3.1089056, + "bytes": [32, 104, 101, 108, 112] + } + ] + }, + { + "token": " you", + "logprob": -5.4385737e-6, + "bytes": [32, 121, 111, 117], + "top_logprobs": [ + { + "token": " you", + "logprob": -5.4385737e-6, + "bytes": [32, 121, 111, 117] + }, + { + "token": " today", + "logprob": -12.807695, + "bytes": [32, 116, 111, 100, 97, 121] + } + ] + }, + { + "token": " today", + "logprob": -0.0040071653, + "bytes": [32, 116, 111, 100, 97, 121], + "top_logprobs": [ + { + "token": " today", + "logprob": -0.0040071653, + "bytes": [32, 116, 111, 100, 97, 121] + }, + { + "token": "?", + "logprob": -5.5247097, + "bytes": [63] + } + ] + }, + { + "token": "?", + "logprob": -0.0008108172, + "bytes": [63], + "top_logprobs": [ + { + "token": "?", + "logprob": -0.0008108172, + "bytes": [63] + }, + { + "token": "?\n", + "logprob": -7.184561, + "bytes": [63, 10] + } + ] + } + ] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 9, + "total_tokens": 18 + }, + "system_fingerprint": null + } + /completions: + post: + operationId: createCompletion + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CreateCompletionRequest' + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/CreateCompletionResponse' + description: OK + summary: Creates a completion for the provided prompt and parameters. + tags: + - Completions + x-oaiMeta: + name: Create completion + group: completions + returns: | + Returns a [completion](/docs/api-reference/completions/object) object, or a sequence of completion objects if the request is streamed. + legacy: true + examples: + - title: No streaming + request: + curl: | + curl https://api.openai.com/v1/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "VAR_model_id", + "prompt": "Say this is a test", + "max_tokens": 7, + "temperature": 0 + }' + python: | + from openai import OpenAI + client = OpenAI() + + client.completions.create( + model="VAR_model_id", + prompt="Say this is a test", + max_tokens=7, + temperature=0 + ) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const completion = await openai.completions.create({ + model: "VAR_model_id", + prompt: "Say this is a test.", + max_tokens: 7, + temperature: 0, + }); + + console.log(completion); + } + main(); + response: | + { + "id": "cmpl-uqkvlQyYK7bGYrRHQ0eXlWi7", + "object": "text_completion", + "created": 1589478378, + "model": "VAR_model_id", + "system_fingerprint": "fp_44709d6fcb", + "choices": [ + { + "text": "\n\nThis is indeed a test", + "index": 0, + "logprobs": null, + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 7, + "total_tokens": 12 + } + } + - title: Streaming + request: + curl: | + curl https://api.openai.com/v1/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "VAR_model_id", + "prompt": "Say this is a test", + "max_tokens": 7, + "temperature": 0, + "stream": true + }' + python: | + from openai import OpenAI + client = OpenAI() + + for chunk in client.completions.create( + model="VAR_model_id", + prompt="Say this is a test", + max_tokens=7, + temperature=0, + stream=True + ): + print(chunk.choices[0].text) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const stream = await openai.completions.create({ + model: "VAR_model_id", + prompt: "Say this is a test.", + stream: true, + }); + + for await (const chunk of stream) { + console.log(chunk.choices[0].text) + } + } + main(); + response: "{\n \"id\": \"cmpl-7iA7iJjj8V2zOkCGvWF2hAkDWBQZe\",\n \"object\"\ + : \"text_completion\",\n \"created\": 1690759702,\n \"choices\": [\n\ + \ {\n \"text\": \"This\",\n \"index\": 0,\n \"logprobs\"\ + : null,\n \"finish_reason\": null\n }\n ],\n \"model\": \"gpt-3.5-turbo-instruct\"\ + \n \"system_fingerprint\": \"fp_44709d6fcb\",\n} \n" + /models: + get: + operationId: listModels + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/ListModelsResponse' + description: OK + summary: "Lists the currently available models, and provides basic information\ + \ about each one such as the owner and availability." + tags: + - Models + x-oaiMeta: + name: List models + group: models + returns: "A list of [model](/docs/api-reference/models/object) objects." + examples: + request: + curl: | + curl https://api.openai.com/v1/models \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.models.list() + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const list = await openai.models.list(); + + for await (const model of list) { + console.log(model); + } + } + main(); + response: | + { + "object": "list", + "data": [ + { + "id": "model-id-0", + "object": "model", + "created": 1686935002, + "owned_by": "organization-owner" + }, + { + "id": "model-id-1", + "object": "model", + "created": 1686935002, + "owned_by": "organization-owner", + }, + { + "id": "model-id-2", + "object": "model", + "created": 1686935002, + "owned_by": "openai" + }, + ], + "object": "list" + } + /models/{model}: + delete: + operationId: deleteModel + parameters: + - description: The model to delete + explode: false + in: path + name: model + required: true + schema: + example: ft:gpt-3.5-turbo:acemeco:suffix:abc123 + type: string + style: simple + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/DeleteModelResponse' + description: OK + summary: Delete a fine-tuned model. You must have the Owner role in your organization + to delete a model. + tags: + - Models + x-oaiMeta: + name: Delete a fine-tuned model + group: models + returns: Deletion status. + examples: + request: + curl: | + curl https://api.openai.com/v1/models/ft:gpt-3.5-turbo:acemeco:suffix:abc123 \ + -X DELETE \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.models.delete("ft:gpt-3.5-turbo:acemeco:suffix:abc123") + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const model = await openai.models.del("ft:gpt-3.5-turbo:acemeco:suffix:abc123"); + + console.log(model); + } + main(); + response: | + { + "id": "ft:gpt-3.5-turbo:acemeco:suffix:abc123", + "object": "model", + "deleted": true + } + get: + operationId: retrieveModel + parameters: + - description: The ID of the model to use for this request + explode: false + in: path + name: model + required: true + schema: + example: gpt-3.5-turbo + type: string + style: simple + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/Model' + description: OK + summary: "Retrieves a model instance, providing basic information about the\ + \ model such as the owner and permissioning." + tags: + - Models + x-oaiMeta: + name: Retrieve model + group: models + returns: "The [model](/docs/api-reference/models/object) object matching the\ + \ specified ID." + examples: + request: + curl: | + curl https://api.openai.com/v1/models/VAR_model_id \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.models.retrieve("VAR_model_id") + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const model = await openai.models.retrieve("VAR_model_id"); + + console.log(model); + } + + main(); + response: | + { + "id": "VAR_model_id", + "object": "model", + "created": 1686935002, + "owned_by": "openai" + } +components: + schemas: + Error: + properties: + code: + nullable: true + title: code + type: string + message: + nullable: false + title: message + type: string + param: + nullable: true + title: param + type: string + type: + nullable: false + title: type + type: string + required: + - code + - message + - param + - type + title: Error + type: object + ErrorResponse: + properties: + error: + $ref: '#/components/schemas/Error' + required: + - error + type: object + ListModelsResponse: + example: + data: + - created: 0 + owned_by: owned_by + id: id + object: model + - created: 0 + owned_by: owned_by + id: id + object: model + object: list + properties: + object: + enum: + - list + title: object + type: string + data: + items: + $ref: '#/components/schemas/Model' + title: data + type: array + required: + - data + - object + title: ListModelsResponse + type: object + DeleteModelResponse: + example: + deleted: true + id: id + object: object + properties: + id: + title: id + type: string + deleted: + title: deleted + type: boolean + object: + title: object + type: string + required: + - deleted + - id + - object + title: DeleteModelResponse + type: object + CreateCompletionRequest: + example: + logit_bias: + key: 1 + seed: -2147483648 + max_tokens: 16 + presence_penalty: 0.25495066265333133 + echo: false + suffix: test. + "n": 1 + logprobs: 2 + top_p: 1 + frequency_penalty: 0.4109824732281613 + best_of: 1 + stop: |2+ + + stream: false + temperature: 1 + model: CreateCompletionRequest_model + prompt: This is a test. + user: user-1234 + properties: + model: + $ref: '#/components/schemas/CreateCompletionRequest_model' + prompt: + $ref: '#/components/schemas/CreateCompletionRequest_prompt' + best_of: + default: 1 + description: | + Generates `best_of` completions server-side and returns the "best" (the one with the highest log probability per token). Results cannot be streamed. + + When used with `n`, `best_of` controls the number of candidate completions and `n` specifies how many to return – `best_of` must be greater than `n`. + + **Note:** Because this parameter generates many completions, it can quickly consume your token quota. Use carefully and ensure that you have reasonable settings for `max_tokens` and `stop`. + maximum: 20 + minimum: 0 + nullable: true + title: best_of + type: integer + echo: + default: false + description: | + Echo back the prompt in addition to the completion + nullable: true + title: echo + type: boolean + frequency_penalty: + default: 0 + description: | + Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. + + [See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details) + maximum: 2 + minimum: -2 + nullable: true + title: frequency_penalty + type: number + logit_bias: + additionalProperties: + type: integer + description: | + Modify the likelihood of specified tokens appearing in the completion. + + Accepts a JSON object that maps tokens (specified by their token ID in the GPT tokenizer) to an associated bias value from -100 to 100. You can use this [tokenizer tool](/tokenizer?view=bpe) to convert text to token IDs. Mathematically, the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should result in a ban or exclusive selection of the relevant token. + + As an example, you can pass `{"50256": -100}` to prevent the <|endoftext|> token from being generated. + nullable: true + title: logit_bias + type: object + x-oaiTypeLabel: map + logprobs: + description: | + Include the log probabilities on the `logprobs` most likely output tokens, as well the chosen tokens. For example, if `logprobs` is 5, the API will return a list of the 5 most likely tokens. The API will always return the `logprob` of the sampled token, so there may be up to `logprobs+1` elements in the response. + + The maximum value for `logprobs` is 5. + maximum: 5 + minimum: 0 + nullable: true + title: logprobs + type: integer + max_tokens: + default: 16 + description: | + The maximum number of [tokens](/tokenizer) that can be generated in the completion. + + The token count of your prompt plus `max_tokens` cannot exceed the model's context length. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens. + example: 16 + minimum: 0 + nullable: true + title: max_tokens + type: integer + "n": + default: 1 + description: | + How many completions to generate for each prompt. + + **Note:** Because this parameter generates many completions, it can quickly consume your token quota. Use carefully and ensure that you have reasonable settings for `max_tokens` and `stop`. + example: 1 + maximum: 128 + minimum: 1 + nullable: true + title: "n" + type: integer + presence_penalty: + default: 0 + description: | + Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. + + [See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details) + maximum: 2 + minimum: -2 + nullable: true + title: presence_penalty + type: number + seed: + description: | + If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same `seed` and parameters should return the same result. + + Determinism is not guaranteed, and you should refer to the `system_fingerprint` response parameter to monitor changes in the backend. + maximum: 9223372036854775807 + minimum: -9223372036854775808 + nullable: true + title: seed + type: integer + stop: + $ref: '#/components/schemas/CreateCompletionRequest_stop' + stream: + default: false + description: | + Whether to stream back partial progress. If set, tokens will be sent as data-only [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) as they become available, with the stream terminated by a `data: [DONE]` message. [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions). + nullable: true + title: stream + type: boolean + suffix: + description: | + The suffix that comes after a completion of inserted text. + + This parameter is only supported for `gpt-3.5-turbo-instruct`. + example: test. + nullable: true + title: suffix + type: string + temperature: + default: 1 + description: | + What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. + + We generally recommend altering this or `top_p` but not both. + example: 1 + maximum: 2 + minimum: 0 + nullable: true + title: temperature + type: number + top_p: + default: 1 + description: | + An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. + + We generally recommend altering this or `temperature` but not both. + example: 1 + maximum: 1 + minimum: 0 + nullable: true + title: top_p + type: number + user: + description: | + A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](/docs/guides/safety-best-practices/end-user-ids). + example: user-1234 + title: user + type: string + required: + - model + - prompt + title: CreateCompletionRequest + type: object + CreateCompletionResponse: + description: | + Represents a completion response from the API. Note: both the streamed and non-streamed response objects share the same shape (unlike the chat endpoint). + example: + created: 5 + usage: + completion_tokens: 7 + prompt_tokens: 9 + total_tokens: 3 + model: model + id: id + choices: + - finish_reason: stop + index: 0 + text: text + logprobs: + top_logprobs: + - key: 5.962133916683182 + - key: 5.962133916683182 + token_logprobs: + - 1.4658129805029452 + - 1.4658129805029452 + tokens: + - tokens + - tokens + text_offset: + - 6 + - 6 + - finish_reason: stop + index: 0 + text: text + logprobs: + top_logprobs: + - key: 5.962133916683182 + - key: 5.962133916683182 + token_logprobs: + - 1.4658129805029452 + - 1.4658129805029452 + tokens: + - tokens + - tokens + text_offset: + - 6 + - 6 + system_fingerprint: system_fingerprint + object: text_completion + properties: + id: + description: A unique identifier for the completion. + title: id + type: string + choices: + description: The list of completion choices the model generated for the + input prompt. + items: + $ref: '#/components/schemas/CreateCompletionResponse_choices_inner' + title: choices + type: array + created: + description: The Unix timestamp (in seconds) of when the completion was + created. + title: created + type: integer + model: + description: The model used for completion. + title: model + type: string + system_fingerprint: + description: | + This fingerprint represents the backend configuration that the model runs with. + + Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. + title: system_fingerprint + type: string + object: + description: "The object type, which is always \"text_completion\"" + enum: + - text_completion + title: object + type: string + usage: + $ref: '#/components/schemas/CompletionUsage' + required: + - choices + - created + - id + - model + - object + title: CreateCompletionResponse + type: object + x-oaiMeta: + name: The completion object + legacy: true + example: | + { + "id": "cmpl-uqkvlQyYK7bGYrRHQ0eXlWi7", + "object": "text_completion", + "created": 1589478378, + "model": "gpt-4-turbo", + "choices": [ + { + "text": "\n\nThis is indeed a test", + "index": 0, + "logprobs": null, + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 7, + "total_tokens": 12 + } + } + ChatCompletionRequestMessageContentPart: + oneOf: + - $ref: '#/components/schemas/ChatCompletionRequestMessageContentPartText' + - $ref: '#/components/schemas/ChatCompletionRequestMessageContentPartImage' + title: ChatCompletionRequestMessageContentPart + x-oaiExpandable: true + ChatCompletionRequestMessageContentPartImage: + properties: + type: + description: The type of the content part. + enum: + - image_url + title: type + type: string + image_url: + $ref: '#/components/schemas/ChatCompletionRequestMessageContentPartImage_image_url' + required: + - image_url + - type + title: Image content part + type: object + ChatCompletionRequestMessageContentPartText: + properties: + type: + description: The type of the content part. + enum: + - text + title: type + type: string + text: + description: The text content. + title: text + type: string + required: + - text + - type + title: Text content part + type: object + ChatCompletionRequestMessage: + oneOf: + - $ref: '#/components/schemas/ChatCompletionRequestSystemMessage' + - $ref: '#/components/schemas/ChatCompletionRequestUserMessage' + - $ref: '#/components/schemas/ChatCompletionRequestAssistantMessage' + - $ref: '#/components/schemas/ChatCompletionRequestToolMessage' + - $ref: '#/components/schemas/ChatCompletionRequestFunctionMessage' + title: ChatCompletionRequestMessage + x-oaiExpandable: true + ChatCompletionRequestSystemMessage: + example: + role: system + name: name + content: content + properties: + content: + description: The contents of the system message. + title: content + type: string + role: + description: "The role of the messages author, in this case `system`." + enum: + - system + title: role + type: string + name: + description: An optional name for the participant. Provides the model information + to differentiate between participants of the same role. + title: name + type: string + required: + - content + - role + title: System message + type: object + ChatCompletionRequestUserMessage: + properties: + content: + $ref: '#/components/schemas/ChatCompletionRequestUserMessage_content' + role: + description: "The role of the messages author, in this case `user`." + enum: + - user + title: role + type: string + name: + description: An optional name for the participant. Provides the model information + to differentiate between participants of the same role. + title: name + type: string + required: + - content + - role + title: User message + type: object + ChatCompletionRequestAssistantMessage: + properties: + content: + description: | + The contents of the assistant message. Required unless `tool_calls` or `function_call` is specified. + nullable: true + title: content + type: string + role: + description: "The role of the messages author, in this case `assistant`." + enum: + - assistant + title: role + type: string + name: + description: An optional name for the participant. Provides the model information + to differentiate between participants of the same role. + title: name + type: string + tool_calls: + description: "The tool calls generated by the model, such as function calls." + items: + $ref: '#/components/schemas/ChatCompletionMessageToolCall' + title: ChatCompletionMessageToolCalls + type: array + function_call: + $ref: '#/components/schemas/ChatCompletionRequestAssistantMessage_function_call' + required: + - role + title: Assistant message + type: object + ChatCompletionRequestToolMessage: + properties: + role: + description: "The role of the messages author, in this case `tool`." + enum: + - tool + title: role + type: string + content: + description: The contents of the tool message. + title: content + type: string + tool_call_id: + description: Tool call that this message is responding to. + title: tool_call_id + type: string + required: + - content + - role + - tool_call_id + title: Tool message + type: object + ChatCompletionRequestFunctionMessage: + deprecated: true + properties: + role: + description: "The role of the messages author, in this case `function`." + enum: + - function + title: role + type: string + content: + description: The contents of the function message. + nullable: true + title: content + type: string + name: + description: The name of the function to call. + title: name + type: string + required: + - content + - name + - role + title: Function message + type: object + FunctionParameters: + additionalProperties: true + description: "The parameters the functions accepts, described as a JSON Schema\ + \ object. See the [guide](/docs/guides/text-generation/function-calling) for\ + \ examples, and the [JSON Schema reference](https://json-schema.org/understanding-json-schema/)\ + \ for documentation about the format. \n\nOmitting `parameters` defines a\ + \ function with an empty parameter list." + title: FunctionParameters + type: object + ChatCompletionFunctions: + deprecated: true + example: + name: name + description: description + parameters: + key: "" + properties: + description: + description: "A description of what the function does, used by the model\ + \ to choose when and how to call the function." + title: description + type: string + name: + description: "The name of the function to be called. Must be a-z, A-Z, 0-9,\ + \ or contain underscores and dashes, with a maximum length of 64." + title: name + type: string + parameters: + additionalProperties: true + description: "The parameters the functions accepts, described as a JSON\ + \ Schema object. See the [guide](/docs/guides/text-generation/function-calling)\ + \ for examples, and the [JSON Schema reference](https://json-schema.org/understanding-json-schema/)\ + \ for documentation about the format. \n\nOmitting `parameters` defines\ + \ a function with an empty parameter list." + title: FunctionParameters + type: object + required: + - name + title: ChatCompletionFunctions + type: object + ChatCompletionFunctionCallOption: + description: | + Specifying a particular function via `{"name": "my_function"}` forces the model to call that function. + properties: + name: + description: The name of the function to call. + title: name + type: string + required: + - name + title: ChatCompletionFunctionCallOption + type: object + ChatCompletionTool: + example: + function: + name: name + description: description + parameters: + key: "" + type: function + properties: + type: + description: "The type of the tool. Currently, only `function` is supported." + enum: + - function + title: type + type: string + function: + $ref: '#/components/schemas/FunctionObject' + required: + - function + - type + title: ChatCompletionTool + type: object + FunctionObject: + example: + name: name + description: description + parameters: + key: "" + properties: + description: + description: "A description of what the function does, used by the model\ + \ to choose when and how to call the function." + title: description + type: string + name: + description: "The name of the function to be called. Must be a-z, A-Z, 0-9,\ + \ or contain underscores and dashes, with a maximum length of 64." + title: name + type: string + parameters: + additionalProperties: true + description: "The parameters the functions accepts, described as a JSON\ + \ Schema object. See the [guide](/docs/guides/text-generation/function-calling)\ + \ for examples, and the [JSON Schema reference](https://json-schema.org/understanding-json-schema/)\ + \ for documentation about the format. \n\nOmitting `parameters` defines\ + \ a function with an empty parameter list." + title: FunctionParameters + type: object + required: + - name + title: FunctionObject + type: object + ChatCompletionToolChoiceOption: + description: | + Controls which (if any) tool is called by the model. + `none` means the model will not call any tool and instead generates a message. + `auto` means the model can pick between generating a message or calling one or more tools. + `required` means the model must call one or more tools. + Specifying a particular tool via `{"type": "function", "function": {"name": "my_function"}}` forces the model to call that tool. + + `none` is the default when no tools are present. `auto` is the default if tools are present. + oneOf: + - description: | + `none` means the model will not call any tool and instead generates a message. `auto` means the model can pick between generating a message or calling one or more tools. `required` means the model must call one or more tools. + enum: + - none + - auto + - required + type: string + - $ref: '#/components/schemas/ChatCompletionNamedToolChoice' + title: ChatCompletionToolChoiceOption + x-oaiExpandable: true + ChatCompletionNamedToolChoice: + description: Specifies a tool the model should use. Use to force the model to + call a specific function. + properties: + type: + description: "The type of the tool. Currently, only `function` is supported." + enum: + - function + title: type + type: string + function: + $ref: '#/components/schemas/ChatCompletionNamedToolChoice_function' + required: + - function + - type + title: ChatCompletionNamedToolChoice + type: object + ChatCompletionMessageToolCalls: + description: "The tool calls generated by the model, such as function calls." + items: + $ref: '#/components/schemas/ChatCompletionMessageToolCall' + title: ChatCompletionMessageToolCalls + type: array + ChatCompletionMessageToolCall: + example: + function: + name: name + arguments: arguments + id: id + type: function + properties: + id: + description: The ID of the tool call. + title: id + type: string + type: + description: "The type of the tool. Currently, only `function` is supported." + enum: + - function + title: type + type: string + function: + $ref: '#/components/schemas/ChatCompletionMessageToolCall_function' + required: + - function + - id + - type + title: ChatCompletionMessageToolCall + type: object + ChatCompletionMessageToolCallChunk: + properties: + index: + title: index + type: integer + id: + description: The ID of the tool call. + title: id + type: string + type: + description: "The type of the tool. Currently, only `function` is supported." + enum: + - function + title: type + type: string + function: + $ref: '#/components/schemas/ChatCompletionMessageToolCallChunk_function' + required: + - index + title: ChatCompletionMessageToolCallChunk + type: object + ChatCompletionRole: + description: The role of the author of a message + enum: + - system + - user + - assistant + - tool + - function + type: string + ChatCompletionResponseMessage: + description: A chat completion message generated by the model. + example: + role: assistant + function_call: + name: name + arguments: arguments + tool_calls: + - function: + name: name + arguments: arguments + id: id + type: function + - function: + name: name + arguments: arguments + id: id + type: function + content: content + properties: + content: + description: The contents of the message. + nullable: true + title: content + type: string + tool_calls: + description: "The tool calls generated by the model, such as function calls." + items: + $ref: '#/components/schemas/ChatCompletionMessageToolCall' + title: ChatCompletionMessageToolCalls + type: array + role: + description: The role of the author of this message. + enum: + - assistant + title: role + type: string + function_call: + $ref: '#/components/schemas/ChatCompletionRequestAssistantMessage_function_call' + required: + - content + - role + title: ChatCompletionResponseMessage + type: object + ChatCompletionStreamResponseDelta: + description: A chat completion delta generated by streamed model responses. + properties: + content: + description: The contents of the chunk message. + nullable: true + title: content + type: string + function_call: + $ref: '#/components/schemas/ChatCompletionStreamResponseDelta_function_call' + tool_calls: + items: + $ref: '#/components/schemas/ChatCompletionMessageToolCallChunk' + title: tool_calls + type: array + role: + description: The role of the author of this message. + enum: + - system + - user + - assistant + - tool + title: role + type: string + title: ChatCompletionStreamResponseDelta + type: object + CreateChatCompletionRequest: + example: + top_logprobs: 2 + logit_bias: + key: 6 + seed: -2147483648 + functions: + - name: name + description: description + parameters: + key: "" + - name: name + description: description + parameters: + key: "" + - name: name + description: description + parameters: + key: "" + - name: name + description: description + parameters: + key: "" + - name: name + description: description + parameters: + key: "" + max_tokens: 5 + function_call: none + presence_penalty: 0.25495066265333133 + tools: + - function: + name: name + description: description + parameters: + key: "" + type: function + - function: + name: name + description: description + parameters: + key: "" + type: function + "n": 1 + logprobs: false + top_p: 1 + frequency_penalty: -1.6796687238155954 + response_format: + type: json_object + stop: CreateChatCompletionRequest_stop + stream: false + temperature: 1 + messages: + - role: system + name: name + content: content + - role: system + name: name + content: content + tool_choice: none + model: gpt-4-turbo + user: user-1234 + properties: + messages: + description: "A list of messages comprising the conversation so far. [Example\ + \ Python code](https://cookbook.openai.com/examples/how_to_format_inputs_to_chatgpt_models)." + items: + $ref: '#/components/schemas/ChatCompletionRequestMessage' + minItems: 1 + title: messages + type: array + model: + $ref: '#/components/schemas/CreateChatCompletionRequest_model' + frequency_penalty: + default: 0 + description: | + Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. + + [See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details) + maximum: 2 + minimum: -2 + nullable: true + title: frequency_penalty + type: number + logit_bias: + additionalProperties: + type: integer + description: | + Modify the likelihood of specified tokens appearing in the completion. + + Accepts a JSON object that maps tokens (specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically, the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should result in a ban or exclusive selection of the relevant token. + nullable: true + title: logit_bias + type: object + x-oaiTypeLabel: map + logprobs: + default: false + description: "Whether to return log probabilities of the output tokens or\ + \ not. If true, returns the log probabilities of each output token returned\ + \ in the `content` of `message`." + nullable: true + title: logprobs + type: boolean + top_logprobs: + description: "An integer between 0 and 20 specifying the number of most\ + \ likely tokens to return at each token position, each with an associated\ + \ log probability. `logprobs` must be set to `true` if this parameter\ + \ is used." + maximum: 20 + minimum: 0 + nullable: true + title: top_logprobs + type: integer + max_tokens: + description: | + The maximum number of [tokens](/tokenizer) that can be generated in the chat completion. + + The total length of input tokens and generated tokens is limited by the model's context length. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens. + nullable: true + title: max_tokens + type: integer + "n": + default: 1 + description: How many chat completion choices to generate for each input + message. Note that you will be charged based on the number of generated + tokens across all of the choices. Keep `n` as `1` to minimize costs. + example: 1 + maximum: 128 + minimum: 1 + nullable: true + title: "n" + type: integer + presence_penalty: + default: 0 + description: | + Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. + + [See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details) + maximum: 2 + minimum: -2 + nullable: true + title: presence_penalty + type: number + response_format: + $ref: '#/components/schemas/CreateChatCompletionRequest_response_format' + seed: + description: | + This feature is in Beta. + If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same `seed` and parameters should return the same result. + Determinism is not guaranteed, and you should refer to the `system_fingerprint` response parameter to monitor changes in the backend. + maximum: 9223372036854775807 + minimum: -9223372036854775808 + nullable: true + title: seed + type: integer + x-oaiMeta: + beta: true + stop: + $ref: '#/components/schemas/CreateChatCompletionRequest_stop' + stream: + default: false + description: | + If set, partial message deltas will be sent, like in ChatGPT. Tokens will be sent as data-only [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) as they become available, with the stream terminated by a `data: [DONE]` message. [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions). + nullable: true + title: stream + type: boolean + temperature: + default: 1 + description: | + What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. + + We generally recommend altering this or `top_p` but not both. + example: 1 + maximum: 2 + minimum: 0 + nullable: true + title: temperature + type: number + top_p: + default: 1 + description: | + An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. + + We generally recommend altering this or `temperature` but not both. + example: 1 + maximum: 1 + minimum: 0 + nullable: true + title: top_p + type: number + tools: + description: | + A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of functions the model may generate JSON inputs for. A max of 128 functions are supported. + items: + $ref: '#/components/schemas/ChatCompletionTool' + title: tools + type: array + tool_choice: + $ref: '#/components/schemas/ChatCompletionToolChoiceOption' + user: + description: | + A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](/docs/guides/safety-best-practices/end-user-ids). + example: user-1234 + title: user + type: string + function_call: + $ref: '#/components/schemas/CreateChatCompletionRequest_function_call' + functions: + deprecated: true + description: | + Deprecated in favor of `tools`. + + A list of functions the model may generate JSON inputs for. + items: + $ref: '#/components/schemas/ChatCompletionFunctions' + maxItems: 128 + minItems: 1 + title: functions + type: array + required: + - messages + - model + title: CreateChatCompletionRequest + type: object + CreateChatCompletionResponse: + description: "Represents a chat completion response returned by model, based\ + \ on the provided input." + example: + created: 2 + usage: + completion_tokens: 7 + prompt_tokens: 9 + total_tokens: 3 + model: model + id: id + choices: + - finish_reason: stop + index: 0 + message: + role: assistant + function_call: + name: name + arguments: arguments + tool_calls: + - function: + name: name + arguments: arguments + id: id + type: function + - function: + name: name + arguments: arguments + id: id + type: function + content: content + logprobs: + content: + - top_logprobs: + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + logprob: 6.027456183070403 + bytes: + - 1 + - 1 + token: token + - top_logprobs: + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + logprob: 6.027456183070403 + bytes: + - 1 + - 1 + token: token + - finish_reason: stop + index: 0 + message: + role: assistant + function_call: + name: name + arguments: arguments + tool_calls: + - function: + name: name + arguments: arguments + id: id + type: function + - function: + name: name + arguments: arguments + id: id + type: function + content: content + logprobs: + content: + - top_logprobs: + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + logprob: 6.027456183070403 + bytes: + - 1 + - 1 + token: token + - top_logprobs: + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + logprob: 6.027456183070403 + bytes: + - 1 + - 1 + token: token + system_fingerprint: system_fingerprint + object: chat.completion + properties: + id: + description: A unique identifier for the chat completion. + title: id + type: string + choices: + description: A list of chat completion choices. Can be more than one if + `n` is greater than 1. + items: + $ref: '#/components/schemas/CreateChatCompletionResponse_choices_inner' + title: choices + type: array + created: + description: The Unix timestamp (in seconds) of when the chat completion + was created. + title: created + type: integer + model: + description: The model used for the chat completion. + title: model + type: string + system_fingerprint: + description: | + This fingerprint represents the backend configuration that the model runs with. + + Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. + title: system_fingerprint + type: string + object: + description: "The object type, which is always `chat.completion`." + enum: + - chat.completion + title: object + type: string + usage: + $ref: '#/components/schemas/CompletionUsage' + required: + - choices + - created + - id + - model + - object + title: CreateChatCompletionResponse + type: object + x-oaiMeta: + name: The chat completion object + group: chat + example: | + { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-3.5-turbo-0125", + "system_fingerprint": "fp_44709d6fcb", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "\n\nHello there, how may I assist you today?", + }, + "logprobs": null, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } + } + CreateChatCompletionFunctionResponse: + description: "Represents a chat completion response returned by model, based\ + \ on the provided input." + properties: + id: + description: A unique identifier for the chat completion. + type: string + choices: + description: A list of chat completion choices. Can be more than one if + `n` is greater than 1. + items: + $ref: '#/components/schemas/CreateChatCompletionFunctionResponse_choices_inner' + type: array + created: + description: The Unix timestamp (in seconds) of when the chat completion + was created. + type: integer + model: + description: The model used for the chat completion. + type: string + system_fingerprint: + description: | + This fingerprint represents the backend configuration that the model runs with. + + Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. + type: string + object: + description: "The object type, which is always `chat.completion`." + enum: + - chat.completion + type: string + usage: + $ref: '#/components/schemas/CompletionUsage' + required: + - choices + - created + - id + - model + - object + type: object + x-oaiMeta: + name: The chat completion object + group: chat + example: | + { + "id": "chatcmpl-abc123", + "object": "chat.completion", + "created": 1699896916, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\n\"location\": \"Boston, MA\"\n}" + } + } + ] + }, + "logprobs": null, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 82, + "completion_tokens": 17, + "total_tokens": 99 + } + } + ChatCompletionTokenLogprob: + example: + top_logprobs: + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + logprob: 6.027456183070403 + bytes: + - 1 + - 1 + token: token + properties: + token: + description: The token. + title: token + type: string + logprob: + description: "The log probability of this token, if it is within the top\ + \ 20 most likely tokens. Otherwise, the value `-9999.0` is used to signify\ + \ that the token is very unlikely." + title: logprob + type: number + bytes: + description: A list of integers representing the UTF-8 bytes representation + of the token. Useful in instances where characters are represented by + multiple tokens and their byte representations must be combined to generate + the correct text representation. Can be `null` if there is no bytes representation + for the token. + items: + type: integer + nullable: true + title: bytes + type: array + top_logprobs: + description: "List of the most likely tokens and their log probability,\ + \ at this token position. In rare cases, there may be fewer than the number\ + \ of requested `top_logprobs` returned." + items: + $ref: '#/components/schemas/ChatCompletionTokenLogprob_top_logprobs_inner' + title: top_logprobs + type: array + required: + - bytes + - logprob + - token + - top_logprobs + title: ChatCompletionTokenLogprob + type: object + CreateChatCompletionStreamResponse: + description: "Represents a streamed chunk of a chat completion response returned\ + \ by model, based on the provided input." + properties: + id: + description: A unique identifier for the chat completion. Each chunk has + the same ID. + type: string + choices: + description: A list of chat completion choices. Can be more than one if + `n` is greater than 1. + items: + $ref: '#/components/schemas/CreateChatCompletionStreamResponse_choices_inner' + type: array + created: + description: The Unix timestamp (in seconds) of when the chat completion + was created. Each chunk has the same timestamp. + type: integer + model: + description: The model to generate the completion. + type: string + system_fingerprint: + description: | + This fingerprint represents the backend configuration that the model runs with. + Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. + type: string + object: + description: "The object type, which is always `chat.completion.chunk`." + enum: + - chat.completion.chunk + type: string + required: + - choices + - created + - id + - model + - object + type: object + x-oaiMeta: + name: The chat completion chunk object + group: chat + example: | + {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0125", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]} + + {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0125", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}]} + + .... + + {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0125", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}]} + CreateChatCompletionImageResponse: + description: "Represents a streamed chunk of a chat completion response returned\ + \ by model, based on the provided input." + type: object + x-oaiMeta: + name: The chat completion chunk object + group: chat + example: | + { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-3.5-turbo-0125", + "system_fingerprint": "fp_44709d6fcb", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "\n\nThis image shows a wooden boardwalk extending through a lush green marshland.", + }, + "logprobs": null, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } + } + Model: + description: Describes an OpenAI model offering that can be used with the API. + example: + created: 0 + owned_by: owned_by + id: id + object: model + properties: + id: + description: "The model identifier, which can be referenced in the API endpoints." + title: id + type: string + created: + description: The Unix timestamp (in seconds) when the model was created. + title: created + type: integer + object: + description: "The object type, which is always \"model\"." + enum: + - model + title: object + type: string + owned_by: + description: The organization that owns the model. + title: owned_by + type: string + required: + - created + - id + - object + - owned_by + title: Model + x-oaiMeta: + name: The model object + example: | + { + "id": "VAR_model_id", + "object": "model", + "created": 1686935002, + "owned_by": "openai" + } + CompletionUsage: + description: Usage statistics for the completion request. + example: + completion_tokens: 7 + prompt_tokens: 9 + total_tokens: 3 + properties: + completion_tokens: + description: Number of tokens in the generated completion. + title: completion_tokens + type: integer + prompt_tokens: + description: Number of tokens in the prompt. + title: prompt_tokens + type: integer + total_tokens: + description: Total number of tokens used in the request (prompt + completion). + title: total_tokens + type: integer + required: + - completion_tokens + - prompt_tokens + - total_tokens + title: CompletionUsage + type: object + ErrorEvent: + description: "Occurs when an [error](/docs/guides/error-codes/api-errors) occurs.\ + \ This can happen due to an internal server error or a timeout." + properties: + event: + enum: + - error + type: string + data: + $ref: '#/components/schemas/Error' + required: + - data + - event + type: object + x-oaiMeta: + dataDescription: "`data` is an [error](/docs/guides/error-codes/api-errors)" + DoneEvent: + description: Occurs when a stream ends. + properties: + event: + enum: + - done + type: string + data: + enum: + - "[DONE]" + type: string + required: + - data + - event + type: object + x-oaiMeta: + dataDescription: "`data` is `[DONE]`" + CreateCompletionRequest_model: + anyOf: + - type: string + - enum: + - gpt-3.5-turbo-instruct + - davinci-002 + - babbage-002 + type: string + description: | + ID of the model to use. You can use the [List models](/docs/api-reference/models/list) API to see all of your available models, or see our [Model overview](/docs/models/overview) for descriptions of them. + title: CreateCompletionRequest_model + x-oaiTypeLabel: string + CreateCompletionRequest_prompt: + default: <|endoftext|> + description: | + The prompt(s) to generate completions for, encoded as a string, array of strings, array of tokens, or array of token arrays. + + Note that <|endoftext|> is the document separator that the model sees during training, so if a prompt is not specified the model will generate as if from the beginning of a new document. + nullable: true + oneOf: + - default: "" + example: This is a test. + type: string + - items: + default: "" + example: This is a test. + type: string + type: array + - example: "[1212, 318, 257, 1332, 13]" + items: + type: integer + minItems: 1 + type: array + - example: "[[1212, 318, 257, 1332, 13]]" + items: + items: + type: integer + minItems: 1 + type: array + minItems: 1 + type: array + title: CreateCompletionRequest_prompt + CreateCompletionRequest_stop: + default: null + description: | + Up to 4 sequences where the API will stop generating further tokens. The returned text will not contain the stop sequence. + nullable: true + oneOf: + - default: <|endoftext|> + example: |2+ + + nullable: true + type: string + - items: + example: "[\"\\n\"]" + type: string + maxItems: 4 + minItems: 1 + type: array + title: CreateCompletionRequest_stop + CreateCompletionResponse_choices_inner_logprobs: + example: + top_logprobs: + - key: 5.962133916683182 + - key: 5.962133916683182 + token_logprobs: + - 1.4658129805029452 + - 1.4658129805029452 + tokens: + - tokens + - tokens + text_offset: + - 6 + - 6 + nullable: true + properties: + text_offset: + items: + type: integer + title: text_offset + type: array + token_logprobs: + items: + type: number + title: token_logprobs + type: array + tokens: + items: + type: string + title: tokens + type: array + top_logprobs: + items: + additionalProperties: + type: number + type: object + title: top_logprobs + type: array + title: CreateCompletionResponse_choices_inner_logprobs + type: object + CreateCompletionResponse_choices_inner: + example: + finish_reason: stop + index: 0 + text: text + logprobs: + top_logprobs: + - key: 5.962133916683182 + - key: 5.962133916683182 + token_logprobs: + - 1.4658129805029452 + - 1.4658129805029452 + tokens: + - tokens + - tokens + text_offset: + - 6 + - 6 + properties: + finish_reason: + description: | + The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, + `length` if the maximum number of tokens specified in the request was reached, + or `content_filter` if content was omitted due to a flag from our content filters. + enum: + - stop + - length + - content_filter + title: finish_reason + type: string + index: + title: index + type: integer + logprobs: + $ref: '#/components/schemas/CreateCompletionResponse_choices_inner_logprobs' + text: + title: text + type: string + required: + - finish_reason + - index + - logprobs + - text + title: CreateCompletionResponse_choices_inner + type: object + ChatCompletionRequestMessageContentPartImage_image_url: + properties: + url: + description: Either a URL of the image or the base64 encoded image data. + format: uri + title: url + type: string + detail: + default: auto + description: "Specifies the detail level of the image. Learn more in the\ + \ [Vision guide](/docs/guides/vision/low-or-high-fidelity-image-understanding)." + enum: + - auto + - low + - high + title: detail + type: string + required: + - url + title: ChatCompletionRequestMessageContentPartImage_image_url + type: object + ChatCompletionRequestUserMessage_content: + description: | + The contents of the user message. + oneOf: + - description: The text contents of the message. + title: Text content + type: string + - description: "An array of content parts with a defined type, each can be of\ + \ type `text` or `image_url` when passing in images. You can pass multiple\ + \ images by adding multiple `image_url` content parts. Image input is only\ + \ supported when using the `gpt-4-visual-preview` model." + items: + $ref: '#/components/schemas/ChatCompletionRequestMessageContentPart' + minItems: 1 + title: Array of content parts + type: array + title: ChatCompletionRequestUserMessage_content + x-oaiExpandable: true + ChatCompletionRequestAssistantMessage_function_call: + deprecated: true + description: "Deprecated and replaced by `tool_calls`. The name and arguments\ + \ of a function that should be called, as generated by the model." + example: + name: name + arguments: arguments + properties: + arguments: + description: "The arguments to call the function with, as generated by the\ + \ model in JSON format. Note that the model does not always generate valid\ + \ JSON, and may hallucinate parameters not defined by your function schema.\ + \ Validate the arguments in your code before calling your function." + title: arguments + type: string + name: + description: The name of the function to call. + title: name + type: string + required: + - arguments + - name + title: ChatCompletionRequestAssistantMessage_function_call + type: object + ChatCompletionNamedToolChoice_function: + properties: + name: + description: The name of the function to call. + title: name + type: string + required: + - name + title: ChatCompletionNamedToolChoice_function + type: object + ChatCompletionMessageToolCall_function: + description: The function that the model called. + example: + name: name + arguments: arguments + properties: + name: + description: The name of the function to call. + title: name + type: string + arguments: + description: "The arguments to call the function with, as generated by the\ + \ model in JSON format. Note that the model does not always generate valid\ + \ JSON, and may hallucinate parameters not defined by your function schema.\ + \ Validate the arguments in your code before calling your function." + title: arguments + type: string + required: + - arguments + - name + title: ChatCompletionMessageToolCall_function + type: object + ChatCompletionMessageToolCallChunk_function: + properties: + name: + description: The name of the function to call. + title: name + type: string + arguments: + description: "The arguments to call the function with, as generated by the\ + \ model in JSON format. Note that the model does not always generate valid\ + \ JSON, and may hallucinate parameters not defined by your function schema.\ + \ Validate the arguments in your code before calling your function." + title: arguments + type: string + title: ChatCompletionMessageToolCallChunk_function + type: object + ChatCompletionStreamResponseDelta_function_call: + deprecated: true + description: "Deprecated and replaced by `tool_calls`. The name and arguments\ + \ of a function that should be called, as generated by the model." + properties: + arguments: + description: "The arguments to call the function with, as generated by the\ + \ model in JSON format. Note that the model does not always generate valid\ + \ JSON, and may hallucinate parameters not defined by your function schema.\ + \ Validate the arguments in your code before calling your function." + title: arguments + type: string + name: + description: The name of the function to call. + title: name + type: string + title: ChatCompletionStreamResponseDelta_function_call + type: object + CreateChatCompletionRequest_model: + anyOf: + - type: string + - enum: + - gpt-4-turbo + - gpt-4-turbo-2024-04-09 + - gpt-4-0125-preview + - gpt-4-turbo-preview + - gpt-4-1106-preview + - gpt-4-vision-preview + - gpt-4 + - gpt-4-0314 + - gpt-4-0613 + - gpt-4-32k + - gpt-4-32k-0314 + - gpt-4-32k-0613 + - gpt-3.5-turbo + - gpt-3.5-turbo-16k + - gpt-3.5-turbo-0301 + - gpt-3.5-turbo-0613 + - gpt-3.5-turbo-1106 + - gpt-3.5-turbo-0125 + - gpt-3.5-turbo-16k-0613 + type: string + description: "ID of the model to use. See the [model endpoint compatibility](/docs/models/model-endpoint-compatibility)\ + \ table for details on which models work with the Chat API." + example: gpt-4-turbo + title: CreateChatCompletionRequest_model + x-oaiTypeLabel: string + CreateChatCompletionRequest_response_format: + description: | + An object specifying the format that the model must output. Compatible with [GPT-4 Turbo](/docs/models/gpt-4-and-gpt-4-turbo) and all GPT-3.5 Turbo models newer than `gpt-3.5-turbo-1106`. + + Setting to `{ "type": "json_object" }` enables JSON mode, which guarantees the message the model generates is valid JSON. + + **Important:** when using JSON mode, you **must** also instruct the model to produce JSON yourself via a system or user message. Without this, the model may generate an unending stream of whitespace until the generation reaches the token limit, resulting in a long-running and seemingly "stuck" request. Also note that the message content may be partially cut off if `finish_reason="length"`, which indicates the generation exceeded `max_tokens` or the conversation exceeded the max context length. + example: + type: json_object + properties: + type: + default: text + description: Must be one of `text` or `json_object`. + enum: + - text + - json_object + example: json_object + title: type + type: string + title: CreateChatCompletionRequest_response_format + type: object + CreateChatCompletionRequest_stop: + default: null + description: | + Up to 4 sequences where the API will stop generating further tokens. + oneOf: + - nullable: true + type: string + - items: + type: string + maxItems: 4 + minItems: 1 + type: array + title: CreateChatCompletionRequest_stop + CreateChatCompletionRequest_function_call: + deprecated: true + description: | + Deprecated in favor of `tool_choice`. + + Controls which (if any) function is called by the model. + `none` means the model will not call a function and instead generates a message. + `auto` means the model can pick between generating a message or calling a function. + Specifying a particular function via `{"name": "my_function"}` forces the model to call that function. + + `none` is the default when no functions are present. `auto` is the default if functions are present. + oneOf: + - description: | + `none` means the model will not call a function and instead generates a message. `auto` means the model can pick between generating a message or calling a function. + enum: + - none + - auto + type: string + - $ref: '#/components/schemas/ChatCompletionFunctionCallOption' + title: CreateChatCompletionRequest_function_call + x-oaiExpandable: true + CreateChatCompletionResponse_choices_inner_logprobs: + description: Log probability information for the choice. + example: + content: + - top_logprobs: + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + logprob: 6.027456183070403 + bytes: + - 1 + - 1 + token: token + - top_logprobs: + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + logprob: 6.027456183070403 + bytes: + - 1 + - 1 + token: token + nullable: true + properties: + content: + description: A list of message content tokens with log probability information. + items: + $ref: '#/components/schemas/ChatCompletionTokenLogprob' + nullable: true + title: content + type: array + required: + - content + title: CreateChatCompletionResponse_choices_inner_logprobs + type: object + CreateChatCompletionResponse_choices_inner: + example: + finish_reason: stop + index: 0 + message: + role: assistant + function_call: + name: name + arguments: arguments + tool_calls: + - function: + name: name + arguments: arguments + id: id + type: function + - function: + name: name + arguments: arguments + id: id + type: function + content: content + logprobs: + content: + - top_logprobs: + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + logprob: 6.027456183070403 + bytes: + - 1 + - 1 + token: token + - top_logprobs: + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + - logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + logprob: 6.027456183070403 + bytes: + - 1 + - 1 + token: token + properties: + finish_reason: + description: | + The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, + `length` if the maximum number of tokens specified in the request was reached, + `content_filter` if content was omitted due to a flag from our content filters, + `tool_calls` if the model called a tool, or `function_call` (deprecated) if the model called a function. + enum: + - stop + - length + - tool_calls + - content_filter + - function_call + title: finish_reason + type: string + index: + description: The index of the choice in the list of choices. + title: index + type: integer + message: + $ref: '#/components/schemas/ChatCompletionResponseMessage' + logprobs: + $ref: '#/components/schemas/CreateChatCompletionResponse_choices_inner_logprobs' + required: + - finish_reason + - index + - logprobs + - message + title: CreateChatCompletionResponse_choices_inner + type: object + CreateChatCompletionFunctionResponse_choices_inner: + properties: + finish_reason: + description: | + The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, `length` if the maximum number of tokens specified in the request was reached, `content_filter` if content was omitted due to a flag from our content filters, or `function_call` if the model called a function. + enum: + - stop + - length + - function_call + - content_filter + title: finish_reason + type: string + index: + description: The index of the choice in the list of choices. + title: index + type: integer + message: + $ref: '#/components/schemas/ChatCompletionResponseMessage' + required: + - finish_reason + - index + - logprobs + - message + title: CreateChatCompletionFunctionResponse_choices_inner + type: object + ChatCompletionTokenLogprob_top_logprobs_inner: + example: + logprob: 5.962133916683182 + bytes: + - 5 + - 5 + token: token + properties: + token: + description: The token. + title: token + type: string + logprob: + description: "The log probability of this token, if it is within the top\ + \ 20 most likely tokens. Otherwise, the value `-9999.0` is used to signify\ + \ that the token is very unlikely." + title: logprob + type: number + bytes: + description: A list of integers representing the UTF-8 bytes representation + of the token. Useful in instances where characters are represented by + multiple tokens and their byte representations must be combined to generate + the correct text representation. Can be `null` if there is no bytes representation + for the token. + items: + type: integer + nullable: true + title: bytes + type: array + required: + - bytes + - logprob + - token + title: ChatCompletionTokenLogprob_top_logprobs_inner + type: object + CreateChatCompletionStreamResponse_choices_inner: + properties: + delta: + $ref: '#/components/schemas/ChatCompletionStreamResponseDelta' + logprobs: + $ref: '#/components/schemas/CreateChatCompletionResponse_choices_inner_logprobs' + finish_reason: + description: | + The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, + `length` if the maximum number of tokens specified in the request was reached, + `content_filter` if content was omitted due to a flag from our content filters, + `tool_calls` if the model called a tool, or `function_call` (deprecated) if the model called a function. + enum: + - stop + - length + - tool_calls + - content_filter + - function_call + nullable: true + title: finish_reason + type: string + index: + description: The index of the choice in the list of choices. + title: index + type: integer + required: + - delta + - finish_reason + - index + title: CreateChatCompletionStreamResponse_choices_inner + type: object + securitySchemes: + ApiKeyAuth: + scheme: bearer + type: http +x-oaiMeta: + navigationGroups: + - id: endpoints + title: Endpoints + - id: legacy + title: Legacy + groups: + - id: chat + title: Chat + description: | + Given a list of messages comprising a conversation, the model will return a response. + + Related guide: [Chat Completions](/docs/guides/text-generation) + navigationGroup: endpoints + sections: + - type: endpoint + key: createChatCompletion + path: create + - type: object + key: CreateChatCompletionResponse + path: object + - type: object + key: CreateChatCompletionStreamResponse + path: streaming + - id: models + title: Models + description: | + List and describe the various models available in the API. You can refer to the [Models](/docs/models) documentation to understand what models are available and the differences between them. + navigationGroup: endpoints + sections: + - type: endpoint + key: listModels + path: list + - type: endpoint + key: retrieveModel + path: retrieve + - type: endpoint + key: deleteModel + path: delete + - type: object + key: Model + path: object + - id: completions + title: Completions + legacy: true + navigationGroup: legacy + description: | + Given a prompt, the model will return one or more predicted completions along with the probabilities of alternative tokens at each position. Most developer should use our [Chat Completions API](/docs/guides/text-generation/text-generation-models) to leverage our best and newest models. + sections: + - type: endpoint + key: createCompletion + path: create + - type: object + key: CreateCompletionResponse + path: object diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/pyproject.toml b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/pyproject.toml new file mode 100644 index 00000000..c2826df3 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 88 +exclude = ''' +( + /( + \.eggs # exclude a few common directories in the + | \.git # root of the project + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist + )/ +) +''' + +[tool.isort] +profile = "black" +skip = [ + '.eggs', '.git', '.hg', '.mypy_cache', '.nox', '.pants.d', '.tox', + '.venv', '_build', 'buck-out', 'build', 'dist', 'node_modules', 'venv', +] +skip_gitignore = true diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/requirements.txt b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/requirements.txt new file mode 100644 index 00000000..a22fe854 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/requirements.txt @@ -0,0 +1,36 @@ +aiofiles==23.1.0 +aniso8601==7.0.0 +async-exit-stack==1.0.1 +async-generator==1.10 +certifi==2023.7.22 +chardet==4.0.0 +click==7.1.2 +dnspython==2.6.1 +email-validator==2.0.0 +fastapi==0.109.2 +graphene==2.1.8 +graphql-core==2.3.2 +graphql-relay==2.0.1 +h11==0.12.0 +httptools==0.1.2 +httpx==0.24.1 +idna==3.7 +itsdangerous==1.1.0 +Jinja2==2.11.3 +MarkupSafe==2.0.1 +orjson==3.9.15 +promise==2.3 +pydantic>=2 +python-dotenv==0.17.1 +python-multipart==0.0.7 +PyYAML +requests==2.31.0 +Rx==1.6.1 +starlette==0.36.3 +typing-extensions==4.8.0 +ujson==4.0.2 +urllib3==1.26.18 +uvicorn==0.13.4 +uvloop==0.19.0 +watchgod==0.7 +websockets==10.0 diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/setup.cfg b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/setup.cfg new file mode 100644 index 00000000..e31d2905 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/setup.cfg @@ -0,0 +1,20 @@ +[metadata] +name = openapi_server +version = 2.0.0 +description = The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. +long_description = file: README.md +keywords = OpenAPI OpenAI API +python_requires = >= 3.7.* +classifiers = + Operating System :: OS Independent + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.7 + +[options] +install_requires = fastapi[all] +setup_requires = setuptools +package_dir = =src +packages = find_namespace: + +[options.packages.find] +where = src diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/__init__.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/chat_api.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/chat_api.py new file mode 100644 index 00000000..013dc1b3 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/chat_api.py @@ -0,0 +1,54 @@ +# coding: utf-8 + +import importlib +import pkgutil +from typing import Dict, List # noqa: F401 + +from fastapi import ( # noqa: F401 + APIRouter, + Body, + Cookie, + Depends, + Form, + Header, + Path, + Query, + Response, + Security, + status, +) + +import openapi_server.impl +from openapi_server.apis.chat_api_base import BaseChatApi +from openapi_server.models.create_chat_completion_request import ( + CreateChatCompletionRequest, +) +from openapi_server.models.create_chat_completion_response import ( + CreateChatCompletionResponse, +) +from openapi_server.models.extra_models import TokenModel # noqa: F401 +from openapi_server.security_api import get_token_ApiKeyAuth + +router = APIRouter() + +ns_pkg = openapi_server.impl +for _, name, _ in pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + "."): + importlib.import_module(name) + + +@router.post( + "/chat/completions", + responses={ + 200: {"model": CreateChatCompletionResponse, "description": "OK"}, + }, + tags=["Chat"], + summary="Creates a model response for the given chat conversation.", + response_model_by_alias=True, +) +async def create_chat_completion( + create_chat_completion_request: CreateChatCompletionRequest = Body( + None, description="" + ), + token_ApiKeyAuth: TokenModel = Security(get_token_ApiKeyAuth), +) -> CreateChatCompletionResponse: + ... diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/chat_api_base.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/chat_api_base.py new file mode 100644 index 00000000..da24a3f1 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/chat_api_base.py @@ -0,0 +1,25 @@ +# coding: utf-8 + +from typing import ClassVar, Dict, List, Tuple # noqa: F401 + +from openapi_server.models.create_chat_completion_request import ( + CreateChatCompletionRequest, +) +from openapi_server.models.create_chat_completion_response import ( + CreateChatCompletionResponse, +) +from openapi_server.security_api import get_token_ApiKeyAuth + + +class BaseChatApi: + subclasses: ClassVar[Tuple] = () + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + BaseChatApi.subclasses = BaseChatApi.subclasses + (cls,) + + def create_chat_completion( + self, + create_chat_completion_request: CreateChatCompletionRequest, + ) -> CreateChatCompletionResponse: + ... diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/completions_api.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/completions_api.py new file mode 100644 index 00000000..39c18780 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/completions_api.py @@ -0,0 +1,101 @@ +# coding: utf-8 + +import copy +import importlib +import pkgutil +import time +import uuid +from typing import Dict, List # noqa: F401 + +import tritonserver + +import openapi_server.impl +from openapi_server.apis.completions_api_base import BaseCompletionsApi + +triton_server = tritonserver.Server( + model_repository="/workspace/llm-models", log_verbose=6, strict_model_config=False +).start() + + +from fastapi import ( # noqa: F401; Security, + APIRouter, + Body, + Cookie, + Depends, + Form, + Header, + Path, + Query, + Response, + status, +) + +from openapi_server.models.create_completion_request import CreateCompletionRequest +from openapi_server.models.create_completion_response import CreateCompletionResponse +from openapi_server.models.create_completion_response_choices_inner import ( + CreateCompletionResponseChoicesInner, +) +from openapi_server.models.create_completion_response_choices_inner_logprobs import ( + CreateCompletionResponseChoicesInnerLogprobs, +) +from openapi_server.models.extra_models import TokenModel # noqa: F401 + +# from openapi_server.security_api import get_token_ApiKeyAuth + +router = APIRouter() + +ns_pkg = openapi_server.impl +for _, name, _ in pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + "."): + importlib.import_module(name) + + +@router.post( + "/completions", + responses={ + 200: {"model": CreateCompletionResponse, "description": "OK"}, + }, + tags=["Completions"], + summary="Creates a completion for the provided prompt and parameters.", + response_model_by_alias=True, +) +async def create_completion( + create_completion_request: CreateCompletionRequest = Body(None, description=""), +) -> CreateCompletionResponse: + exclude_input_in_output = True + + if create_completion_request.echo: + exclude_input_in_output = False + + model = triton_server.model("llama-3-8b-instruct") + parameters = copy.deepcopy(create_completion_request.dict()) + del parameters["prompt"] + del parameters["stream"] + del parameters["echo"] + del parameters["model"] + + response = list( + model.infer( + inputs={ + "text_input": [create_completion_request.prompt], + "stream": [False], + "exclude_input_in_output": [exclude_input_in_output], + }, + parameters=parameters, + ) + )[0] + + choice = CreateCompletionResponseChoicesInner( + finish_reason="stop", + index=0, + text=response.outputs["text_output"].to_string_array()[0], + logprobs=CreateCompletionResponseChoicesInnerLogprobs(), + ) + + return CreateCompletionResponse( + id=f"cmpl-{uuid.uuid1()}", + created=int(time.time()), + model=model.name, + choices=[choice], + system_fingerprint=None, + object="text_completion", + ) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/completions_api_base.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/completions_api_base.py new file mode 100644 index 00000000..5e2826d7 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/completions_api_base.py @@ -0,0 +1,22 @@ +# coding: utf-8 + +from typing import ClassVar, Dict, List, Tuple # noqa: F401 + +from openapi_server.models.create_completion_request import CreateCompletionRequest +from openapi_server.models.create_completion_response import CreateCompletionResponse + +# from openapi_server.security_api import get_token_ApiKeyAuth + + +class BaseCompletionsApi: + subclasses: ClassVar[Tuple] = () + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + BaseCompletionsApi.subclasses = BaseCompletionsApi.subclasses + (cls,) + + def create_completion( + self, + create_completion_request: CreateCompletionRequest, + ) -> CreateCompletionResponse: + ... diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/models_api.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/models_api.py new file mode 100644 index 00000000..045ce15b --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/models_api.py @@ -0,0 +1,80 @@ +# coding: utf-8 + +import importlib +import pkgutil +from typing import Dict, List # noqa: F401 + +from fastapi import ( # noqa: F401 + APIRouter, + Body, + Cookie, + Depends, + Form, + Header, + Path, + Query, + Response, + Security, + status, +) + +import openapi_server.impl +from openapi_server.apis.models_api_base import BaseModelsApi +from openapi_server.models.delete_model_response import DeleteModelResponse +from openapi_server.models.extra_models import TokenModel # noqa: F401 +from openapi_server.models.list_models_response import ListModelsResponse +from openapi_server.models.model import Model +from openapi_server.security_api import get_token_ApiKeyAuth + +router = APIRouter() + +ns_pkg = openapi_server.impl +for _, name, _ in pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + "."): + importlib.import_module(name) + + +@router.delete( + "/models/{model}", + responses={ + 200: {"model": DeleteModelResponse, "description": "OK"}, + }, + tags=["Models"], + summary="Delete a fine-tuned model. You must have the Owner role in your organization to delete a model.", + response_model_by_alias=True, +) +async def delete_model( + model: str = Path(..., description="The model to delete"), + token_ApiKeyAuth: TokenModel = Security(get_token_ApiKeyAuth), +) -> DeleteModelResponse: + ... + + +@router.get( + "/models", + responses={ + 200: {"model": ListModelsResponse, "description": "OK"}, + }, + tags=["Models"], + summary="Lists the currently available models, and provides basic information about each one such as the owner and availability.", + response_model_by_alias=True, +) +async def list_models( + token_ApiKeyAuth: TokenModel = Security(get_token_ApiKeyAuth), +) -> ListModelsResponse: + ... + + +@router.get( + "/models/{model}", + responses={ + 200: {"model": Model, "description": "OK"}, + }, + tags=["Models"], + summary="Retrieves a model instance, providing basic information about the model such as the owner and permissioning.", + response_model_by_alias=True, +) +async def retrieve_model( + model: str = Path(..., description="The ID of the model to use for this request"), + token_ApiKeyAuth: TokenModel = Security(get_token_ApiKeyAuth), +) -> Model: + ... diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/models_api_base.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/models_api_base.py new file mode 100644 index 00000000..a1093a64 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/models_api_base.py @@ -0,0 +1,33 @@ +# coding: utf-8 + +from typing import ClassVar, Dict, List, Tuple # noqa: F401 + +from openapi_server.models.delete_model_response import DeleteModelResponse +from openapi_server.models.list_models_response import ListModelsResponse +from openapi_server.models.model import Model +from openapi_server.security_api import get_token_ApiKeyAuth + + +class BaseModelsApi: + subclasses: ClassVar[Tuple] = () + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + BaseModelsApi.subclasses = BaseModelsApi.subclasses + (cls,) + + def delete_model( + self, + model: str, + ) -> DeleteModelResponse: + ... + + def list_models( + self, + ) -> ListModelsResponse: + ... + + def retrieve_model( + self, + model: str, + ) -> Model: + ... diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/impl/__init__.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/impl/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/main.py new file mode 100644 index 00000000..92404d9d --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/main.py @@ -0,0 +1,29 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from fastapi import FastAPI + +from openapi_server.apis.chat_api import router as ChatApiRouter +from openapi_server.apis.completions_api import router as CompletionsApiRouter +from openapi_server.apis.models_api import router as ModelsApiRouter + +app = FastAPI( + title="OpenAI API", + description="The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details.", + version="2.0.0", +) + +app.include_router(ChatApiRouter) +app.include_router(CompletionsApiRouter) +app.include_router(ModelsApiRouter) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/__init__.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_function_call_option.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_function_call_option.py new file mode 100644 index 00000000..055ddd2f --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_function_call_option.py @@ -0,0 +1,85 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, Field, StrictStr + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionFunctionCallOption(BaseModel): + """ + Specifying a particular function via `{\"name\": \"my_function\"}` forces the model to call that function. + """ # noqa: E501 + + name: StrictStr = Field(description="The name of the function to call.") + __properties: ClassVar[List[str]] = ["name"] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionFunctionCallOption from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionFunctionCallOption from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({"name": obj.get("name")}) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_functions.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_functions.py new file mode 100644 index 00000000..53c07885 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_functions.py @@ -0,0 +1,101 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictStr + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionFunctions(BaseModel): + """ + ChatCompletionFunctions + """ # noqa: E501 + + description: Optional[StrictStr] = Field( + default=None, + description="A description of what the function does, used by the model to choose when and how to call the function.", + ) + name: StrictStr = Field( + description="The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64." + ) + parameters: Optional[Dict[str, Any]] = Field( + default=None, + description="The parameters the functions accepts, described as a JSON Schema object. See the [guide](/docs/guides/text-generation/function-calling) for examples, and the [JSON Schema reference](https://json-schema.org/understanding-json-schema/) for documentation about the format. Omitting `parameters` defines a function with an empty parameter list.", + ) + __properties: ClassVar[List[str]] = ["description", "name", "parameters"] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionFunctions from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionFunctions from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "description": obj.get("description"), + "name": obj.get("name"), + "parameters": obj.get("parameters"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call.py new file mode 100644 index 00000000..341a2a74 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call.py @@ -0,0 +1,113 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator + +from openapi_server.models.chat_completion_message_tool_call_function import ( + ChatCompletionMessageToolCallFunction, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionMessageToolCall(BaseModel): + """ + ChatCompletionMessageToolCall + """ # noqa: E501 + + id: StrictStr = Field(description="The ID of the tool call.") + type: StrictStr = Field( + description="The type of the tool. Currently, only `function` is supported." + ) + function: ChatCompletionMessageToolCallFunction + __properties: ClassVar[List[str]] = ["id", "type", "function"] + + @field_validator("type") + def type_validate_enum(cls, value): + """Validates the enum""" + if value not in ("function"): + raise ValueError("must be one of enum values ('function')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionMessageToolCall from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of function + if self.function: + _dict["function"] = self.function.to_dict() + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionMessageToolCall from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "id": obj.get("id"), + "type": obj.get("type"), + "function": ChatCompletionMessageToolCallFunction.from_dict( + obj.get("function") + ) + if obj.get("function") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk.py new file mode 100644 index 00000000..a03dca04 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk.py @@ -0,0 +1,121 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator + +from openapi_server.models.chat_completion_message_tool_call_chunk_function import ( + ChatCompletionMessageToolCallChunkFunction, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionMessageToolCallChunk(BaseModel): + """ + ChatCompletionMessageToolCallChunk + """ # noqa: E501 + + index: StrictInt + id: Optional[StrictStr] = Field( + default=None, description="The ID of the tool call." + ) + type: Optional[StrictStr] = Field( + default=None, + description="The type of the tool. Currently, only `function` is supported.", + ) + function: Optional[ChatCompletionMessageToolCallChunkFunction] = None + __properties: ClassVar[List[str]] = ["index", "id", "type", "function"] + + @field_validator("type") + def type_validate_enum(cls, value): + """Validates the enum""" + if value is None: + return value + + if value not in ("function"): + raise ValueError("must be one of enum values ('function')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionMessageToolCallChunk from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of function + if self.function: + _dict["function"] = self.function.to_dict() + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionMessageToolCallChunk from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "index": obj.get("index"), + "id": obj.get("id"), + "type": obj.get("type"), + "function": ChatCompletionMessageToolCallChunkFunction.from_dict( + obj.get("function") + ) + if obj.get("function") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk_function.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk_function.py new file mode 100644 index 00000000..d88ea365 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk_function.py @@ -0,0 +1,93 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictStr + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionMessageToolCallChunkFunction(BaseModel): + """ + ChatCompletionMessageToolCallChunkFunction + """ # noqa: E501 + + name: Optional[StrictStr] = Field( + default=None, description="The name of the function to call." + ) + arguments: Optional[StrictStr] = Field( + default=None, + description="The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function.", + ) + __properties: ClassVar[List[str]] = ["name", "arguments"] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionMessageToolCallChunkFunction from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionMessageToolCallChunkFunction from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + {"name": obj.get("name"), "arguments": obj.get("arguments")} + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_function.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_function.py new file mode 100644 index 00000000..4f380064 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_function.py @@ -0,0 +1,90 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, Field, StrictStr + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionMessageToolCallFunction(BaseModel): + """ + The function that the model called. + """ # noqa: E501 + + name: StrictStr = Field(description="The name of the function to call.") + arguments: StrictStr = Field( + description="The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function." + ) + __properties: ClassVar[List[str]] = ["name", "arguments"] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionMessageToolCallFunction from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionMessageToolCallFunction from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + {"name": obj.get("name"), "arguments": obj.get("arguments")} + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_named_tool_choice.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_named_tool_choice.py new file mode 100644 index 00000000..7244baea --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_named_tool_choice.py @@ -0,0 +1,111 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator + +from openapi_server.models.chat_completion_named_tool_choice_function import ( + ChatCompletionNamedToolChoiceFunction, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionNamedToolChoice(BaseModel): + """ + Specifies a tool the model should use. Use to force the model to call a specific function. + """ # noqa: E501 + + type: StrictStr = Field( + description="The type of the tool. Currently, only `function` is supported." + ) + function: ChatCompletionNamedToolChoiceFunction + __properties: ClassVar[List[str]] = ["type", "function"] + + @field_validator("type") + def type_validate_enum(cls, value): + """Validates the enum""" + if value not in ("function"): + raise ValueError("must be one of enum values ('function')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionNamedToolChoice from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of function + if self.function: + _dict["function"] = self.function.to_dict() + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionNamedToolChoice from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "type": obj.get("type"), + "function": ChatCompletionNamedToolChoiceFunction.from_dict( + obj.get("function") + ) + if obj.get("function") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_named_tool_choice_function.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_named_tool_choice_function.py new file mode 100644 index 00000000..92560184 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_named_tool_choice_function.py @@ -0,0 +1,85 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, Field, StrictStr + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionNamedToolChoiceFunction(BaseModel): + """ + ChatCompletionNamedToolChoiceFunction + """ # noqa: E501 + + name: StrictStr = Field(description="The name of the function to call.") + __properties: ClassVar[List[str]] = ["name"] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionNamedToolChoiceFunction from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionNamedToolChoiceFunction from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({"name": obj.get("name")}) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_assistant_message.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_assistant_message.py new file mode 100644 index 00000000..2e3da02c --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_assistant_message.py @@ -0,0 +1,152 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator + +from openapi_server.models.chat_completion_message_tool_call import ( + ChatCompletionMessageToolCall, +) +from openapi_server.models.chat_completion_request_assistant_message_function_call import ( + ChatCompletionRequestAssistantMessageFunctionCall, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionRequestAssistantMessage(BaseModel): + """ + ChatCompletionRequestAssistantMessage + """ # noqa: E501 + + content: Optional[StrictStr] = Field( + default=None, + description="The contents of the assistant message. Required unless `tool_calls` or `function_call` is specified. ", + ) + role: StrictStr = Field( + description="The role of the messages author, in this case `assistant`." + ) + name: Optional[StrictStr] = Field( + default=None, + description="An optional name for the participant. Provides the model information to differentiate between participants of the same role.", + ) + tool_calls: Optional[List[ChatCompletionMessageToolCall]] = Field( + default=None, + description="The tool calls generated by the model, such as function calls.", + ) + function_call: Optional[ChatCompletionRequestAssistantMessageFunctionCall] = None + __properties: ClassVar[List[str]] = [ + "content", + "role", + "name", + "tool_calls", + "function_call", + ] + + @field_validator("role") + def role_validate_enum(cls, value): + """Validates the enum""" + if value not in ("assistant"): + raise ValueError("must be one of enum values ('assistant')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionRequestAssistantMessage from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of each item in tool_calls (list) + _items = [] + if self.tool_calls: + for _item in self.tool_calls: + if _item: + _items.append(_item.to_dict()) + _dict["tool_calls"] = _items + # override the default output from pydantic by calling `to_dict()` of function_call + if self.function_call: + _dict["function_call"] = self.function_call.to_dict() + # set to None if content (nullable) is None + # and model_fields_set contains the field + if self.content is None and "content" in self.model_fields_set: + _dict["content"] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionRequestAssistantMessage from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "content": obj.get("content"), + "role": obj.get("role"), + "name": obj.get("name"), + "tool_calls": [ + ChatCompletionMessageToolCall.from_dict(_item) + for _item in obj.get("tool_calls") + ] + if obj.get("tool_calls") is not None + else None, + "function_call": ChatCompletionRequestAssistantMessageFunctionCall.from_dict( + obj.get("function_call") + ) + if obj.get("function_call") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_assistant_message_function_call.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_assistant_message_function_call.py new file mode 100644 index 00000000..6e6b293d --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_assistant_message_function_call.py @@ -0,0 +1,90 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, Field, StrictStr + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionRequestAssistantMessageFunctionCall(BaseModel): + """ + Deprecated and replaced by `tool_calls`. The name and arguments of a function that should be called, as generated by the model. + """ # noqa: E501 + + arguments: StrictStr = Field( + description="The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function." + ) + name: StrictStr = Field(description="The name of the function to call.") + __properties: ClassVar[List[str]] = ["arguments", "name"] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionRequestAssistantMessageFunctionCall from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionRequestAssistantMessageFunctionCall from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + {"arguments": obj.get("arguments"), "name": obj.get("name")} + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_function_message.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_function_message.py new file mode 100644 index 00000000..bfa3d7b8 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_function_message.py @@ -0,0 +1,109 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionRequestFunctionMessage(BaseModel): + """ + ChatCompletionRequestFunctionMessage + """ # noqa: E501 + + role: StrictStr = Field( + description="The role of the messages author, in this case `function`." + ) + content: Optional[StrictStr] = Field( + description="The contents of the function message." + ) + name: StrictStr = Field(description="The name of the function to call.") + __properties: ClassVar[List[str]] = ["role", "content", "name"] + + @field_validator("role") + def role_validate_enum(cls, value): + """Validates the enum""" + if value not in ("function"): + raise ValueError("must be one of enum values ('function')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionRequestFunctionMessage from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # set to None if content (nullable) is None + # and model_fields_set contains the field + if self.content is None and "content" in self.model_fields_set: + _dict["content"] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionRequestFunctionMessage from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "role": obj.get("role"), + "content": obj.get("content"), + "name": obj.get("name"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message.py new file mode 100644 index 00000000..dc47e6b7 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message.py @@ -0,0 +1,261 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from inspect import getfullargspec +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictStr, + ValidationError, + field_validator, +) +from typing_extensions import Literal + +from openapi_server.models.chat_completion_request_assistant_message import ( + ChatCompletionRequestAssistantMessage, +) +from openapi_server.models.chat_completion_request_function_message import ( + ChatCompletionRequestFunctionMessage, +) +from openapi_server.models.chat_completion_request_system_message import ( + ChatCompletionRequestSystemMessage, +) +from openapi_server.models.chat_completion_request_tool_message import ( + ChatCompletionRequestToolMessage, +) +from openapi_server.models.chat_completion_request_user_message import ( + ChatCompletionRequestUserMessage, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + +CHATCOMPLETIONREQUESTMESSAGE_ONE_OF_SCHEMAS = [ + "ChatCompletionRequestAssistantMessage", + "ChatCompletionRequestFunctionMessage", + "ChatCompletionRequestSystemMessage", + "ChatCompletionRequestToolMessage", + "ChatCompletionRequestUserMessage", +] + + +class ChatCompletionRequestMessage(BaseModel): + """ + ChatCompletionRequestMessage + """ + + # data type: ChatCompletionRequestSystemMessage + oneof_schema_1_validator: Optional[ChatCompletionRequestSystemMessage] = None + # data type: ChatCompletionRequestUserMessage + oneof_schema_2_validator: Optional[ChatCompletionRequestUserMessage] = None + # data type: ChatCompletionRequestAssistantMessage + oneof_schema_3_validator: Optional[ChatCompletionRequestAssistantMessage] = None + # data type: ChatCompletionRequestToolMessage + oneof_schema_4_validator: Optional[ChatCompletionRequestToolMessage] = None + # data type: ChatCompletionRequestFunctionMessage + oneof_schema_5_validator: Optional[ChatCompletionRequestFunctionMessage] = None + actual_instance: Optional[ + Union[ + ChatCompletionRequestAssistantMessage, + ChatCompletionRequestFunctionMessage, + ChatCompletionRequestSystemMessage, + ChatCompletionRequestToolMessage, + ChatCompletionRequestUserMessage, + ] + ] = None + one_of_schemas: List[str] = Literal[ + "ChatCompletionRequestAssistantMessage", + "ChatCompletionRequestFunctionMessage", + "ChatCompletionRequestSystemMessage", + "ChatCompletionRequestToolMessage", + "ChatCompletionRequestUserMessage", + ] + + model_config = { + "validate_assignment": True, + "protected_namespaces": (), + } + + def __init__(self, *args, **kwargs) -> None: + if args: + if len(args) > 1: + raise ValueError( + "If a position argument is used, only 1 is allowed to set `actual_instance`" + ) + if kwargs: + raise ValueError( + "If a position argument is used, keyword arguments cannot be used." + ) + super().__init__(actual_instance=args[0]) + else: + super().__init__(**kwargs) + + @field_validator("actual_instance") + def actual_instance_must_validate_oneof(cls, v): + instance = ChatCompletionRequestMessage.model_construct() + error_messages = [] + match = 0 + # validate data type: ChatCompletionRequestSystemMessage + if not isinstance(v, ChatCompletionRequestSystemMessage): + error_messages.append( + f"Error! Input type `{type(v)}` is not `ChatCompletionRequestSystemMessage`" + ) + else: + match += 1 + # validate data type: ChatCompletionRequestUserMessage + if not isinstance(v, ChatCompletionRequestUserMessage): + error_messages.append( + f"Error! Input type `{type(v)}` is not `ChatCompletionRequestUserMessage`" + ) + else: + match += 1 + # validate data type: ChatCompletionRequestAssistantMessage + if not isinstance(v, ChatCompletionRequestAssistantMessage): + error_messages.append( + f"Error! Input type `{type(v)}` is not `ChatCompletionRequestAssistantMessage`" + ) + else: + match += 1 + # validate data type: ChatCompletionRequestToolMessage + if not isinstance(v, ChatCompletionRequestToolMessage): + error_messages.append( + f"Error! Input type `{type(v)}` is not `ChatCompletionRequestToolMessage`" + ) + else: + match += 1 + # validate data type: ChatCompletionRequestFunctionMessage + if not isinstance(v, ChatCompletionRequestFunctionMessage): + error_messages.append( + f"Error! Input type `{type(v)}` is not `ChatCompletionRequestFunctionMessage`" + ) + else: + match += 1 + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when setting `actual_instance` in ChatCompletionRequestMessage with oneOf schemas: ChatCompletionRequestAssistantMessage, ChatCompletionRequestFunctionMessage, ChatCompletionRequestSystemMessage, ChatCompletionRequestToolMessage, ChatCompletionRequestUserMessage. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when setting `actual_instance` in ChatCompletionRequestMessage with oneOf schemas: ChatCompletionRequestAssistantMessage, ChatCompletionRequestFunctionMessage, ChatCompletionRequestSystemMessage, ChatCompletionRequestToolMessage, ChatCompletionRequestUserMessage. Details: " + + ", ".join(error_messages) + ) + else: + return v + + @classmethod + def from_dict(cls, obj: dict) -> Self: + return cls.from_json(json.dumps(obj)) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Returns the object represented by the json string""" + instance = cls.model_construct() + error_messages = [] + match = 0 + + # deserialize data into ChatCompletionRequestSystemMessage + try: + instance.actual_instance = ChatCompletionRequestSystemMessage.from_json( + json_str + ) + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into ChatCompletionRequestUserMessage + try: + instance.actual_instance = ChatCompletionRequestUserMessage.from_json( + json_str + ) + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into ChatCompletionRequestAssistantMessage + try: + instance.actual_instance = ChatCompletionRequestAssistantMessage.from_json( + json_str + ) + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into ChatCompletionRequestToolMessage + try: + instance.actual_instance = ChatCompletionRequestToolMessage.from_json( + json_str + ) + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into ChatCompletionRequestFunctionMessage + try: + instance.actual_instance = ChatCompletionRequestFunctionMessage.from_json( + json_str + ) + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when deserializing the JSON string into ChatCompletionRequestMessage with oneOf schemas: ChatCompletionRequestAssistantMessage, ChatCompletionRequestFunctionMessage, ChatCompletionRequestSystemMessage, ChatCompletionRequestToolMessage, ChatCompletionRequestUserMessage. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when deserializing the JSON string into ChatCompletionRequestMessage with oneOf schemas: ChatCompletionRequestAssistantMessage, ChatCompletionRequestFunctionMessage, ChatCompletionRequestSystemMessage, ChatCompletionRequestToolMessage, ChatCompletionRequestUserMessage. Details: " + + ", ".join(error_messages) + ) + else: + return instance + + def to_json(self) -> str: + """Returns the JSON representation of the actual instance""" + if self.actual_instance is None: + return "null" + + to_json = getattr(self.actual_instance, "to_json", None) + if callable(to_json): + return self.actual_instance.to_json() + else: + return json.dumps(self.actual_instance) + + def to_dict(self) -> Dict: + """Returns the dict representation of the actual instance""" + if self.actual_instance is None: + return None + + to_dict = getattr(self.actual_instance, "to_dict", None) + if callable(to_dict): + return self.actual_instance.to_dict() + else: + # primitive type + return self.actual_instance + + def to_str(self) -> str: + """Returns the string representation of the actual instance""" + return pprint.pformat(self.model_dump()) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part.py new file mode 100644 index 00000000..c1f4025d --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part.py @@ -0,0 +1,196 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from inspect import getfullargspec +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictStr, + ValidationError, + field_validator, +) +from typing_extensions import Literal + +from openapi_server.models.chat_completion_request_message_content_part_image import ( + ChatCompletionRequestMessageContentPartImage, +) +from openapi_server.models.chat_completion_request_message_content_part_text import ( + ChatCompletionRequestMessageContentPartText, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + +CHATCOMPLETIONREQUESTMESSAGECONTENTPART_ONE_OF_SCHEMAS = [ + "ChatCompletionRequestMessageContentPartImage", + "ChatCompletionRequestMessageContentPartText", +] + + +class ChatCompletionRequestMessageContentPart(BaseModel): + """ + ChatCompletionRequestMessageContentPart + """ + + # data type: ChatCompletionRequestMessageContentPartText + oneof_schema_1_validator: Optional[ + ChatCompletionRequestMessageContentPartText + ] = None + # data type: ChatCompletionRequestMessageContentPartImage + oneof_schema_2_validator: Optional[ + ChatCompletionRequestMessageContentPartImage + ] = None + actual_instance: Optional[ + Union[ + ChatCompletionRequestMessageContentPartImage, + ChatCompletionRequestMessageContentPartText, + ] + ] = None + one_of_schemas: List[str] = Literal[ + "ChatCompletionRequestMessageContentPartImage", + "ChatCompletionRequestMessageContentPartText", + ] + + model_config = { + "validate_assignment": True, + "protected_namespaces": (), + } + + def __init__(self, *args, **kwargs) -> None: + if args: + if len(args) > 1: + raise ValueError( + "If a position argument is used, only 1 is allowed to set `actual_instance`" + ) + if kwargs: + raise ValueError( + "If a position argument is used, keyword arguments cannot be used." + ) + super().__init__(actual_instance=args[0]) + else: + super().__init__(**kwargs) + + @field_validator("actual_instance") + def actual_instance_must_validate_oneof(cls, v): + instance = ChatCompletionRequestMessageContentPart.model_construct() + error_messages = [] + match = 0 + # validate data type: ChatCompletionRequestMessageContentPartText + if not isinstance(v, ChatCompletionRequestMessageContentPartText): + error_messages.append( + f"Error! Input type `{type(v)}` is not `ChatCompletionRequestMessageContentPartText`" + ) + else: + match += 1 + # validate data type: ChatCompletionRequestMessageContentPartImage + if not isinstance(v, ChatCompletionRequestMessageContentPartImage): + error_messages.append( + f"Error! Input type `{type(v)}` is not `ChatCompletionRequestMessageContentPartImage`" + ) + else: + match += 1 + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when setting `actual_instance` in ChatCompletionRequestMessageContentPart with oneOf schemas: ChatCompletionRequestMessageContentPartImage, ChatCompletionRequestMessageContentPartText. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when setting `actual_instance` in ChatCompletionRequestMessageContentPart with oneOf schemas: ChatCompletionRequestMessageContentPartImage, ChatCompletionRequestMessageContentPartText. Details: " + + ", ".join(error_messages) + ) + else: + return v + + @classmethod + def from_dict(cls, obj: dict) -> Self: + return cls.from_json(json.dumps(obj)) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Returns the object represented by the json string""" + instance = cls.model_construct() + error_messages = [] + match = 0 + + # deserialize data into ChatCompletionRequestMessageContentPartText + try: + instance.actual_instance = ( + ChatCompletionRequestMessageContentPartText.from_json(json_str) + ) + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into ChatCompletionRequestMessageContentPartImage + try: + instance.actual_instance = ( + ChatCompletionRequestMessageContentPartImage.from_json(json_str) + ) + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when deserializing the JSON string into ChatCompletionRequestMessageContentPart with oneOf schemas: ChatCompletionRequestMessageContentPartImage, ChatCompletionRequestMessageContentPartText. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when deserializing the JSON string into ChatCompletionRequestMessageContentPart with oneOf schemas: ChatCompletionRequestMessageContentPartImage, ChatCompletionRequestMessageContentPartText. Details: " + + ", ".join(error_messages) + ) + else: + return instance + + def to_json(self) -> str: + """Returns the JSON representation of the actual instance""" + if self.actual_instance is None: + return "null" + + to_json = getattr(self.actual_instance, "to_json", None) + if callable(to_json): + return self.actual_instance.to_json() + else: + return json.dumps(self.actual_instance) + + def to_dict(self) -> Dict: + """Returns the dict representation of the actual instance""" + if self.actual_instance is None: + return None + + to_dict = getattr(self.actual_instance, "to_dict", None) + if callable(to_dict): + return self.actual_instance.to_dict() + else: + # primitive type + return self.actual_instance + + def to_str(self) -> str: + """Returns the string representation of the actual instance""" + return pprint.pformat(self.model_dump()) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image.py new file mode 100644 index 00000000..a0de7cc4 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image.py @@ -0,0 +1,109 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator + +from openapi_server.models.chat_completion_request_message_content_part_image_image_url import ( + ChatCompletionRequestMessageContentPartImageImageUrl, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionRequestMessageContentPartImage(BaseModel): + """ + ChatCompletionRequestMessageContentPartImage + """ # noqa: E501 + + type: StrictStr = Field(description="The type of the content part.") + image_url: ChatCompletionRequestMessageContentPartImageImageUrl + __properties: ClassVar[List[str]] = ["type", "image_url"] + + @field_validator("type") + def type_validate_enum(cls, value): + """Validates the enum""" + if value not in ("image_url"): + raise ValueError("must be one of enum values ('image_url')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionRequestMessageContentPartImage from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of image_url + if self.image_url: + _dict["image_url"] = self.image_url.to_dict() + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionRequestMessageContentPartImage from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "type": obj.get("type"), + "image_url": ChatCompletionRequestMessageContentPartImageImageUrl.from_dict( + obj.get("image_url") + ) + if obj.get("image_url") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image_image_url.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image_image_url.py new file mode 100644 index 00000000..64cc2664 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image_image_url.py @@ -0,0 +1,108 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionRequestMessageContentPartImageImageUrl(BaseModel): + """ + ChatCompletionRequestMessageContentPartImageImageUrl + """ # noqa: E501 + + url: StrictStr = Field( + description="Either a URL of the image or the base64 encoded image data." + ) + detail: Optional[StrictStr] = Field( + default="auto", + description="Specifies the detail level of the image. Learn more in the [Vision guide](/docs/guides/vision/low-or-high-fidelity-image-understanding).", + ) + __properties: ClassVar[List[str]] = ["url", "detail"] + + @field_validator("detail") + def detail_validate_enum(cls, value): + """Validates the enum""" + if value is None: + return value + + if value not in ("auto", "low", "high"): + raise ValueError("must be one of enum values ('auto', 'low', 'high')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionRequestMessageContentPartImageImageUrl from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionRequestMessageContentPartImageImageUrl from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "url": obj.get("url"), + "detail": obj.get("detail") + if obj.get("detail") is not None + else "auto", + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_text.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_text.py new file mode 100644 index 00000000..b7071e69 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_text.py @@ -0,0 +1,93 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionRequestMessageContentPartText(BaseModel): + """ + ChatCompletionRequestMessageContentPartText + """ # noqa: E501 + + type: StrictStr = Field(description="The type of the content part.") + text: StrictStr = Field(description="The text content.") + __properties: ClassVar[List[str]] = ["type", "text"] + + @field_validator("type") + def type_validate_enum(cls, value): + """Validates the enum""" + if value not in ("text"): + raise ValueError("must be one of enum values ('text')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionRequestMessageContentPartText from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionRequestMessageContentPartText from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({"type": obj.get("type"), "text": obj.get("text")}) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_system_message.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_system_message.py new file mode 100644 index 00000000..9350599a --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_system_message.py @@ -0,0 +1,105 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionRequestSystemMessage(BaseModel): + """ + ChatCompletionRequestSystemMessage + """ # noqa: E501 + + content: StrictStr = Field(description="The contents of the system message.") + role: StrictStr = Field( + description="The role of the messages author, in this case `system`." + ) + name: Optional[StrictStr] = Field( + default=None, + description="An optional name for the participant. Provides the model information to differentiate between participants of the same role.", + ) + __properties: ClassVar[List[str]] = ["content", "role", "name"] + + @field_validator("role") + def role_validate_enum(cls, value): + """Validates the enum""" + if value not in ("system"): + raise ValueError("must be one of enum values ('system')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionRequestSystemMessage from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionRequestSystemMessage from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "content": obj.get("content"), + "role": obj.get("role"), + "name": obj.get("name"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_tool_message.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_tool_message.py new file mode 100644 index 00000000..f9d0f65a --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_tool_message.py @@ -0,0 +1,104 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionRequestToolMessage(BaseModel): + """ + ChatCompletionRequestToolMessage + """ # noqa: E501 + + role: StrictStr = Field( + description="The role of the messages author, in this case `tool`." + ) + content: StrictStr = Field(description="The contents of the tool message.") + tool_call_id: StrictStr = Field( + description="Tool call that this message is responding to." + ) + __properties: ClassVar[List[str]] = ["role", "content", "tool_call_id"] + + @field_validator("role") + def role_validate_enum(cls, value): + """Validates the enum""" + if value not in ("tool"): + raise ValueError("must be one of enum values ('tool')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionRequestToolMessage from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionRequestToolMessage from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "role": obj.get("role"), + "content": obj.get("content"), + "tool_call_id": obj.get("tool_call_id"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_user_message.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_user_message.py new file mode 100644 index 00000000..0ae1d3c2 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_user_message.py @@ -0,0 +1,116 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator + +from openapi_server.models.chat_completion_request_user_message_content import ( + ChatCompletionRequestUserMessageContent, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionRequestUserMessage(BaseModel): + """ + ChatCompletionRequestUserMessage + """ # noqa: E501 + + content: ChatCompletionRequestUserMessageContent + role: StrictStr = Field( + description="The role of the messages author, in this case `user`." + ) + name: Optional[StrictStr] = Field( + default=None, + description="An optional name for the participant. Provides the model information to differentiate between participants of the same role.", + ) + __properties: ClassVar[List[str]] = ["content", "role", "name"] + + @field_validator("role") + def role_validate_enum(cls, value): + """Validates the enum""" + if value not in ("user"): + raise ValueError("must be one of enum values ('user')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionRequestUserMessage from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of content + if self.content: + _dict["content"] = self.content.to_dict() + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionRequestUserMessage from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "content": ChatCompletionRequestUserMessageContent.from_dict( + obj.get("content") + ) + if obj.get("content") is not None + else None, + "role": obj.get("role"), + "name": obj.get("name"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_user_message_content.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_user_message_content.py new file mode 100644 index 00000000..00611944 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_user_message_content.py @@ -0,0 +1,192 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from inspect import getfullargspec +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictStr, + ValidationError, + field_validator, +) +from typing_extensions import Annotated, Literal + +from openapi_server.models.chat_completion_request_message_content_part import ( + ChatCompletionRequestMessageContentPart, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + +CHATCOMPLETIONREQUESTUSERMESSAGECONTENT_ONE_OF_SCHEMAS = [ + "List[ChatCompletionRequestMessageContentPart]", + "str", +] + + +class ChatCompletionRequestUserMessageContent(BaseModel): + """ + The contents of the user message. + """ + + # data type: str + oneof_schema_1_validator: Optional[StrictStr] = Field( + default=None, description="The text contents of the message." + ) + # data type: List[ChatCompletionRequestMessageContentPart] + oneof_schema_2_validator: Optional[ + Annotated[List[ChatCompletionRequestMessageContentPart], Field(min_length=1)] + ] = Field( + default=None, + description="An array of content parts with a defined type, each can be of type `text` or `image_url` when passing in images. You can pass multiple images by adding multiple `image_url` content parts. Image input is only supported when using the `gpt-4-visual-preview` model.", + ) + actual_instance: Optional[ + Union[List[ChatCompletionRequestMessageContentPart], str] + ] = None + one_of_schemas: List[str] = Literal[ + "List[ChatCompletionRequestMessageContentPart]", "str" + ] + + model_config = { + "validate_assignment": True, + "protected_namespaces": (), + } + + def __init__(self, *args, **kwargs) -> None: + if args: + if len(args) > 1: + raise ValueError( + "If a position argument is used, only 1 is allowed to set `actual_instance`" + ) + if kwargs: + raise ValueError( + "If a position argument is used, keyword arguments cannot be used." + ) + super().__init__(actual_instance=args[0]) + else: + super().__init__(**kwargs) + + @field_validator("actual_instance") + def actual_instance_must_validate_oneof(cls, v): + instance = ChatCompletionRequestUserMessageContent.model_construct() + error_messages = [] + match = 0 + # validate data type: str + try: + instance.oneof_schema_1_validator = v + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # validate data type: List[ChatCompletionRequestMessageContentPart] + try: + instance.oneof_schema_2_validator = v + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when setting `actual_instance` in ChatCompletionRequestUserMessageContent with oneOf schemas: List[ChatCompletionRequestMessageContentPart], str. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when setting `actual_instance` in ChatCompletionRequestUserMessageContent with oneOf schemas: List[ChatCompletionRequestMessageContentPart], str. Details: " + + ", ".join(error_messages) + ) + else: + return v + + @classmethod + def from_dict(cls, obj: dict) -> Self: + return cls.from_json(json.dumps(obj)) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Returns the object represented by the json string""" + instance = cls.model_construct() + error_messages = [] + match = 0 + + # deserialize data into str + try: + # validation + instance.oneof_schema_1_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.oneof_schema_1_validator + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into List[ChatCompletionRequestMessageContentPart] + try: + # validation + instance.oneof_schema_2_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.oneof_schema_2_validator + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when deserializing the JSON string into ChatCompletionRequestUserMessageContent with oneOf schemas: List[ChatCompletionRequestMessageContentPart], str. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when deserializing the JSON string into ChatCompletionRequestUserMessageContent with oneOf schemas: List[ChatCompletionRequestMessageContentPart], str. Details: " + + ", ".join(error_messages) + ) + else: + return instance + + def to_json(self) -> str: + """Returns the JSON representation of the actual instance""" + if self.actual_instance is None: + return "null" + + to_json = getattr(self.actual_instance, "to_json", None) + if callable(to_json): + return self.actual_instance.to_json() + else: + return json.dumps(self.actual_instance) + + def to_dict(self) -> Dict: + """Returns the dict representation of the actual instance""" + if self.actual_instance is None: + return None + + to_dict = getattr(self.actual_instance, "to_dict", None) + if callable(to_dict): + return self.actual_instance.to_dict() + else: + # primitive type + return self.actual_instance + + def to_str(self) -> str: + """Returns the string representation of the actual instance""" + return pprint.pformat(self.model_dump()) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_response_message.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_response_message.py new file mode 100644 index 00000000..586bc2c8 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_response_message.py @@ -0,0 +1,141 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator + +from openapi_server.models.chat_completion_message_tool_call import ( + ChatCompletionMessageToolCall, +) +from openapi_server.models.chat_completion_request_assistant_message_function_call import ( + ChatCompletionRequestAssistantMessageFunctionCall, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionResponseMessage(BaseModel): + """ + A chat completion message generated by the model. + """ # noqa: E501 + + content: Optional[StrictStr] = Field(description="The contents of the message.") + tool_calls: Optional[List[ChatCompletionMessageToolCall]] = Field( + default=None, + description="The tool calls generated by the model, such as function calls.", + ) + role: StrictStr = Field(description="The role of the author of this message.") + function_call: Optional[ChatCompletionRequestAssistantMessageFunctionCall] = None + __properties: ClassVar[List[str]] = [ + "content", + "tool_calls", + "role", + "function_call", + ] + + @field_validator("role") + def role_validate_enum(cls, value): + """Validates the enum""" + if value not in ("assistant"): + raise ValueError("must be one of enum values ('assistant')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionResponseMessage from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of each item in tool_calls (list) + _items = [] + if self.tool_calls: + for _item in self.tool_calls: + if _item: + _items.append(_item.to_dict()) + _dict["tool_calls"] = _items + # override the default output from pydantic by calling `to_dict()` of function_call + if self.function_call: + _dict["function_call"] = self.function_call.to_dict() + # set to None if content (nullable) is None + # and model_fields_set contains the field + if self.content is None and "content" in self.model_fields_set: + _dict["content"] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionResponseMessage from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "content": obj.get("content"), + "tool_calls": [ + ChatCompletionMessageToolCall.from_dict(_item) + for _item in obj.get("tool_calls") + ] + if obj.get("tool_calls") is not None + else None, + "role": obj.get("role"), + "function_call": ChatCompletionRequestAssistantMessageFunctionCall.from_dict( + obj.get("function_call") + ) + if obj.get("function_call") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_role.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_role.py new file mode 100644 index 00000000..6176113b --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_role.py @@ -0,0 +1,45 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from enum import Enum + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionRole(str, Enum): + """ + The role of the author of a message + """ + + """ + allowed enum values + """ + SYSTEM = "system" + USER = "user" + ASSISTANT = "assistant" + TOOL = "tool" + FUNCTION = "function" + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionRole from a JSON string""" + return cls(json.loads(json_str)) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_stream_response_delta.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_stream_response_delta.py new file mode 100644 index 00000000..3b1ef2fb --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_stream_response_delta.py @@ -0,0 +1,147 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator + +from openapi_server.models.chat_completion_message_tool_call_chunk import ( + ChatCompletionMessageToolCallChunk, +) +from openapi_server.models.chat_completion_stream_response_delta_function_call import ( + ChatCompletionStreamResponseDeltaFunctionCall, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionStreamResponseDelta(BaseModel): + """ + A chat completion delta generated by streamed model responses. + """ # noqa: E501 + + content: Optional[StrictStr] = Field( + default=None, description="The contents of the chunk message." + ) + function_call: Optional[ChatCompletionStreamResponseDeltaFunctionCall] = None + tool_calls: Optional[List[ChatCompletionMessageToolCallChunk]] = None + role: Optional[StrictStr] = Field( + default=None, description="The role of the author of this message." + ) + __properties: ClassVar[List[str]] = [ + "content", + "function_call", + "tool_calls", + "role", + ] + + @field_validator("role") + def role_validate_enum(cls, value): + """Validates the enum""" + if value is None: + return value + + if value not in ("system", "user", "assistant", "tool"): + raise ValueError( + "must be one of enum values ('system', 'user', 'assistant', 'tool')" + ) + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionStreamResponseDelta from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of function_call + if self.function_call: + _dict["function_call"] = self.function_call.to_dict() + # override the default output from pydantic by calling `to_dict()` of each item in tool_calls (list) + _items = [] + if self.tool_calls: + for _item in self.tool_calls: + if _item: + _items.append(_item.to_dict()) + _dict["tool_calls"] = _items + # set to None if content (nullable) is None + # and model_fields_set contains the field + if self.content is None and "content" in self.model_fields_set: + _dict["content"] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionStreamResponseDelta from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "content": obj.get("content"), + "function_call": ChatCompletionStreamResponseDeltaFunctionCall.from_dict( + obj.get("function_call") + ) + if obj.get("function_call") is not None + else None, + "tool_calls": [ + ChatCompletionMessageToolCallChunk.from_dict(_item) + for _item in obj.get("tool_calls") + ] + if obj.get("tool_calls") is not None + else None, + "role": obj.get("role"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_stream_response_delta_function_call.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_stream_response_delta_function_call.py new file mode 100644 index 00000000..65864157 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_stream_response_delta_function_call.py @@ -0,0 +1,93 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictStr + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionStreamResponseDeltaFunctionCall(BaseModel): + """ + Deprecated and replaced by `tool_calls`. The name and arguments of a function that should be called, as generated by the model. + """ # noqa: E501 + + arguments: Optional[StrictStr] = Field( + default=None, + description="The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function.", + ) + name: Optional[StrictStr] = Field( + default=None, description="The name of the function to call." + ) + __properties: ClassVar[List[str]] = ["arguments", "name"] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionStreamResponseDeltaFunctionCall from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionStreamResponseDeltaFunctionCall from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + {"arguments": obj.get("arguments"), "name": obj.get("name")} + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_token_logprob.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_token_logprob.py new file mode 100644 index 00000000..690b933e --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_token_logprob.py @@ -0,0 +1,122 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt, StrictStr + +from openapi_server.models.chat_completion_token_logprob_top_logprobs_inner import ( + ChatCompletionTokenLogprobTopLogprobsInner, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionTokenLogprob(BaseModel): + """ + ChatCompletionTokenLogprob + """ # noqa: E501 + + token: StrictStr = Field(description="The token.") + logprob: Union[StrictFloat, StrictInt] = Field( + description="The log probability of this token, if it is within the top 20 most likely tokens. Otherwise, the value `-9999.0` is used to signify that the token is very unlikely." + ) + bytes: Optional[List[StrictInt]] = Field( + description="A list of integers representing the UTF-8 bytes representation of the token. Useful in instances where characters are represented by multiple tokens and their byte representations must be combined to generate the correct text representation. Can be `null` if there is no bytes representation for the token." + ) + top_logprobs: List[ChatCompletionTokenLogprobTopLogprobsInner] = Field( + description="List of the most likely tokens and their log probability, at this token position. In rare cases, there may be fewer than the number of requested `top_logprobs` returned." + ) + __properties: ClassVar[List[str]] = ["token", "logprob", "bytes", "top_logprobs"] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionTokenLogprob from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of each item in top_logprobs (list) + _items = [] + if self.top_logprobs: + for _item in self.top_logprobs: + if _item: + _items.append(_item.to_dict()) + _dict["top_logprobs"] = _items + # set to None if bytes (nullable) is None + # and model_fields_set contains the field + if self.bytes is None and "bytes" in self.model_fields_set: + _dict["bytes"] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionTokenLogprob from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "token": obj.get("token"), + "logprob": obj.get("logprob"), + "bytes": obj.get("bytes"), + "top_logprobs": [ + ChatCompletionTokenLogprobTopLogprobsInner.from_dict(_item) + for _item in obj.get("top_logprobs") + ] + if obj.get("top_logprobs") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_token_logprob_top_logprobs_inner.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_token_logprob_top_logprobs_inner.py new file mode 100644 index 00000000..d82266e0 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_token_logprob_top_logprobs_inner.py @@ -0,0 +1,102 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt, StrictStr + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionTokenLogprobTopLogprobsInner(BaseModel): + """ + ChatCompletionTokenLogprobTopLogprobsInner + """ # noqa: E501 + + token: StrictStr = Field(description="The token.") + logprob: Union[StrictFloat, StrictInt] = Field( + description="The log probability of this token, if it is within the top 20 most likely tokens. Otherwise, the value `-9999.0` is used to signify that the token is very unlikely." + ) + bytes: Optional[List[StrictInt]] = Field( + description="A list of integers representing the UTF-8 bytes representation of the token. Useful in instances where characters are represented by multiple tokens and their byte representations must be combined to generate the correct text representation. Can be `null` if there is no bytes representation for the token." + ) + __properties: ClassVar[List[str]] = ["token", "logprob", "bytes"] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionTokenLogprobTopLogprobsInner from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # set to None if bytes (nullable) is None + # and model_fields_set contains the field + if self.bytes is None and "bytes" in self.model_fields_set: + _dict["bytes"] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionTokenLogprobTopLogprobsInner from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "token": obj.get("token"), + "logprob": obj.get("logprob"), + "bytes": obj.get("bytes"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_tool.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_tool.py new file mode 100644 index 00000000..430aa31e --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_tool.py @@ -0,0 +1,107 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator + +from openapi_server.models.function_object import FunctionObject + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ChatCompletionTool(BaseModel): + """ + ChatCompletionTool + """ # noqa: E501 + + type: StrictStr = Field( + description="The type of the tool. Currently, only `function` is supported." + ) + function: FunctionObject + __properties: ClassVar[List[str]] = ["type", "function"] + + @field_validator("type") + def type_validate_enum(cls, value): + """Validates the enum""" + if value not in ("function"): + raise ValueError("must be one of enum values ('function')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ChatCompletionTool from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of function + if self.function: + _dict["function"] = self.function.to_dict() + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ChatCompletionTool from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "type": obj.get("type"), + "function": FunctionObject.from_dict(obj.get("function")) + if obj.get("function") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_tool_choice_option.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_tool_choice_option.py new file mode 100644 index 00000000..d6c41b50 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_tool_choice_option.py @@ -0,0 +1,179 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from inspect import getfullargspec +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictStr, + ValidationError, + field_validator, +) +from typing_extensions import Literal + +from openapi_server.models.chat_completion_named_tool_choice import ( + ChatCompletionNamedToolChoice, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + +CHATCOMPLETIONTOOLCHOICEOPTION_ONE_OF_SCHEMAS = ["ChatCompletionNamedToolChoice", "str"] + + +class ChatCompletionToolChoiceOption(BaseModel): + """ + Controls which (if any) tool is called by the model. `none` means the model will not call any tool and instead generates a message. `auto` means the model can pick between generating a message or calling one or more tools. `required` means the model must call one or more tools. Specifying a particular tool via `{\"type\": \"function\", \"function\": {\"name\": \"my_function\"}}` forces the model to call that tool. `none` is the default when no tools are present. `auto` is the default if tools are present. + """ + + # data type: str + oneof_schema_1_validator: Optional[StrictStr] = Field( + default=None, + description="`none` means the model will not call any tool and instead generates a message. `auto` means the model can pick between generating a message or calling one or more tools. `required` means the model must call one or more tools. ", + ) + # data type: ChatCompletionNamedToolChoice + oneof_schema_2_validator: Optional[ChatCompletionNamedToolChoice] = None + actual_instance: Optional[Union[ChatCompletionNamedToolChoice, str]] = None + one_of_schemas: List[str] = Literal["ChatCompletionNamedToolChoice", "str"] + + model_config = { + "validate_assignment": True, + "protected_namespaces": (), + } + + def __init__(self, *args, **kwargs) -> None: + if args: + if len(args) > 1: + raise ValueError( + "If a position argument is used, only 1 is allowed to set `actual_instance`" + ) + if kwargs: + raise ValueError( + "If a position argument is used, keyword arguments cannot be used." + ) + super().__init__(actual_instance=args[0]) + else: + super().__init__(**kwargs) + + @field_validator("actual_instance") + def actual_instance_must_validate_oneof(cls, v): + instance = ChatCompletionToolChoiceOption.model_construct() + error_messages = [] + match = 0 + # validate data type: str + try: + instance.oneof_schema_1_validator = v + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # validate data type: ChatCompletionNamedToolChoice + if not isinstance(v, ChatCompletionNamedToolChoice): + error_messages.append( + f"Error! Input type `{type(v)}` is not `ChatCompletionNamedToolChoice`" + ) + else: + match += 1 + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when setting `actual_instance` in ChatCompletionToolChoiceOption with oneOf schemas: ChatCompletionNamedToolChoice, str. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when setting `actual_instance` in ChatCompletionToolChoiceOption with oneOf schemas: ChatCompletionNamedToolChoice, str. Details: " + + ", ".join(error_messages) + ) + else: + return v + + @classmethod + def from_dict(cls, obj: dict) -> Self: + return cls.from_json(json.dumps(obj)) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Returns the object represented by the json string""" + instance = cls.model_construct() + error_messages = [] + match = 0 + + # deserialize data into str + try: + # validation + instance.oneof_schema_1_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.oneof_schema_1_validator + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into ChatCompletionNamedToolChoice + try: + instance.actual_instance = ChatCompletionNamedToolChoice.from_json(json_str) + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when deserializing the JSON string into ChatCompletionToolChoiceOption with oneOf schemas: ChatCompletionNamedToolChoice, str. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when deserializing the JSON string into ChatCompletionToolChoiceOption with oneOf schemas: ChatCompletionNamedToolChoice, str. Details: " + + ", ".join(error_messages) + ) + else: + return instance + + def to_json(self) -> str: + """Returns the JSON representation of the actual instance""" + if self.actual_instance is None: + return "null" + + to_json = getattr(self.actual_instance, "to_json", None) + if callable(to_json): + return self.actual_instance.to_json() + else: + return json.dumps(self.actual_instance) + + def to_dict(self) -> Dict: + """Returns the dict representation of the actual instance""" + if self.actual_instance is None: + return None + + to_dict = getattr(self.actual_instance, "to_dict", None) + if callable(to_dict): + return self.actual_instance.to_dict() + else: + # primitive type + return self.actual_instance + + def to_str(self) -> str: + """Returns the string representation of the actual instance""" + return pprint.pformat(self.model_dump()) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/completion_usage.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/completion_usage.py new file mode 100644 index 00000000..6e8b2902 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/completion_usage.py @@ -0,0 +1,101 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, Field, StrictInt + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class CompletionUsage(BaseModel): + """ + Usage statistics for the completion request. + """ # noqa: E501 + + completion_tokens: StrictInt = Field( + description="Number of tokens in the generated completion." + ) + prompt_tokens: StrictInt = Field(description="Number of tokens in the prompt.") + total_tokens: StrictInt = Field( + description="Total number of tokens used in the request (prompt + completion)." + ) + __properties: ClassVar[List[str]] = [ + "completion_tokens", + "prompt_tokens", + "total_tokens", + ] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of CompletionUsage from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of CompletionUsage from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "completion_tokens": obj.get("completion_tokens"), + "prompt_tokens": obj.get("prompt_tokens"), + "total_tokens": obj.get("total_tokens"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_function_response.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_function_response.py new file mode 100644 index 00000000..b3c6049e --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_function_response.py @@ -0,0 +1,147 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator + +from openapi_server.models.completion_usage import CompletionUsage +from openapi_server.models.create_chat_completion_function_response_choices_inner import ( + CreateChatCompletionFunctionResponseChoicesInner, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class CreateChatCompletionFunctionResponse(BaseModel): + """ + Represents a chat completion response returned by model, based on the provided input. + """ # noqa: E501 + + id: StrictStr = Field(description="A unique identifier for the chat completion.") + choices: List[CreateChatCompletionFunctionResponseChoicesInner] = Field( + description="A list of chat completion choices. Can be more than one if `n` is greater than 1." + ) + created: StrictInt = Field( + description="The Unix timestamp (in seconds) of when the chat completion was created." + ) + model: StrictStr = Field(description="The model used for the chat completion.") + system_fingerprint: Optional[StrictStr] = Field( + default=None, + description="This fingerprint represents the backend configuration that the model runs with. Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. ", + ) + object: StrictStr = Field( + description="The object type, which is always `chat.completion`." + ) + usage: Optional[CompletionUsage] = None + __properties: ClassVar[List[str]] = [ + "id", + "choices", + "created", + "model", + "system_fingerprint", + "object", + "usage", + ] + + @field_validator("object") + def object_validate_enum(cls, value): + """Validates the enum""" + if value not in ("chat.completion"): + raise ValueError("must be one of enum values ('chat.completion')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of CreateChatCompletionFunctionResponse from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of each item in choices (list) + _items = [] + if self.choices: + for _item in self.choices: + if _item: + _items.append(_item.to_dict()) + _dict["choices"] = _items + # override the default output from pydantic by calling `to_dict()` of usage + if self.usage: + _dict["usage"] = self.usage.to_dict() + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of CreateChatCompletionFunctionResponse from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "id": obj.get("id"), + "choices": [ + CreateChatCompletionFunctionResponseChoicesInner.from_dict(_item) + for _item in obj.get("choices") + ] + if obj.get("choices") is not None + else None, + "created": obj.get("created"), + "model": obj.get("model"), + "system_fingerprint": obj.get("system_fingerprint"), + "object": obj.get("object"), + "usage": CompletionUsage.from_dict(obj.get("usage")) + if obj.get("usage") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_function_response_choices_inner.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_function_response_choices_inner.py new file mode 100644 index 00000000..0dd07942 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_function_response_choices_inner.py @@ -0,0 +1,115 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator + +from openapi_server.models.chat_completion_response_message import ( + ChatCompletionResponseMessage, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class CreateChatCompletionFunctionResponseChoicesInner(BaseModel): + """ + CreateChatCompletionFunctionResponseChoicesInner + """ # noqa: E501 + + finish_reason: StrictStr = Field( + description="The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, `length` if the maximum number of tokens specified in the request was reached, `content_filter` if content was omitted due to a flag from our content filters, or `function_call` if the model called a function. " + ) + index: StrictInt = Field( + description="The index of the choice in the list of choices." + ) + message: ChatCompletionResponseMessage + __properties: ClassVar[List[str]] = ["finish_reason", "index", "message"] + + @field_validator("finish_reason") + def finish_reason_validate_enum(cls, value): + """Validates the enum""" + if value not in ("stop", "length", "function_call", "content_filter"): + raise ValueError( + "must be one of enum values ('stop', 'length', 'function_call', 'content_filter')" + ) + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of CreateChatCompletionFunctionResponseChoicesInner from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of message + if self.message: + _dict["message"] = self.message.to_dict() + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of CreateChatCompletionFunctionResponseChoicesInner from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "finish_reason": obj.get("finish_reason"), + "index": obj.get("index"), + "message": ChatCompletionResponseMessage.from_dict(obj.get("message")) + if obj.get("message") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request.py new file mode 100644 index 00000000..06bdab48 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request.py @@ -0,0 +1,377 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, StrictStr +from typing_extensions import Annotated + +from openapi_server.models.chat_completion_functions import ChatCompletionFunctions +from openapi_server.models.chat_completion_request_message import ( + ChatCompletionRequestMessage, +) +from openapi_server.models.chat_completion_tool import ChatCompletionTool +from openapi_server.models.chat_completion_tool_choice_option import ( + ChatCompletionToolChoiceOption, +) +from openapi_server.models.create_chat_completion_request_function_call import ( + CreateChatCompletionRequestFunctionCall, +) +from openapi_server.models.create_chat_completion_request_model import ( + CreateChatCompletionRequestModel, +) +from openapi_server.models.create_chat_completion_request_response_format import ( + CreateChatCompletionRequestResponseFormat, +) +from openapi_server.models.create_chat_completion_request_stop import ( + CreateChatCompletionRequestStop, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class CreateChatCompletionRequest(BaseModel): + """ + CreateChatCompletionRequest + """ # noqa: E501 + + messages: Annotated[ + List[ChatCompletionRequestMessage], Field(min_length=1) + ] = Field( + description="A list of messages comprising the conversation so far. [Example Python code](https://cookbook.openai.com/examples/how_to_format_inputs_to_chatgpt_models)." + ) + model: CreateChatCompletionRequestModel + frequency_penalty: Optional[ + Union[ + Annotated[float, Field(le=2, strict=True, ge=-2)], + Annotated[int, Field(le=2, strict=True, ge=-2)], + ] + ] = Field( + default=0, + description="Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. [See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details) ", + ) + logit_bias: Optional[Dict[str, StrictInt]] = Field( + default=None, + description="Modify the likelihood of specified tokens appearing in the completion. Accepts a JSON object that maps tokens (specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically, the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should result in a ban or exclusive selection of the relevant token. ", + ) + logprobs: Optional[StrictBool] = Field( + default=False, + description="Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the `content` of `message`.", + ) + top_logprobs: Optional[Annotated[int, Field(le=20, strict=True, ge=0)]] = Field( + default=None, + description="An integer between 0 and 20 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to `true` if this parameter is used.", + ) + max_tokens: Optional[StrictInt] = Field( + default=None, + description="The maximum number of [tokens](/tokenizer) that can be generated in the chat completion. The total length of input tokens and generated tokens is limited by the model's context length. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens. ", + ) + n: Optional[Annotated[int, Field(le=128, strict=True, ge=1)]] = Field( + default=1, + description="How many chat completion choices to generate for each input message. Note that you will be charged based on the number of generated tokens across all of the choices. Keep `n` as `1` to minimize costs.", + ) + presence_penalty: Optional[ + Union[ + Annotated[float, Field(le=2, strict=True, ge=-2)], + Annotated[int, Field(le=2, strict=True, ge=-2)], + ] + ] = Field( + default=0, + description="Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. [See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details) ", + ) + response_format: Optional[CreateChatCompletionRequestResponseFormat] = None + seed: Optional[ + Annotated[ + int, Field(le=9223372036854775807, strict=True, ge=-9223372036854775808) + ] + ] = Field( + default=None, + description="This feature is in Beta. If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same `seed` and parameters should return the same result. Determinism is not guaranteed, and you should refer to the `system_fingerprint` response parameter to monitor changes in the backend. ", + ) + stop: Optional[CreateChatCompletionRequestStop] = None + stream: Optional[StrictBool] = Field( + default=False, + description="If set, partial message deltas will be sent, like in ChatGPT. Tokens will be sent as data-only [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) as they become available, with the stream terminated by a `data: [DONE]` message. [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions). ", + ) + temperature: Optional[ + Union[ + Annotated[float, Field(le=2, strict=True, ge=0)], + Annotated[int, Field(le=2, strict=True, ge=0)], + ] + ] = Field( + default=1, + description="What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. We generally recommend altering this or `top_p` but not both. ", + ) + top_p: Optional[ + Union[ + Annotated[float, Field(le=1, strict=True, ge=0)], + Annotated[int, Field(le=1, strict=True, ge=0)], + ] + ] = Field( + default=1, + description="An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or `temperature` but not both. ", + ) + tools: Optional[List[ChatCompletionTool]] = Field( + default=None, + description="A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of functions the model may generate JSON inputs for. A max of 128 functions are supported. ", + ) + tool_choice: Optional[ChatCompletionToolChoiceOption] = None + user: Optional[StrictStr] = Field( + default=None, + description="A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](/docs/guides/safety-best-practices/end-user-ids). ", + ) + function_call: Optional[CreateChatCompletionRequestFunctionCall] = None + functions: Optional[ + Annotated[List[ChatCompletionFunctions], Field(min_length=1, max_length=128)] + ] = Field( + default=None, + description="Deprecated in favor of `tools`. A list of functions the model may generate JSON inputs for. ", + ) + __properties: ClassVar[List[str]] = [ + "messages", + "model", + "frequency_penalty", + "logit_bias", + "logprobs", + "top_logprobs", + "max_tokens", + "n", + "presence_penalty", + "response_format", + "seed", + "stop", + "stream", + "temperature", + "top_p", + "tools", + "tool_choice", + "user", + "function_call", + "functions", + ] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of CreateChatCompletionRequest from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of each item in messages (list) + _items = [] + if self.messages: + for _item in self.messages: + if _item: + _items.append(_item.to_dict()) + _dict["messages"] = _items + # override the default output from pydantic by calling `to_dict()` of model + if self.model: + _dict["model"] = self.model.to_dict() + # override the default output from pydantic by calling `to_dict()` of response_format + if self.response_format: + _dict["response_format"] = self.response_format.to_dict() + # override the default output from pydantic by calling `to_dict()` of stop + if self.stop: + _dict["stop"] = self.stop.to_dict() + # override the default output from pydantic by calling `to_dict()` of each item in tools (list) + _items = [] + if self.tools: + for _item in self.tools: + if _item: + _items.append(_item.to_dict()) + _dict["tools"] = _items + # override the default output from pydantic by calling `to_dict()` of tool_choice + if self.tool_choice: + _dict["tool_choice"] = self.tool_choice.to_dict() + # override the default output from pydantic by calling `to_dict()` of function_call + if self.function_call: + _dict["function_call"] = self.function_call.to_dict() + # override the default output from pydantic by calling `to_dict()` of each item in functions (list) + _items = [] + if self.functions: + for _item in self.functions: + if _item: + _items.append(_item.to_dict()) + _dict["functions"] = _items + # set to None if frequency_penalty (nullable) is None + # and model_fields_set contains the field + if ( + self.frequency_penalty is None + and "frequency_penalty" in self.model_fields_set + ): + _dict["frequency_penalty"] = None + + # set to None if logit_bias (nullable) is None + # and model_fields_set contains the field + if self.logit_bias is None and "logit_bias" in self.model_fields_set: + _dict["logit_bias"] = None + + # set to None if logprobs (nullable) is None + # and model_fields_set contains the field + if self.logprobs is None and "logprobs" in self.model_fields_set: + _dict["logprobs"] = None + + # set to None if top_logprobs (nullable) is None + # and model_fields_set contains the field + if self.top_logprobs is None and "top_logprobs" in self.model_fields_set: + _dict["top_logprobs"] = None + + # set to None if max_tokens (nullable) is None + # and model_fields_set contains the field + if self.max_tokens is None and "max_tokens" in self.model_fields_set: + _dict["max_tokens"] = None + + # set to None if n (nullable) is None + # and model_fields_set contains the field + if self.n is None and "n" in self.model_fields_set: + _dict["n"] = None + + # set to None if presence_penalty (nullable) is None + # and model_fields_set contains the field + if ( + self.presence_penalty is None + and "presence_penalty" in self.model_fields_set + ): + _dict["presence_penalty"] = None + + # set to None if seed (nullable) is None + # and model_fields_set contains the field + if self.seed is None and "seed" in self.model_fields_set: + _dict["seed"] = None + + # set to None if stream (nullable) is None + # and model_fields_set contains the field + if self.stream is None and "stream" in self.model_fields_set: + _dict["stream"] = None + + # set to None if temperature (nullable) is None + # and model_fields_set contains the field + if self.temperature is None and "temperature" in self.model_fields_set: + _dict["temperature"] = None + + # set to None if top_p (nullable) is None + # and model_fields_set contains the field + if self.top_p is None and "top_p" in self.model_fields_set: + _dict["top_p"] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of CreateChatCompletionRequest from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "messages": [ + ChatCompletionRequestMessage.from_dict(_item) + for _item in obj.get("messages") + ] + if obj.get("messages") is not None + else None, + "model": CreateChatCompletionRequestModel.from_dict(obj.get("model")) + if obj.get("model") is not None + else None, + "frequency_penalty": obj.get("frequency_penalty") + if obj.get("frequency_penalty") is not None + else 0, + "logit_bias": obj.get("logit_bias"), + "logprobs": obj.get("logprobs") + if obj.get("logprobs") is not None + else False, + "top_logprobs": obj.get("top_logprobs"), + "max_tokens": obj.get("max_tokens"), + "n": obj.get("n") if obj.get("n") is not None else 1, + "presence_penalty": obj.get("presence_penalty") + if obj.get("presence_penalty") is not None + else 0, + "response_format": CreateChatCompletionRequestResponseFormat.from_dict( + obj.get("response_format") + ) + if obj.get("response_format") is not None + else None, + "seed": obj.get("seed"), + "stop": CreateChatCompletionRequestStop.from_dict(obj.get("stop")) + if obj.get("stop") is not None + else None, + "stream": obj.get("stream") if obj.get("stream") is not None else False, + "temperature": obj.get("temperature") + if obj.get("temperature") is not None + else 1, + "top_p": obj.get("top_p") if obj.get("top_p") is not None else 1, + "tools": [ + ChatCompletionTool.from_dict(_item) for _item in obj.get("tools") + ] + if obj.get("tools") is not None + else None, + "tool_choice": ChatCompletionToolChoiceOption.from_dict( + obj.get("tool_choice") + ) + if obj.get("tool_choice") is not None + else None, + "user": obj.get("user"), + "function_call": CreateChatCompletionRequestFunctionCall.from_dict( + obj.get("function_call") + ) + if obj.get("function_call") is not None + else None, + "functions": [ + ChatCompletionFunctions.from_dict(_item) + for _item in obj.get("functions") + ] + if obj.get("functions") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_function_call.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_function_call.py new file mode 100644 index 00000000..4d92a7c4 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_function_call.py @@ -0,0 +1,184 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from inspect import getfullargspec +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictStr, + ValidationError, + field_validator, +) +from typing_extensions import Literal + +from openapi_server.models.chat_completion_function_call_option import ( + ChatCompletionFunctionCallOption, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + +CREATECHATCOMPLETIONREQUESTFUNCTIONCALL_ONE_OF_SCHEMAS = [ + "ChatCompletionFunctionCallOption", + "str", +] + + +class CreateChatCompletionRequestFunctionCall(BaseModel): + """ + Deprecated in favor of `tool_choice`. Controls which (if any) function is called by the model. `none` means the model will not call a function and instead generates a message. `auto` means the model can pick between generating a message or calling a function. Specifying a particular function via `{\"name\": \"my_function\"}` forces the model to call that function. `none` is the default when no functions are present. `auto` is the default if functions are present. + """ + + # data type: str + oneof_schema_1_validator: Optional[StrictStr] = Field( + default=None, + description="`none` means the model will not call a function and instead generates a message. `auto` means the model can pick between generating a message or calling a function. ", + ) + # data type: ChatCompletionFunctionCallOption + oneof_schema_2_validator: Optional[ChatCompletionFunctionCallOption] = None + actual_instance: Optional[Union[ChatCompletionFunctionCallOption, str]] = None + one_of_schemas: List[str] = Literal["ChatCompletionFunctionCallOption", "str"] + + model_config = { + "validate_assignment": True, + "protected_namespaces": (), + } + + def __init__(self, *args, **kwargs) -> None: + if args: + if len(args) > 1: + raise ValueError( + "If a position argument is used, only 1 is allowed to set `actual_instance`" + ) + if kwargs: + raise ValueError( + "If a position argument is used, keyword arguments cannot be used." + ) + super().__init__(actual_instance=args[0]) + else: + super().__init__(**kwargs) + + @field_validator("actual_instance") + def actual_instance_must_validate_oneof(cls, v): + instance = CreateChatCompletionRequestFunctionCall.model_construct() + error_messages = [] + match = 0 + # validate data type: str + try: + instance.oneof_schema_1_validator = v + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # validate data type: ChatCompletionFunctionCallOption + if not isinstance(v, ChatCompletionFunctionCallOption): + error_messages.append( + f"Error! Input type `{type(v)}` is not `ChatCompletionFunctionCallOption`" + ) + else: + match += 1 + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when setting `actual_instance` in CreateChatCompletionRequestFunctionCall with oneOf schemas: ChatCompletionFunctionCallOption, str. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when setting `actual_instance` in CreateChatCompletionRequestFunctionCall with oneOf schemas: ChatCompletionFunctionCallOption, str. Details: " + + ", ".join(error_messages) + ) + else: + return v + + @classmethod + def from_dict(cls, obj: dict) -> Self: + return cls.from_json(json.dumps(obj)) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Returns the object represented by the json string""" + instance = cls.model_construct() + error_messages = [] + match = 0 + + # deserialize data into str + try: + # validation + instance.oneof_schema_1_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.oneof_schema_1_validator + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into ChatCompletionFunctionCallOption + try: + instance.actual_instance = ChatCompletionFunctionCallOption.from_json( + json_str + ) + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when deserializing the JSON string into CreateChatCompletionRequestFunctionCall with oneOf schemas: ChatCompletionFunctionCallOption, str. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when deserializing the JSON string into CreateChatCompletionRequestFunctionCall with oneOf schemas: ChatCompletionFunctionCallOption, str. Details: " + + ", ".join(error_messages) + ) + else: + return instance + + def to_json(self) -> str: + """Returns the JSON representation of the actual instance""" + if self.actual_instance is None: + return "null" + + to_json = getattr(self.actual_instance, "to_json", None) + if callable(to_json): + return self.actual_instance.to_json() + else: + return json.dumps(self.actual_instance) + + def to_dict(self) -> Dict: + """Returns the dict representation of the actual instance""" + if self.actual_instance is None: + return None + + to_dict = getattr(self.actual_instance, "to_dict", None) + if callable(to_dict): + return self.actual_instance.to_dict() + else: + # primitive type + return self.actual_instance + + def to_str(self) -> str: + """Returns the string representation of the actual instance""" + return pprint.pformat(self.model_dump()) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_model.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_model.py new file mode 100644 index 00000000..cbff81de --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_model.py @@ -0,0 +1,161 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from inspect import getfullargspec +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictStr, + ValidationError, + field_validator, +) +from typing_extensions import Literal + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + +CREATECHATCOMPLETIONREQUESTMODEL_ANY_OF_SCHEMAS = ["str"] + + +class CreateChatCompletionRequestModel(BaseModel): + """ + ID of the model to use. See the [model endpoint compatibility](/docs/models/model-endpoint-compatibility) table for details on which models work with the Chat API. + """ + + # data type: str + anyof_schema_1_validator: Optional[StrictStr] = None + # data type: str + anyof_schema_2_validator: Optional[StrictStr] = None + if TYPE_CHECKING: + actual_instance: Optional[Union[str]] = None + else: + actual_instance: Any = None + any_of_schemas: List[str] = Literal[CREATECHATCOMPLETIONREQUESTMODEL_ANY_OF_SCHEMAS] + + model_config = { + "validate_assignment": True, + "protected_namespaces": (), + } + + def __init__(self, *args, **kwargs) -> None: + if args: + if len(args) > 1: + raise ValueError( + "If a position argument is used, only 1 is allowed to set `actual_instance`" + ) + if kwargs: + raise ValueError( + "If a position argument is used, keyword arguments cannot be used." + ) + super().__init__(actual_instance=args[0]) + else: + super().__init__(**kwargs) + + @field_validator("actual_instance") + def actual_instance_must_validate_anyof(cls, v): + instance = CreateChatCompletionRequestModel.model_construct() + error_messages = [] + # validate data type: str + try: + instance.anyof_schema_1_validator = v + return v + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # validate data type: str + try: + instance.anyof_schema_2_validator = v + return v + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + if error_messages: + # no match + raise ValueError( + "No match found when setting the actual_instance in CreateChatCompletionRequestModel with anyOf schemas: str. Details: " + + ", ".join(error_messages) + ) + else: + return v + + @classmethod + def from_dict(cls, obj: dict) -> Self: + return cls.from_json(json.dumps(obj)) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Returns the object represented by the json string""" + instance = cls.model_construct() + error_messages = [] + # deserialize data into str + try: + # validation + instance.anyof_schema_1_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.anyof_schema_1_validator + return instance + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into str + try: + # validation + instance.anyof_schema_2_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.anyof_schema_2_validator + return instance + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + + if error_messages: + # no match + raise ValueError( + "No match found when deserializing the JSON string into CreateChatCompletionRequestModel with anyOf schemas: str. Details: " + + ", ".join(error_messages) + ) + else: + return instance + + def to_json(self) -> str: + """Returns the JSON representation of the actual instance""" + if self.actual_instance is None: + return "null" + + to_json = getattr(self.actual_instance, "to_json", None) + if callable(to_json): + return self.actual_instance.to_json() + else: + return json.dumps(self.actual_instance) + + def to_dict(self) -> Dict: + """Returns the dict representation of the actual instance""" + if self.actual_instance is None: + return "null" + + to_json = getattr(self.actual_instance, "to_json", None) + if callable(to_json): + return self.actual_instance.to_dict() + else: + return json.dumps(self.actual_instance) + + def to_str(self) -> str: + """Returns the string representation of the actual instance""" + return pprint.pformat(self.model_dump()) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_response_format.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_response_format.py new file mode 100644 index 00000000..15be6ff4 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_response_format.py @@ -0,0 +1,99 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class CreateChatCompletionRequestResponseFormat(BaseModel): + """ + An object specifying the format that the model must output. Compatible with [GPT-4 Turbo](/docs/models/gpt-4-and-gpt-4-turbo) and all GPT-3.5 Turbo models newer than `gpt-3.5-turbo-1106`. Setting to `{ \"type\": \"json_object\" }` enables JSON mode, which guarantees the message the model generates is valid JSON. **Important:** when using JSON mode, you **must** also instruct the model to produce JSON yourself via a system or user message. Without this, the model may generate an unending stream of whitespace until the generation reaches the token limit, resulting in a long-running and seemingly \"stuck\" request. Also note that the message content may be partially cut off if `finish_reason=\"length\"`, which indicates the generation exceeded `max_tokens` or the conversation exceeded the max context length. + """ # noqa: E501 + + type: Optional[StrictStr] = Field( + default="text", description="Must be one of `text` or `json_object`." + ) + __properties: ClassVar[List[str]] = ["type"] + + @field_validator("type") + def type_validate_enum(cls, value): + """Validates the enum""" + if value is None: + return value + + if value not in ("text", "json_object"): + raise ValueError("must be one of enum values ('text', 'json_object')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of CreateChatCompletionRequestResponseFormat from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of CreateChatCompletionRequestResponseFormat from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + {"type": obj.get("type") if obj.get("type") is not None else "text"} + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_stop.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_stop.py new file mode 100644 index 00000000..4ec6e869 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_stop.py @@ -0,0 +1,176 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from inspect import getfullargspec +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictStr, + ValidationError, + field_validator, +) +from typing_extensions import Annotated, Literal + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + +CREATECHATCOMPLETIONREQUESTSTOP_ONE_OF_SCHEMAS = ["List[str]", "str"] + + +class CreateChatCompletionRequestStop(BaseModel): + """ + Up to 4 sequences where the API will stop generating further tokens. + """ + + # data type: str + oneof_schema_1_validator: Optional[StrictStr] = None + # data type: List[str] + oneof_schema_2_validator: Optional[ + Annotated[List[StrictStr], Field(min_length=1, max_length=4)] + ] = None + actual_instance: Optional[Union[List[str], str]] = None + one_of_schemas: List[str] = Literal["List[str]", "str"] + + model_config = { + "validate_assignment": True, + "protected_namespaces": (), + } + + def __init__(self, *args, **kwargs) -> None: + if args: + if len(args) > 1: + raise ValueError( + "If a position argument is used, only 1 is allowed to set `actual_instance`" + ) + if kwargs: + raise ValueError( + "If a position argument is used, keyword arguments cannot be used." + ) + super().__init__(actual_instance=args[0]) + else: + super().__init__(**kwargs) + + @field_validator("actual_instance") + def actual_instance_must_validate_oneof(cls, v): + instance = CreateChatCompletionRequestStop.model_construct() + error_messages = [] + match = 0 + # validate data type: str + try: + instance.oneof_schema_1_validator = v + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # validate data type: List[str] + try: + instance.oneof_schema_2_validator = v + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when setting `actual_instance` in CreateChatCompletionRequestStop with oneOf schemas: List[str], str. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when setting `actual_instance` in CreateChatCompletionRequestStop with oneOf schemas: List[str], str. Details: " + + ", ".join(error_messages) + ) + else: + return v + + @classmethod + def from_dict(cls, obj: dict) -> Self: + return cls.from_json(json.dumps(obj)) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Returns the object represented by the json string""" + instance = cls.model_construct() + error_messages = [] + match = 0 + + # deserialize data into str + try: + # validation + instance.oneof_schema_1_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.oneof_schema_1_validator + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into List[str] + try: + # validation + instance.oneof_schema_2_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.oneof_schema_2_validator + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when deserializing the JSON string into CreateChatCompletionRequestStop with oneOf schemas: List[str], str. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when deserializing the JSON string into CreateChatCompletionRequestStop with oneOf schemas: List[str], str. Details: " + + ", ".join(error_messages) + ) + else: + return instance + + def to_json(self) -> str: + """Returns the JSON representation of the actual instance""" + if self.actual_instance is None: + return "null" + + to_json = getattr(self.actual_instance, "to_json", None) + if callable(to_json): + return self.actual_instance.to_json() + else: + return json.dumps(self.actual_instance) + + def to_dict(self) -> Dict: + """Returns the dict representation of the actual instance""" + if self.actual_instance is None: + return None + + to_dict = getattr(self.actual_instance, "to_dict", None) + if callable(to_dict): + return self.actual_instance.to_dict() + else: + # primitive type + return self.actual_instance + + def to_str(self) -> str: + """Returns the string representation of the actual instance""" + return pprint.pformat(self.model_dump()) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response.py new file mode 100644 index 00000000..bc43e294 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response.py @@ -0,0 +1,147 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator + +from openapi_server.models.completion_usage import CompletionUsage +from openapi_server.models.create_chat_completion_response_choices_inner import ( + CreateChatCompletionResponseChoicesInner, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class CreateChatCompletionResponse(BaseModel): + """ + Represents a chat completion response returned by model, based on the provided input. + """ # noqa: E501 + + id: StrictStr = Field(description="A unique identifier for the chat completion.") + choices: List[CreateChatCompletionResponseChoicesInner] = Field( + description="A list of chat completion choices. Can be more than one if `n` is greater than 1." + ) + created: StrictInt = Field( + description="The Unix timestamp (in seconds) of when the chat completion was created." + ) + model: StrictStr = Field(description="The model used for the chat completion.") + system_fingerprint: Optional[StrictStr] = Field( + default=None, + description="This fingerprint represents the backend configuration that the model runs with. Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. ", + ) + object: StrictStr = Field( + description="The object type, which is always `chat.completion`." + ) + usage: Optional[CompletionUsage] = None + __properties: ClassVar[List[str]] = [ + "id", + "choices", + "created", + "model", + "system_fingerprint", + "object", + "usage", + ] + + @field_validator("object") + def object_validate_enum(cls, value): + """Validates the enum""" + if value not in ("chat.completion"): + raise ValueError("must be one of enum values ('chat.completion')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of CreateChatCompletionResponse from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of each item in choices (list) + _items = [] + if self.choices: + for _item in self.choices: + if _item: + _items.append(_item.to_dict()) + _dict["choices"] = _items + # override the default output from pydantic by calling `to_dict()` of usage + if self.usage: + _dict["usage"] = self.usage.to_dict() + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of CreateChatCompletionResponse from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "id": obj.get("id"), + "choices": [ + CreateChatCompletionResponseChoicesInner.from_dict(_item) + for _item in obj.get("choices") + ] + if obj.get("choices") is not None + else None, + "created": obj.get("created"), + "model": obj.get("model"), + "system_fingerprint": obj.get("system_fingerprint"), + "object": obj.get("object"), + "usage": CompletionUsage.from_dict(obj.get("usage")) + if obj.get("usage") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner.py new file mode 100644 index 00000000..fd1e7ea9 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner.py @@ -0,0 +1,143 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator + +from openapi_server.models.chat_completion_response_message import ( + ChatCompletionResponseMessage, +) +from openapi_server.models.create_chat_completion_response_choices_inner_logprobs import ( + CreateChatCompletionResponseChoicesInnerLogprobs, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class CreateChatCompletionResponseChoicesInner(BaseModel): + """ + CreateChatCompletionResponseChoicesInner + """ # noqa: E501 + + finish_reason: StrictStr = Field( + description="The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, `length` if the maximum number of tokens specified in the request was reached, `content_filter` if content was omitted due to a flag from our content filters, `tool_calls` if the model called a tool, or `function_call` (deprecated) if the model called a function. " + ) + index: StrictInt = Field( + description="The index of the choice in the list of choices." + ) + message: ChatCompletionResponseMessage + logprobs: Optional[CreateChatCompletionResponseChoicesInnerLogprobs] + __properties: ClassVar[List[str]] = [ + "finish_reason", + "index", + "message", + "logprobs", + ] + + @field_validator("finish_reason") + def finish_reason_validate_enum(cls, value): + """Validates the enum""" + if value not in ( + "stop", + "length", + "tool_calls", + "content_filter", + "function_call", + ): + raise ValueError( + "must be one of enum values ('stop', 'length', 'tool_calls', 'content_filter', 'function_call')" + ) + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of CreateChatCompletionResponseChoicesInner from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of message + if self.message: + _dict["message"] = self.message.to_dict() + # override the default output from pydantic by calling `to_dict()` of logprobs + if self.logprobs: + _dict["logprobs"] = self.logprobs.to_dict() + # set to None if logprobs (nullable) is None + # and model_fields_set contains the field + if self.logprobs is None and "logprobs" in self.model_fields_set: + _dict["logprobs"] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of CreateChatCompletionResponseChoicesInner from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "finish_reason": obj.get("finish_reason"), + "index": obj.get("index"), + "message": ChatCompletionResponseMessage.from_dict(obj.get("message")) + if obj.get("message") is not None + else None, + "logprobs": CreateChatCompletionResponseChoicesInnerLogprobs.from_dict( + obj.get("logprobs") + ) + if obj.get("logprobs") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner_logprobs.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner_logprobs.py new file mode 100644 index 00000000..86f46094 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner_logprobs.py @@ -0,0 +1,112 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field + +from openapi_server.models.chat_completion_token_logprob import ( + ChatCompletionTokenLogprob, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class CreateChatCompletionResponseChoicesInnerLogprobs(BaseModel): + """ + Log probability information for the choice. + """ # noqa: E501 + + content: Optional[List[ChatCompletionTokenLogprob]] = Field( + description="A list of message content tokens with log probability information." + ) + __properties: ClassVar[List[str]] = ["content"] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of CreateChatCompletionResponseChoicesInnerLogprobs from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of each item in content (list) + _items = [] + if self.content: + for _item in self.content: + if _item: + _items.append(_item.to_dict()) + _dict["content"] = _items + # set to None if content (nullable) is None + # and model_fields_set contains the field + if self.content is None and "content" in self.model_fields_set: + _dict["content"] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of CreateChatCompletionResponseChoicesInnerLogprobs from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "content": [ + ChatCompletionTokenLogprob.from_dict(_item) + for _item in obj.get("content") + ] + if obj.get("content") is not None + else None + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_stream_response.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_stream_response.py new file mode 100644 index 00000000..bd464c73 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_stream_response.py @@ -0,0 +1,140 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator + +from openapi_server.models.create_chat_completion_stream_response_choices_inner import ( + CreateChatCompletionStreamResponseChoicesInner, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class CreateChatCompletionStreamResponse(BaseModel): + """ + Represents a streamed chunk of a chat completion response returned by model, based on the provided input. + """ # noqa: E501 + + id: StrictStr = Field( + description="A unique identifier for the chat completion. Each chunk has the same ID." + ) + choices: List[CreateChatCompletionStreamResponseChoicesInner] = Field( + description="A list of chat completion choices. Can be more than one if `n` is greater than 1." + ) + created: StrictInt = Field( + description="The Unix timestamp (in seconds) of when the chat completion was created. Each chunk has the same timestamp." + ) + model: StrictStr = Field(description="The model to generate the completion.") + system_fingerprint: Optional[StrictStr] = Field( + default=None, + description="This fingerprint represents the backend configuration that the model runs with. Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. ", + ) + object: StrictStr = Field( + description="The object type, which is always `chat.completion.chunk`." + ) + __properties: ClassVar[List[str]] = [ + "id", + "choices", + "created", + "model", + "system_fingerprint", + "object", + ] + + @field_validator("object") + def object_validate_enum(cls, value): + """Validates the enum""" + if value not in ("chat.completion.chunk"): + raise ValueError("must be one of enum values ('chat.completion.chunk')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of CreateChatCompletionStreamResponse from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of each item in choices (list) + _items = [] + if self.choices: + for _item in self.choices: + if _item: + _items.append(_item.to_dict()) + _dict["choices"] = _items + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of CreateChatCompletionStreamResponse from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "id": obj.get("id"), + "choices": [ + CreateChatCompletionStreamResponseChoicesInner.from_dict(_item) + for _item in obj.get("choices") + ] + if obj.get("choices") is not None + else None, + "created": obj.get("created"), + "model": obj.get("model"), + "system_fingerprint": obj.get("system_fingerprint"), + "object": obj.get("object"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_stream_response_choices_inner.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_stream_response_choices_inner.py new file mode 100644 index 00000000..910caa2f --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_stream_response_choices_inner.py @@ -0,0 +1,146 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator + +from openapi_server.models.chat_completion_stream_response_delta import ( + ChatCompletionStreamResponseDelta, +) +from openapi_server.models.create_chat_completion_response_choices_inner_logprobs import ( + CreateChatCompletionResponseChoicesInnerLogprobs, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class CreateChatCompletionStreamResponseChoicesInner(BaseModel): + """ + CreateChatCompletionStreamResponseChoicesInner + """ # noqa: E501 + + delta: ChatCompletionStreamResponseDelta + logprobs: Optional[CreateChatCompletionResponseChoicesInnerLogprobs] = None + finish_reason: Optional[StrictStr] = Field( + description="The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, `length` if the maximum number of tokens specified in the request was reached, `content_filter` if content was omitted due to a flag from our content filters, `tool_calls` if the model called a tool, or `function_call` (deprecated) if the model called a function. " + ) + index: StrictInt = Field( + description="The index of the choice in the list of choices." + ) + __properties: ClassVar[List[str]] = ["delta", "logprobs", "finish_reason", "index"] + + @field_validator("finish_reason") + def finish_reason_validate_enum(cls, value): + """Validates the enum""" + if value is None: + return value + + if value not in ( + "stop", + "length", + "tool_calls", + "content_filter", + "function_call", + ): + raise ValueError( + "must be one of enum values ('stop', 'length', 'tool_calls', 'content_filter', 'function_call')" + ) + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of CreateChatCompletionStreamResponseChoicesInner from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of delta + if self.delta: + _dict["delta"] = self.delta.to_dict() + # override the default output from pydantic by calling `to_dict()` of logprobs + if self.logprobs: + _dict["logprobs"] = self.logprobs.to_dict() + # set to None if logprobs (nullable) is None + # and model_fields_set contains the field + if self.logprobs is None and "logprobs" in self.model_fields_set: + _dict["logprobs"] = None + + # set to None if finish_reason (nullable) is None + # and model_fields_set contains the field + if self.finish_reason is None and "finish_reason" in self.model_fields_set: + _dict["finish_reason"] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of CreateChatCompletionStreamResponseChoicesInner from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "delta": ChatCompletionStreamResponseDelta.from_dict(obj.get("delta")) + if obj.get("delta") is not None + else None, + "logprobs": CreateChatCompletionResponseChoicesInnerLogprobs.from_dict( + obj.get("logprobs") + ) + if obj.get("logprobs") is not None + else None, + "finish_reason": obj.get("finish_reason"), + "index": obj.get("index"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request.py new file mode 100644 index 00000000..bfcc91e0 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request.py @@ -0,0 +1,320 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, StrictStr +from typing_extensions import Annotated + +from openapi_server.models.create_completion_request_model import ( + CreateCompletionRequestModel, +) +from openapi_server.models.create_completion_request_prompt import ( + CreateCompletionRequestPrompt, +) +from openapi_server.models.create_completion_request_stop import ( + CreateCompletionRequestStop, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class CreateCompletionRequest(BaseModel): + """ + CreateCompletionRequest + """ # noqa: E501 + + model: str + prompt: str + best_of: Optional[Annotated[int, Field(le=20, strict=True, ge=0)]] = Field( + default=1, + description='Generates `best_of` completions server-side and returns the "best" (the one with the highest log probability per token). Results cannot be streamed. When used with `n`, `best_of` controls the number of candidate completions and `n` specifies how many to return – `best_of` must be greater than `n`. **Note:** Because this parameter generates many completions, it can quickly consume your token quota. Use carefully and ensure that you have reasonable settings for `max_tokens` and `stop`. ', + ) + echo: Optional[StrictBool] = Field( + default=False, description="Echo back the prompt in addition to the completion " + ) + frequency_penalty: Optional[ + Union[ + Annotated[float, Field(le=2, strict=True, ge=-2)], + Annotated[int, Field(le=2, strict=True, ge=-2)], + ] + ] = Field( + default=0, + description="Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. [See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details) ", + ) + logit_bias: Optional[Dict[str, StrictInt]] = Field( + default=None, + description='Modify the likelihood of specified tokens appearing in the completion. Accepts a JSON object that maps tokens (specified by their token ID in the GPT tokenizer) to an associated bias value from -100 to 100. You can use this [tokenizer tool](/tokenizer?view=bpe) to convert text to token IDs. Mathematically, the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should result in a ban or exclusive selection of the relevant token. As an example, you can pass `{"50256": -100}` to prevent the <|endoftext|> token from being generated. ', + ) + logprobs: Optional[Annotated[int, Field(le=5, strict=True, ge=0)]] = Field( + default=None, + description="Include the log probabilities on the `logprobs` most likely output tokens, as well the chosen tokens. For example, if `logprobs` is 5, the API will return a list of the 5 most likely tokens. The API will always return the `logprob` of the sampled token, so there may be up to `logprobs+1` elements in the response. The maximum value for `logprobs` is 5. ", + ) + max_tokens: Optional[Annotated[int, Field(strict=True, ge=0)]] = Field( + default=16, + description="The maximum number of [tokens](/tokenizer) that can be generated in the completion. The token count of your prompt plus `max_tokens` cannot exceed the model's context length. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens. ", + ) + n: Optional[Annotated[int, Field(le=128, strict=True, ge=1)]] = Field( + default=1, + description="How many completions to generate for each prompt. **Note:** Because this parameter generates many completions, it can quickly consume your token quota. Use carefully and ensure that you have reasonable settings for `max_tokens` and `stop`. ", + ) + presence_penalty: Optional[ + Union[ + Annotated[float, Field(le=2, strict=True, ge=-2)], + Annotated[int, Field(le=2, strict=True, ge=-2)], + ] + ] = Field( + default=0, + description="Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. [See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details) ", + ) + seed: Optional[ + Annotated[ + int, Field(le=9223372036854775807, strict=True, ge=-9223372036854775808) + ] + ] = Field( + default=None, + description="If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same `seed` and parameters should return the same result. Determinism is not guaranteed, and you should refer to the `system_fingerprint` response parameter to monitor changes in the backend. ", + ) + stop: Optional[CreateCompletionRequestStop] = None + stream: Optional[StrictBool] = Field( + default=False, + description="Whether to stream back partial progress. If set, tokens will be sent as data-only [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) as they become available, with the stream terminated by a `data: [DONE]` message. [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions). ", + ) + suffix: Optional[StrictStr] = Field( + default=None, + description="The suffix that comes after a completion of inserted text. This parameter is only supported for `gpt-3.5-turbo-instruct`. ", + ) + temperature: Optional[ + Union[ + Annotated[float, Field(le=2, strict=True, ge=0)], + Annotated[int, Field(le=2, strict=True, ge=0)], + ] + ] = Field( + default=1, + description="What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. We generally recommend altering this or `top_p` but not both. ", + ) + top_p: Optional[ + Union[ + Annotated[float, Field(le=1, strict=True, ge=0)], + Annotated[int, Field(le=1, strict=True, ge=0)], + ] + ] = Field( + default=1, + description="An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or `temperature` but not both. ", + ) + user: Optional[StrictStr] = Field( + default=None, + description="A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](/docs/guides/safety-best-practices/end-user-ids). ", + ) + __properties: ClassVar[List[str]] = [ + "model", + "prompt", + "best_of", + "echo", + "frequency_penalty", + "logit_bias", + "logprobs", + "max_tokens", + "n", + "presence_penalty", + "seed", + "stop", + "stream", + "suffix", + "temperature", + "top_p", + "user", + ] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of CreateCompletionRequest from a JSON string""" + print("here!") + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of model + if self.model: + _dict["model"] = self.model.to_dict() + # override the default output from pydantic by calling `to_dict()` of prompt + if self.prompt: + _dict["prompt"] = self.prompt.to_dict() + # override the default output from pydantic by calling `to_dict()` of stop + if self.stop: + _dict["stop"] = self.stop.to_dict() + # set to None if prompt (nullable) is None + # and model_fields_set contains the field + if self.prompt is None and "prompt" in self.model_fields_set: + _dict["prompt"] = None + + # set to None if best_of (nullable) is None + # and model_fields_set contains the field + if self.best_of is None and "best_of" in self.model_fields_set: + _dict["best_of"] = None + + # set to None if echo (nullable) is None + # and model_fields_set contains the field + if self.echo is None and "echo" in self.model_fields_set: + _dict["echo"] = None + + # set to None if frequency_penalty (nullable) is None + # and model_fields_set contains the field + if ( + self.frequency_penalty is None + and "frequency_penalty" in self.model_fields_set + ): + _dict["frequency_penalty"] = None + + # set to None if logit_bias (nullable) is None + # and model_fields_set contains the field + if self.logit_bias is None and "logit_bias" in self.model_fields_set: + _dict["logit_bias"] = None + + # set to None if logprobs (nullable) is None + # and model_fields_set contains the field + if self.logprobs is None and "logprobs" in self.model_fields_set: + _dict["logprobs"] = None + + # set to None if max_tokens (nullable) is None + # and model_fields_set contains the field + if self.max_tokens is None and "max_tokens" in self.model_fields_set: + _dict["max_tokens"] = None + + # set to None if n (nullable) is None + # and model_fields_set contains the field + if self.n is None and "n" in self.model_fields_set: + _dict["n"] = None + + # set to None if presence_penalty (nullable) is None + # and model_fields_set contains the field + if ( + self.presence_penalty is None + and "presence_penalty" in self.model_fields_set + ): + _dict["presence_penalty"] = None + + # set to None if seed (nullable) is None + # and model_fields_set contains the field + if self.seed is None and "seed" in self.model_fields_set: + _dict["seed"] = None + + # set to None if stop (nullable) is None + # and model_fields_set contains the field + if self.stop is None and "stop" in self.model_fields_set: + _dict["stop"] = None + + # set to None if stream (nullable) is None + # and model_fields_set contains the field + if self.stream is None and "stream" in self.model_fields_set: + _dict["stream"] = None + + # set to None if suffix (nullable) is None + # and model_fields_set contains the field + if self.suffix is None and "suffix" in self.model_fields_set: + _dict["suffix"] = None + + # set to None if temperature (nullable) is None + # and model_fields_set contains the field + if self.temperature is None and "temperature" in self.model_fields_set: + _dict["temperature"] = None + + # set to None if top_p (nullable) is None + # and model_fields_set contains the field + if self.top_p is None and "top_p" in self.model_fields_set: + _dict["top_p"] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of CreateCompletionRequest from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "model": CreateCompletionRequestModel.from_dict(obj.get("model")) + if obj.get("model") is not None + else None, + "prompt": CreateCompletionRequestPrompt.from_dict(obj.get("prompt")) + if obj.get("prompt") is not None + else None, + "best_of": obj.get("best_of") if obj.get("best_of") is not None else 1, + "echo": obj.get("echo") if obj.get("echo") is not None else False, + "frequency_penalty": obj.get("frequency_penalty") + if obj.get("frequency_penalty") is not None + else 0, + "logit_bias": obj.get("logit_bias"), + "logprobs": obj.get("logprobs"), + "max_tokens": obj.get("max_tokens") + if obj.get("max_tokens") is not None + else 16, + "n": obj.get("n") if obj.get("n") is not None else 1, + "presence_penalty": obj.get("presence_penalty") + if obj.get("presence_penalty") is not None + else 0, + "seed": obj.get("seed"), + "stop": CreateCompletionRequestStop.from_dict(obj.get("stop")) + if obj.get("stop") is not None + else None, + "stream": obj.get("stream") if obj.get("stream") is not None else False, + "suffix": obj.get("suffix"), + "temperature": obj.get("temperature") + if obj.get("temperature") is not None + else 1, + "top_p": obj.get("top_p") if obj.get("top_p") is not None else 1, + "user": obj.get("user"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_model.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_model.py new file mode 100644 index 00000000..2955400b --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_model.py @@ -0,0 +1,161 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from inspect import getfullargspec +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictStr, + ValidationError, + field_validator, +) +from typing_extensions import Literal + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + +CREATECOMPLETIONREQUESTMODEL_ANY_OF_SCHEMAS = ["str"] + + +class CreateCompletionRequestModel(BaseModel): + """ + ID of the model to use. You can use the [List models](/docs/api-reference/models/list) API to see all of your available models, or see our [Model overview](/docs/models/overview) for descriptions of them. + """ + + # data type: str + anyof_schema_1_validator: Optional[StrictStr] = None + # data type: str + anyof_schema_2_validator: Optional[StrictStr] = None + if TYPE_CHECKING: + actual_instance: Optional[Union[str]] = None + else: + actual_instance: Any = None + any_of_schemas: List[str] = Literal[CREATECOMPLETIONREQUESTMODEL_ANY_OF_SCHEMAS] + + model_config = { + "validate_assignment": True, + "protected_namespaces": (), + } + + def __init__(self, *args, **kwargs) -> None: + if args: + if len(args) > 1: + raise ValueError( + "If a position argument is used, only 1 is allowed to set `actual_instance`" + ) + if kwargs: + raise ValueError( + "If a position argument is used, keyword arguments cannot be used." + ) + super().__init__(actual_instance=args[0]) + else: + super().__init__(**kwargs) + + @field_validator("actual_instance") + def actual_instance_must_validate_anyof(cls, v): + instance = CreateCompletionRequestModel.model_construct() + error_messages = [] + # validate data type: str + try: + instance.anyof_schema_1_validator = v + return v + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # validate data type: str + try: + instance.anyof_schema_2_validator = v + return v + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + if error_messages: + # no match + raise ValueError( + "No match found when setting the actual_instance in CreateCompletionRequestModel with anyOf schemas: str. Details: " + + ", ".join(error_messages) + ) + else: + return v + + @classmethod + def from_dict(cls, obj: dict) -> Self: + return cls.from_json(json.dumps(obj)) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Returns the object represented by the json string""" + instance = cls.model_construct() + error_messages = [] + # deserialize data into str + try: + # validation + instance.anyof_schema_1_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.anyof_schema_1_validator + return instance + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into str + try: + # validation + instance.anyof_schema_2_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.anyof_schema_2_validator + return instance + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + + if error_messages: + # no match + raise ValueError( + "No match found when deserializing the JSON string into CreateCompletionRequestModel with anyOf schemas: str. Details: " + + ", ".join(error_messages) + ) + else: + return instance + + def to_json(self) -> str: + """Returns the JSON representation of the actual instance""" + if self.actual_instance is None: + return "null" + + to_json = getattr(self.actual_instance, "to_json", None) + if callable(to_json): + return self.actual_instance.to_json() + else: + return json.dumps(self.actual_instance) + + def to_dict(self) -> Dict: + """Returns the dict representation of the actual instance""" + if self.actual_instance is None: + return "null" + + to_json = getattr(self.actual_instance, "to_json", None) + if callable(to_json): + return self.actual_instance.to_dict() + else: + return json.dumps(self.actual_instance) + + def to_str(self) -> str: + """Returns the string representation of the actual instance""" + return pprint.pformat(self.model_dump()) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_prompt.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_prompt.py new file mode 100644 index 00000000..bbb1a789 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_prompt.py @@ -0,0 +1,230 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from inspect import getfullargspec +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictInt, + StrictStr, + ValidationError, + field_validator, +) +from typing_extensions import Annotated, Literal + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + +CREATECOMPLETIONREQUESTPROMPT_ONE_OF_SCHEMAS = [ + "List[List[int]]", + "List[int]", + "List[str]", + "str", +] + + +class CreateCompletionRequestPrompt(BaseModel): + """ + The prompt(s) to generate completions for, encoded as a string, array of strings, array of tokens, or array of token arrays. Note that <|endoftext|> is the document separator that the model sees during training, so if a prompt is not specified the model will generate as if from the beginning of a new document. + """ + + # data type: str + oneof_schema_1_validator: Optional[StrictStr] = "" + # data type: List[str] + oneof_schema_2_validator: Optional[List[StrictStr]] = None + # data type: List[int] + oneof_schema_3_validator: Optional[ + Annotated[List[StrictInt], Field(min_length=1)] + ] = None + # data type: List[List[int]] + oneof_schema_4_validator: Optional[ + Annotated[ + List[Annotated[List[StrictInt], Field(min_length=1)]], Field(min_length=1) + ] + ] = None + actual_instance: Optional[Union[List[List[int]], List[int], List[str], str]] = None + one_of_schemas: List[str] = Literal[ + "List[List[int]]", "List[int]", "List[str]", "str" + ] + + model_config = { + "validate_assignment": True, + "protected_namespaces": (), + } + + def __init__(self, *args, **kwargs) -> None: + if args: + if len(args) > 1: + raise ValueError( + "If a position argument is used, only 1 is allowed to set `actual_instance`" + ) + if kwargs: + raise ValueError( + "If a position argument is used, keyword arguments cannot be used." + ) + super().__init__(actual_instance=args[0]) + else: + super().__init__(**kwargs) + + @field_validator("actual_instance") + def actual_instance_must_validate_oneof(cls, v): + if v is None: + return v + + instance = CreateCompletionRequestPrompt.model_construct() + error_messages = [] + match = 0 + # validate data type: str + try: + instance.oneof_schema_1_validator = v + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # validate data type: List[str] + try: + instance.oneof_schema_2_validator = v + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # validate data type: List[int] + try: + instance.oneof_schema_3_validator = v + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # validate data type: List[List[int]] + try: + instance.oneof_schema_4_validator = v + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when setting `actual_instance` in CreateCompletionRequestPrompt with oneOf schemas: List[List[int]], List[int], List[str], str. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when setting `actual_instance` in CreateCompletionRequestPrompt with oneOf schemas: List[List[int]], List[int], List[str], str. Details: " + + ", ".join(error_messages) + ) + else: + return v + + @classmethod + def from_dict(cls, obj: dict) -> Self: + return cls.from_json(json.dumps(obj)) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Returns the object represented by the json string""" + + print("here!") + instance = cls.model_construct() + if json_str is None: + return instance + + error_messages = [] + match = 0 + + # deserialize data into str + try: + # validation + instance.oneof_schema_1_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.oneof_schema_1_validator + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into List[str] + try: + # validation + instance.oneof_schema_2_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.oneof_schema_2_validator + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into List[int] + try: + # validation + instance.oneof_schema_3_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.oneof_schema_3_validator + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into List[List[int]] + try: + # validation + instance.oneof_schema_4_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.oneof_schema_4_validator + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when deserializing the JSON string into CreateCompletionRequestPrompt with oneOf schemas: List[List[int]], List[int], List[str], str. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when deserializing the JSON string into CreateCompletionRequestPrompt with oneOf schemas: List[List[int]], List[int], List[str], str. Details: " + + ", ".join(error_messages) + ) + else: + return instance + + def to_json(self) -> str: + """Returns the JSON representation of the actual instance""" + if self.actual_instance is None: + return "null" + + to_json = getattr(self.actual_instance, "to_json", None) + if callable(to_json): + return self.actual_instance.to_json() + else: + return json.dumps(self.actual_instance) + + def to_dict(self) -> Dict: + """Returns the dict representation of the actual instance""" + if self.actual_instance is None: + return None + + to_dict = getattr(self.actual_instance, "to_dict", None) + if callable(to_dict): + return self.actual_instance.to_dict() + else: + # primitive type + return self.actual_instance + + def to_str(self) -> str: + """Returns the string representation of the actual instance""" + return pprint.pformat(self.model_dump()) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_stop.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_stop.py new file mode 100644 index 00000000..722a4e51 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_stop.py @@ -0,0 +1,182 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from inspect import getfullargspec +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictStr, + ValidationError, + field_validator, +) +from typing_extensions import Annotated, Literal + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + +CREATECOMPLETIONREQUESTSTOP_ONE_OF_SCHEMAS = ["List[str]", "str"] + + +class CreateCompletionRequestStop(BaseModel): + """ + Up to 4 sequences where the API will stop generating further tokens. The returned text will not contain the stop sequence. + """ + + # data type: str + oneof_schema_1_validator: Optional[StrictStr] = "<|endoftext|>" + # data type: List[str] + oneof_schema_2_validator: Optional[ + Annotated[List[StrictStr], Field(min_length=1, max_length=4)] + ] = None + actual_instance: Optional[Union[List[str], str]] = None + one_of_schemas: List[str] = Literal["List[str]", "str"] + + model_config = { + "validate_assignment": True, + "protected_namespaces": (), + } + + def __init__(self, *args, **kwargs) -> None: + if args: + if len(args) > 1: + raise ValueError( + "If a position argument is used, only 1 is allowed to set `actual_instance`" + ) + if kwargs: + raise ValueError( + "If a position argument is used, keyword arguments cannot be used." + ) + super().__init__(actual_instance=args[0]) + else: + super().__init__(**kwargs) + + @field_validator("actual_instance") + def actual_instance_must_validate_oneof(cls, v): + if v is None: + return v + + instance = CreateCompletionRequestStop.model_construct() + error_messages = [] + match = 0 + # validate data type: str + try: + instance.oneof_schema_1_validator = v + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # validate data type: List[str] + try: + instance.oneof_schema_2_validator = v + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when setting `actual_instance` in CreateCompletionRequestStop with oneOf schemas: List[str], str. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when setting `actual_instance` in CreateCompletionRequestStop with oneOf schemas: List[str], str. Details: " + + ", ".join(error_messages) + ) + else: + return v + + @classmethod + def from_dict(cls, obj: dict) -> Self: + return cls.from_json(json.dumps(obj)) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Returns the object represented by the json string""" + instance = cls.model_construct() + if json_str is None: + return instance + + error_messages = [] + match = 0 + + # deserialize data into str + try: + # validation + instance.oneof_schema_1_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.oneof_schema_1_validator + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into List[str] + try: + # validation + instance.oneof_schema_2_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.oneof_schema_2_validator + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + + if match > 1: + # more than 1 match + raise ValueError( + "Multiple matches found when deserializing the JSON string into CreateCompletionRequestStop with oneOf schemas: List[str], str. Details: " + + ", ".join(error_messages) + ) + elif match == 0: + # no match + raise ValueError( + "No match found when deserializing the JSON string into CreateCompletionRequestStop with oneOf schemas: List[str], str. Details: " + + ", ".join(error_messages) + ) + else: + return instance + + def to_json(self) -> str: + """Returns the JSON representation of the actual instance""" + if self.actual_instance is None: + return "null" + + to_json = getattr(self.actual_instance, "to_json", None) + if callable(to_json): + return self.actual_instance.to_json() + else: + return json.dumps(self.actual_instance) + + def to_dict(self) -> Dict: + """Returns the dict representation of the actual instance""" + if self.actual_instance is None: + return None + + to_dict = getattr(self.actual_instance, "to_dict", None) + if callable(to_dict): + return self.actual_instance.to_dict() + else: + # primitive type + return self.actual_instance + + def to_str(self) -> str: + """Returns the string representation of the actual instance""" + return pprint.pformat(self.model_dump()) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response.py new file mode 100644 index 00000000..84d00be6 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response.py @@ -0,0 +1,147 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator + +from openapi_server.models.completion_usage import CompletionUsage +from openapi_server.models.create_completion_response_choices_inner import ( + CreateCompletionResponseChoicesInner, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class CreateCompletionResponse(BaseModel): + """ + Represents a completion response from the API. Note: both the streamed and non-streamed response objects share the same shape (unlike the chat endpoint). + """ # noqa: E501 + + id: StrictStr = Field(description="A unique identifier for the completion.") + choices: List[CreateCompletionResponseChoicesInner] = Field( + description="The list of completion choices the model generated for the input prompt." + ) + created: StrictInt = Field( + description="The Unix timestamp (in seconds) of when the completion was created." + ) + model: StrictStr = Field(description="The model used for completion.") + system_fingerprint: Optional[StrictStr] = Field( + default=None, + description="This fingerprint represents the backend configuration that the model runs with. Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. ", + ) + object: StrictStr = Field( + description='The object type, which is always "text_completion"' + ) + usage: Optional[CompletionUsage] = None + __properties: ClassVar[List[str]] = [ + "id", + "choices", + "created", + "model", + "system_fingerprint", + "object", + "usage", + ] + + @field_validator("object") + def object_validate_enum(cls, value): + """Validates the enum""" + if value not in ("text_completion"): + raise ValueError("must be one of enum values ('text_completion')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of CreateCompletionResponse from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of each item in choices (list) + _items = [] + if self.choices: + for _item in self.choices: + if _item: + _items.append(_item.to_dict()) + _dict["choices"] = _items + # override the default output from pydantic by calling `to_dict()` of usage + if self.usage: + _dict["usage"] = self.usage.to_dict() + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of CreateCompletionResponse from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "id": obj.get("id"), + "choices": [ + CreateCompletionResponseChoicesInner.from_dict(_item) + for _item in obj.get("choices") + ] + if obj.get("choices") is not None + else None, + "created": obj.get("created"), + "model": obj.get("model"), + "system_fingerprint": obj.get("system_fingerprint"), + "object": obj.get("object"), + "usage": CompletionUsage.from_dict(obj.get("usage")) + if obj.get("usage") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response_choices_inner.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response_choices_inner.py new file mode 100644 index 00000000..482dfa8e --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response_choices_inner.py @@ -0,0 +1,122 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator + +from openapi_server.models.create_completion_response_choices_inner_logprobs import ( + CreateCompletionResponseChoicesInnerLogprobs, +) + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class CreateCompletionResponseChoicesInner(BaseModel): + """ + CreateCompletionResponseChoicesInner + """ # noqa: E501 + + finish_reason: StrictStr = Field( + description="The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, `length` if the maximum number of tokens specified in the request was reached, or `content_filter` if content was omitted due to a flag from our content filters. " + ) + index: StrictInt + logprobs: Optional[CreateCompletionResponseChoicesInnerLogprobs] + text: StrictStr + __properties: ClassVar[List[str]] = ["finish_reason", "index", "logprobs", "text"] + + @field_validator("finish_reason") + def finish_reason_validate_enum(cls, value): + """Validates the enum""" + if value not in ("stop", "length", "content_filter"): + raise ValueError( + "must be one of enum values ('stop', 'length', 'content_filter')" + ) + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of CreateCompletionResponseChoicesInner from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of logprobs + if self.logprobs: + _dict["logprobs"] = self.logprobs.to_dict() + # set to None if logprobs (nullable) is None + # and model_fields_set contains the field + if self.logprobs is None and "logprobs" in self.model_fields_set: + _dict["logprobs"] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of CreateCompletionResponseChoicesInner from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "finish_reason": obj.get("finish_reason"), + "index": obj.get("index"), + "logprobs": CreateCompletionResponseChoicesInnerLogprobs.from_dict( + obj.get("logprobs") + ) + if obj.get("logprobs") is not None + else None, + "text": obj.get("text"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response_choices_inner_logprobs.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response_choices_inner_logprobs.py new file mode 100644 index 00000000..bd9c3507 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response_choices_inner_logprobs.py @@ -0,0 +1,100 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional, Union + +from pydantic import BaseModel, ConfigDict, StrictFloat, StrictInt, StrictStr + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class CreateCompletionResponseChoicesInnerLogprobs(BaseModel): + """ + CreateCompletionResponseChoicesInnerLogprobs + """ # noqa: E501 + + text_offset: Optional[List[StrictInt]] = None + token_logprobs: Optional[List[Union[StrictFloat, StrictInt]]] = None + tokens: Optional[List[StrictStr]] = None + top_logprobs: Optional[List[Dict[str, Union[StrictFloat, StrictInt]]]] = None + __properties: ClassVar[List[str]] = [ + "text_offset", + "token_logprobs", + "tokens", + "top_logprobs", + ] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of CreateCompletionResponseChoicesInnerLogprobs from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of CreateCompletionResponseChoicesInnerLogprobs from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "text_offset": obj.get("text_offset"), + "token_logprobs": obj.get("token_logprobs"), + "tokens": obj.get("tokens"), + "top_logprobs": obj.get("top_logprobs"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/delete_model_response.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/delete_model_response.py new file mode 100644 index 00000000..dc5e0226 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/delete_model_response.py @@ -0,0 +1,93 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, StrictBool, StrictStr + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class DeleteModelResponse(BaseModel): + """ + DeleteModelResponse + """ # noqa: E501 + + id: StrictStr + deleted: StrictBool + object: StrictStr + __properties: ClassVar[List[str]] = ["id", "deleted", "object"] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of DeleteModelResponse from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of DeleteModelResponse from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "id": obj.get("id"), + "deleted": obj.get("deleted"), + "object": obj.get("object"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/done_event.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/done_event.py new file mode 100644 index 00000000..655dbf1b --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/done_event.py @@ -0,0 +1,100 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, StrictStr, field_validator + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class DoneEvent(BaseModel): + """ + Occurs when a stream ends. + """ # noqa: E501 + + event: StrictStr + data: StrictStr + __properties: ClassVar[List[str]] = ["event", "data"] + + @field_validator("event") + def event_validate_enum(cls, value): + """Validates the enum""" + if value not in ("done"): + raise ValueError("must be one of enum values ('done')") + return value + + @field_validator("data") + def data_validate_enum(cls, value): + """Validates the enum""" + if value not in ("[DONE]"): + raise ValueError("must be one of enum values ('[DONE]')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of DoneEvent from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of DoneEvent from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({"event": obj.get("event"), "data": obj.get("data")}) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error.py new file mode 100644 index 00000000..aa9b9caa --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error.py @@ -0,0 +1,105 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, StrictStr + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class Error(BaseModel): + """ + Error + """ # noqa: E501 + + code: Optional[StrictStr] + message: StrictStr + param: Optional[StrictStr] + type: StrictStr + __properties: ClassVar[List[str]] = ["code", "message", "param", "type"] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of Error from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # set to None if code (nullable) is None + # and model_fields_set contains the field + if self.code is None and "code" in self.model_fields_set: + _dict["code"] = None + + # set to None if param (nullable) is None + # and model_fields_set contains the field + if self.param is None and "param" in self.model_fields_set: + _dict["param"] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of Error from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "code": obj.get("code"), + "message": obj.get("message"), + "param": obj.get("param"), + "type": obj.get("type"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error_event.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error_event.py new file mode 100644 index 00000000..d9ea8e1c --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error_event.py @@ -0,0 +1,105 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, StrictStr, field_validator + +from openapi_server.models.error import Error + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ErrorEvent(BaseModel): + """ + Occurs when an [error](/docs/guides/error-codes/api-errors) occurs. This can happen due to an internal server error or a timeout. + """ # noqa: E501 + + event: StrictStr + data: Error + __properties: ClassVar[List[str]] = ["event", "data"] + + @field_validator("event") + def event_validate_enum(cls, value): + """Validates the enum""" + if value not in ("error"): + raise ValueError("must be one of enum values ('error')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ErrorEvent from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of data + if self.data: + _dict["data"] = self.data.to_dict() + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ErrorEvent from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "event": obj.get("event"), + "data": Error.from_dict(obj.get("data")) + if obj.get("data") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error_response.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error_response.py new file mode 100644 index 00000000..f265cd28 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error_response.py @@ -0,0 +1,96 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict + +from openapi_server.models.error import Error + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ErrorResponse(BaseModel): + """ + ErrorResponse + """ # noqa: E501 + + error: Error + __properties: ClassVar[List[str]] = ["error"] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ErrorResponse from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of error + if self.error: + _dict["error"] = self.error.to_dict() + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ErrorResponse from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "error": Error.from_dict(obj.get("error")) + if obj.get("error") is not None + else None + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/extra_models.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/extra_models.py new file mode 100644 index 00000000..f0588d2a --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/extra_models.py @@ -0,0 +1,9 @@ +# coding: utf-8 + +from pydantic import BaseModel + + +class TokenModel(BaseModel): + """Defines a token model.""" + + sub: str diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/function_object.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/function_object.py new file mode 100644 index 00000000..b1607ec4 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/function_object.py @@ -0,0 +1,101 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, StrictStr + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class FunctionObject(BaseModel): + """ + FunctionObject + """ # noqa: E501 + + description: Optional[StrictStr] = Field( + default=None, + description="A description of what the function does, used by the model to choose when and how to call the function.", + ) + name: StrictStr = Field( + description="The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64." + ) + parameters: Optional[Dict[str, Any]] = Field( + default=None, + description="The parameters the functions accepts, described as a JSON Schema object. See the [guide](/docs/guides/text-generation/function-calling) for examples, and the [JSON Schema reference](https://json-schema.org/understanding-json-schema/) for documentation about the format. Omitting `parameters` defines a function with an empty parameter list.", + ) + __properties: ClassVar[List[str]] = ["description", "name", "parameters"] + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of FunctionObject from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of FunctionObject from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "description": obj.get("description"), + "name": obj.get("name"), + "parameters": obj.get("parameters"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/list_models_response.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/list_models_response.py new file mode 100644 index 00000000..e02d2013 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/list_models_response.py @@ -0,0 +1,109 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, StrictStr, field_validator + +from openapi_server.models.model import Model + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class ListModelsResponse(BaseModel): + """ + ListModelsResponse + """ # noqa: E501 + + object: StrictStr + data: List[Model] + __properties: ClassVar[List[str]] = ["object", "data"] + + @field_validator("object") + def object_validate_enum(cls, value): + """Validates the enum""" + if value not in ("list"): + raise ValueError("must be one of enum values ('list')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of ListModelsResponse from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of each item in data (list) + _items = [] + if self.data: + for _item in self.data: + if _item: + _items.append(_item.to_dict()) + _dict["data"] = _items + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of ListModelsResponse from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "object": obj.get("object"), + "data": [Model.from_dict(_item) for _item in obj.get("data")] + if obj.get("data") is not None + else None, + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/model.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/model.py new file mode 100644 index 00000000..4b2783b9 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/model.py @@ -0,0 +1,106 @@ +# coding: utf-8 + +""" + OpenAI API + + The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + + The version of the OpenAPI document: 2.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations + +import json +import pprint +import re # noqa: F401 +from typing import Any, ClassVar, Dict, List + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator + +try: + from typing import Self +except ImportError: + from typing_extensions import Self + + +class Model(BaseModel): + """ + Describes an OpenAI model offering that can be used with the API. + """ # noqa: E501 + + id: StrictStr = Field( + description="The model identifier, which can be referenced in the API endpoints." + ) + created: StrictInt = Field( + description="The Unix timestamp (in seconds) when the model was created." + ) + object: StrictStr = Field(description='The object type, which is always "model".') + owned_by: StrictStr = Field(description="The organization that owns the model.") + __properties: ClassVar[List[str]] = ["id", "created", "object", "owned_by"] + + @field_validator("object") + def object_validate_enum(cls, value): + """Validates the enum""" + if value not in ("model"): + raise ValueError("must be one of enum values ('model')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of Model from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={}, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of Model from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate( + { + "id": obj.get("id"), + "created": obj.get("created"), + "object": obj.get("object"), + "owned_by": obj.get("owned_by"), + } + ) + return _obj diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/security_api.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/security_api.py new file mode 100644 index 00000000..368f87b8 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/security_api.py @@ -0,0 +1,40 @@ +# coding: utf-8 + +from typing import List + +from fastapi import Depends, Security # noqa: F401 +from fastapi.openapi.models import OAuthFlowImplicit, OAuthFlows # noqa: F401 +from fastapi.security import ( # noqa: F401 + HTTPAuthorizationCredentials, + HTTPBasic, + HTTPBasicCredentials, + HTTPBearer, + OAuth2, + OAuth2AuthorizationCodeBearer, + OAuth2PasswordBearer, + SecurityScopes, +) +from fastapi.security.api_key import ( # noqa: F401 + APIKeyCookie, + APIKeyHeader, + APIKeyQuery, +) + +from openapi_server.models.extra_models import TokenModel + +bearer_auth = HTTPBearer() + + +def get_token_ApiKeyAuth( + credentials: HTTPAuthorizationCredentials = Depends(bearer_auth), +) -> TokenModel: + """ + Check and retrieve authentication information from custom bearer token. + + :param credentials Credentials provided by Authorization header + :type credentials: HTTPAuthorizationCredentials + :return: Decoded token information or None if token is invalid + :rtype: TokenModel | None + """ + + return None diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/conftest.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/conftest.py new file mode 100644 index 00000000..cbde552c --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/conftest.py @@ -0,0 +1,17 @@ +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from openapi_server.main import app as application + + +@pytest.fixture +def app() -> FastAPI: + application.dependency_overrides = {} + + return application + + +@pytest.fixture +def client(app) -> TestClient: + return TestClient(app) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_chat_api.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_chat_api.py new file mode 100644 index 00000000..c4e48701 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_chat_api.py @@ -0,0 +1,79 @@ +# coding: utf-8 + +from fastapi.testclient import TestClient + +from openapi_server.models.create_chat_completion_request import ( # noqa: F401 + CreateChatCompletionRequest, +) +from openapi_server.models.create_chat_completion_response import ( # noqa: F401 + CreateChatCompletionResponse, +) + + +def test_create_chat_completion(client: TestClient): + """Test case for create_chat_completion + + Creates a model response for the given chat conversation. + """ + create_chat_completion_request = { + "top_logprobs": 2, + "logit_bias": {"key": 6}, + "seed": -2147483648, + "functions": [ + {"name": "name", "description": "description", "parameters": {"key": ""}}, + {"name": "name", "description": "description", "parameters": {"key": ""}}, + {"name": "name", "description": "description", "parameters": {"key": ""}}, + {"name": "name", "description": "description", "parameters": {"key": ""}}, + {"name": "name", "description": "description", "parameters": {"key": ""}}, + ], + "max_tokens": 5, + "function_call": "none", + "presence_penalty": 0.25495066265333133, + "tools": [ + { + "function": { + "name": "name", + "description": "description", + "parameters": {"key": ""}, + }, + "type": "function", + }, + { + "function": { + "name": "name", + "description": "description", + "parameters": {"key": ""}, + }, + "type": "function", + }, + ], + "n": 1, + "logprobs": 0, + "top_p": 1, + "frequency_penalty": -1.6796687238155954, + "response_format": {"type": "json_object"}, + "stop": "CreateChatCompletionRequest_stop", + "stream": 0, + "temperature": 1, + "messages": [ + {"role": "system", "name": "name", "content": "content"}, + {"role": "system", "name": "name", "content": "content"}, + ], + "tool_choice": "none", + "model": "gpt-4-turbo", + "user": "user-1234", + } + + headers = { + "Authorization": "Bearer special-key", + } + # uncomment below to make a request + # response = client.request( + # "POST", + # "/chat/completions", + # headers=headers, + # json=create_chat_completion_request, + # ) + + # uncomment below to assert the status code of the HTTP response + # assert response.status_code == 200 diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_completions_api.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_completions_api.py new file mode 100644 index 00000000..aea23486 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_completions_api.py @@ -0,0 +1,50 @@ +# coding: utf-8 + +from fastapi.testclient import TestClient + +from openapi_server.models.create_completion_request import ( # noqa: F401 + CreateCompletionRequest, +) +from openapi_server.models.create_completion_response import ( # noqa: F401 + CreateCompletionResponse, +) + + +def test_create_completion(client: TestClient): + """Test case for create_completion + + Creates a completion for the provided prompt and parameters. + """ + create_completion_request = { + "logit_bias": {"key": 1}, + "seed": -2147483648, + "max_tokens": 16, + "presence_penalty": 0.25495066265333133, + "echo": 0, + "suffix": "test.", + "n": 1, + "logprobs": 2, + "top_p": 1, + "frequency_penalty": 0.4109824732281613, + "best_of": 1, + "stop": "\n", + "stream": 0, + "temperature": 1, + "model": "CreateCompletionRequest_model", + "prompt": "This is a test.", + "user": "user-1234", + } + + headers = { + "Authorization": "Bearer special-key", + } + # uncomment below to make a request + # response = client.request( + # "POST", + # "/completions", + # headers=headers, + # json=create_completion_request, + # ) + + # uncomment below to assert the status code of the HTTP response + # assert response.status_code == 200 diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_models_api.py b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_models_api.py new file mode 100644 index 00000000..47bc763a --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_models_api.py @@ -0,0 +1,69 @@ +# coding: utf-8 + +from fastapi.testclient import TestClient + +from openapi_server.models.delete_model_response import ( # noqa: F401 + DeleteModelResponse, +) +from openapi_server.models.list_models_response import ListModelsResponse # noqa: F401 +from openapi_server.models.model import Model # noqa: F401 + + +def test_delete_model(client: TestClient): + """Test case for delete_model + + Delete a fine-tuned model. You must have the Owner role in your organization to delete a model. + """ + + headers = { + "Authorization": "Bearer special-key", + } + # uncomment below to make a request + # response = client.request( + # "DELETE", + # "/models/{model}".format(model='ft:gpt-3.5-turbo:acemeco:suffix:abc123'), + # headers=headers, + # ) + + # uncomment below to assert the status code of the HTTP response + # assert response.status_code == 200 + + +def test_list_models(client: TestClient): + """Test case for list_models + + Lists the currently available models, and provides basic information about each one such as the owner and availability. + """ + + headers = { + "Authorization": "Bearer special-key", + } + # uncomment below to make a request + # response = client.request( + # "GET", + # "/models", + # headers=headers, + # ) + + # uncomment below to assert the status code of the HTTP response + # assert response.status_code == 200 + + +def test_retrieve_model(client: TestClient): + """Test case for retrieve_model + + Retrieves a model instance, providing basic information about the model such as the owner and permissioning. + """ + + headers = { + "Authorization": "Bearer special-key", + } + # uncomment below to make a request + # response = client.request( + # "GET", + # "/models/{model}".format(model='gpt-3.5-turbo'), + # headers=headers, + # ) + + # uncomment below to assert the status code of the HTTP response + # assert response.status_code == 200 From 0c8ee72cb2065bbed698b51f8727abd0f9c49238 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sat, 4 May 2024 15:04:47 -0700 Subject: [PATCH 13/62] adding in initial chat completions --- .../examples/fastapi/simple/main.py | 55 ++++++++++++++++++- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/simple/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/simple/main.py index 109d84fc..2a56e572 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/simple/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/simple/main.py @@ -7,13 +7,19 @@ import copy import time import uuid +from typing import TypedDict import tritonserver from fastapi import FastAPI +from vllm.transformers_utils.tokenizer import get_tokenizer triton_server = tritonserver.Server( model_repository="/workspace/llm-models", log_verbose=6, strict_model_config=False -).start() +).start(wait_until_ready=True) + +model = triton_server.model("llama-3-8b-instruct") + +model.tokenizer = get_tokenizer(tokenizer_name="meta-llama/Meta-Llama-3-8B-Instruct") from models import ( ChatCompletionResponseMessage, @@ -49,6 +55,11 @@ ) +class ConversationMessage(TypedDict): + role: str + content: str + + @app.post( "/chat/completions", response_model=CreateChatCompletionResponse, tags=["Chat"] ) @@ -58,6 +69,43 @@ def create_chat_completion( """ Creates a model response for the given chat conversation. """ + + conversation = [ + ConversationMessage( + role=str(message.dict()["role"]), content=str(message.dict()["content"]) + ) + for message in body.messages + ] + + prompt = model.tokenizer.apply_chat_template( + conversation=conversation, tokenize=False, add_generation_prompt=False + ) + + exclude_input_in_output = True + + parameters = copy.deepcopy(body.dict()) + if "prompt" in parameters: + del parameters["prompt"] + if "stream" in parameters: + del parameters["stream"] + if "echo" in parameters: + del parameters["echo"] + if "model" in parameters: + del parameters["model"] + if "messages" in parameters: + del parameters["messages"] + + response = list( + model.infer( + inputs={ + "text_input": [prompt], + "stream": [False], + "exclude_input_in_output": [exclude_input_in_output], + }, + parameters=parameters, + ) + )[0] + return CreateChatCompletionResponse( id="foo", choices=[ @@ -65,7 +113,9 @@ def create_chat_completion( finish_reason=FinishReason1.stop, index=0, message=ChatCompletionResponseMessage( - content="hello", role=Role5.assistant, function_call=None + content=response.outputs["text_output"].to_string_array()[0], + role=Role5.assistant, + function_call=None, ), logprobs=Logprobs2(content=[]), ) @@ -87,7 +137,6 @@ def create_completion(body: CreateCompletionRequest) -> CreateCompletionResponse if body.echo: exclude_input_in_output = False - model = triton_server.model("llama-3-8b-instruct") parameters = copy.deepcopy(body.dict()) del parameters["prompt"] del parameters["stream"] From 884278f85cf006603fdf9b01990e1ecc785a24b4 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 5 May 2024 11:19:06 -0700 Subject: [PATCH 14/62] updated to exclude openai.yml taken directly https://raw.githubusercontent.com/openai/openai-openapi/master/openapi.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f44f8153..0434c44a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,7 +65,7 @@ repos: - id: check-json - id: check-toml - id: check-yaml - exclude: ^deploy(\/[^\/]+)*\/templates\/.*$ + exclude: ^deploy(\/[^\/]+)*\/templates\/.*$|^fastapi/api-spec/openai.yml$ - id: check-shebang-scripts-are-executable - id: end-of-file-fixer types_or: [c, c++, cuda, proto, textproto, java, python] From 72135a52bdb3dcd70f10877d8842a5f9b0a1827b Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 5 May 2024 11:26:31 -0700 Subject: [PATCH 15/62] updated to exclude external unmodified api spec from pre-commit --- .pre-commit-config.yaml | 4 +- .../opanai_trimmed.yml} | 3 + .../examples/fastapi/api-spec/openai.yml | 13326 ++++++++++++++++ 3 files changed, 13331 insertions(+), 2 deletions(-) rename Triton_Inference_Server_Python_API/examples/fastapi/{openapi_modified.yaml => api-spec/opanai_trimmed.yml} (99%) create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/api-spec/openai.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0434c44a..8db4821a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,7 +53,7 @@ repos: - id: codespell additional_dependencies: [tomli] args: ["--toml", "pyproject.toml"] - exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$) + exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$)||^Triton_Inference_Server_Python_API/examples/fastapi/api-spec/openai.yml$ # More details about these pre-commit hooks here: # https://pre-commit.com/hooks.html - repo: https://github.com/pre-commit/pre-commit-hooks @@ -65,7 +65,7 @@ repos: - id: check-json - id: check-toml - id: check-yaml - exclude: ^deploy(\/[^\/]+)*\/templates\/.*$|^fastapi/api-spec/openai.yml$ + exclude: ^deploy(\/[^\/]+)*\/templates\/.*$|^Triton_Inference_Server_Python_API/examples/fastapi/api-spec/openai.yml$ - id: check-shebang-scripts-are-executable - id: end-of-file-fixer types_or: [c, c++, cuda, proto, textproto, java, python] diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openapi_modified.yaml b/Triton_Inference_Server_Python_API/examples/fastapi/api-spec/opanai_trimmed.yml similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openapi_modified.yaml rename to Triton_Inference_Server_Python_API/examples/fastapi/api-spec/opanai_trimmed.yml index 775a264b..2cfa92ae 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openapi_modified.yaml +++ b/Triton_Inference_Server_Python_API/examples/fastapi/api-spec/opanai_trimmed.yml @@ -1,3 +1,6 @@ +# Modified From: +# https://raw.githubusercontent.com/openai/openai-openapi/25d9dacc86a94df1db98725fe87494564317cafa/openapi.yaml + openapi: 3.0.0 info: title: OpenAI API diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/api-spec/openai.yml b/Triton_Inference_Server_Python_API/examples/fastapi/api-spec/openai.yml new file mode 100644 index 00000000..1aafccb1 --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/api-spec/openai.yml @@ -0,0 +1,13326 @@ +# Unmodified From: +# https://raw.githubusercontent.com/openai/openai-openapi/25d9dacc86a94df1db98725fe87494564317cafa/openapi.yaml + +openapi: 3.0.0 +info: + title: OpenAI API + description: The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. + version: "2.0.0" + termsOfService: https://openai.com/policies/terms-of-use + contact: + name: OpenAI Support + url: https://help.openai.com/ + license: + name: MIT + url: https://github.com/openai/openai-openapi/blob/master/LICENSE +servers: + - url: https://api.openai.com/v1 +tags: + - name: Assistants + description: Build Assistants that can call models and use tools. + - name: Audio + description: Learn how to turn audio into text or text into audio. + - name: Chat + description: Given a list of messages comprising a conversation, the model will return a response. + - name: Completions + description: Given a prompt, the model will return one or more predicted completions, and can also return the probabilities of alternative tokens at each position. + - name: Embeddings + description: Get a vector representation of a given input that can be easily consumed by machine learning models and algorithms. + - name: Fine-tuning + description: Manage fine-tuning jobs to tailor a model to your specific training data. + - name: Batch + description: Create large batches of API requests to run asynchronously. + - name: Files + description: Files are used to upload documents that can be used with features like Assistants and Fine-tuning. + - name: Images + description: Given a prompt and/or an input image, the model will generate a new image. + - name: Models + description: List and describe the various models available in the API. + - name: Moderations + description: Given a input text, outputs if the model classifies it as potentially harmful. +paths: + # Note: When adding an endpoint, make sure you also add it in the `groups` section, in the end of this file, + # under the appropriate group + /chat/completions: + post: + operationId: createChatCompletion + tags: + - Chat + summary: Creates a model response for the given chat conversation. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateChatCompletionRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/CreateChatCompletionResponse" + + x-oaiMeta: + name: Create chat completion + group: chat + returns: | + Returns a [chat completion](/docs/api-reference/chat/object) object, or a streamed sequence of [chat completion chunk](/docs/api-reference/chat/streaming) objects if the request is streamed. + path: create + examples: + - title: Default + request: + curl: | + curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "VAR_model_id", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Hello!" + } + ] + }' + python: | + from openai import OpenAI + client = OpenAI() + + completion = client.chat.completions.create( + model="VAR_model_id", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"} + ] + ) + + print(completion.choices[0].message) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const completion = await openai.chat.completions.create({ + messages: [{ role: "system", content: "You are a helpful assistant." }], + model: "VAR_model_id", + }); + + console.log(completion.choices[0]); + } + + main(); + response: &chat_completion_example | + { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-3.5-turbo-0125", + "system_fingerprint": "fp_44709d6fcb", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "\n\nHello there, how may I assist you today?", + }, + "logprobs": null, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } + } + - title: Image input + request: + curl: | + curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "gpt-4-turbo", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What'\''s in this image?" + }, + { + "type": "image_url", + "image_url": { + "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + } + } + ] + } + ], + "max_tokens": 300 + }' + python: | + from openai import OpenAI + + client = OpenAI() + + response = client.chat.completions.create( + model="gpt-4-turbo", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", + }, + ], + } + ], + max_tokens=300, + ) + + print(response.choices[0]) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const response = await openai.chat.completions.create({ + model: "gpt-4-turbo", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What's in this image?" }, + { + type: "image_url", + image_url: + "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", + }, + ], + }, + ], + }); + console.log(response.choices[0]); + } + main(); + response: &chat_completion_image_example | + { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-3.5-turbo-0125", + "system_fingerprint": "fp_44709d6fcb", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "\n\nThis image shows a wooden boardwalk extending through a lush green marshland.", + }, + "logprobs": null, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } + } + - title: Streaming + request: + curl: | + curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "VAR_model_id", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Hello!" + } + ], + "stream": true + }' + python: | + from openai import OpenAI + client = OpenAI() + + completion = client.chat.completions.create( + model="VAR_model_id", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"} + ], + stream=True + ) + + for chunk in completion: + print(chunk.choices[0].delta) + + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const completion = await openai.chat.completions.create({ + model: "VAR_model_id", + messages: [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"} + ], + stream: true, + }); + + for await (const chunk of completion) { + console.log(chunk.choices[0].delta.content); + } + } + + main(); + response: &chat_completion_chunk_example | + {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0125", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]} + + {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0125", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}]} + + .... + + {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"gpt-3.5-turbo-0125", "system_fingerprint": "fp_44709d6fcb", "choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}]} + - title: Functions + request: + curl: | + curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "gpt-4-turbo", + "messages": [ + { + "role": "user", + "content": "What'\''s the weather like in Boston today?" + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"] + } + }, + "required": ["location"] + } + } + } + ], + "tool_choice": "auto" + }' + python: | + from openai import OpenAI + client = OpenAI() + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + } + ] + messages = [{"role": "user", "content": "What's the weather like in Boston today?"}] + completion = client.chat.completions.create( + model="VAR_model_id", + messages=messages, + tools=tools, + tool_choice="auto" + ) + + print(completion) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const messages = [{"role": "user", "content": "What's the weather like in Boston today?"}]; + const tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + } + ]; + + const response = await openai.chat.completions.create({ + model: "gpt-4-turbo", + messages: messages, + tools: tools, + tool_choice: "auto", + }); + + console.log(response); + } + + main(); + response: &chat_completion_function_example | + { + "id": "chatcmpl-abc123", + "object": "chat.completion", + "created": 1699896916, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\n\"location\": \"Boston, MA\"\n}" + } + } + ] + }, + "logprobs": null, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 82, + "completion_tokens": 17, + "total_tokens": 99 + } + } + - title: Logprobs + request: + curl: | + curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "VAR_model_id", + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ], + "logprobs": true, + "top_logprobs": 2 + }' + python: | + from openai import OpenAI + client = OpenAI() + + completion = client.chat.completions.create( + model="VAR_model_id", + messages=[ + {"role": "user", "content": "Hello!"} + ], + logprobs=True, + top_logprobs=2 + ) + + print(completion.choices[0].message) + print(completion.choices[0].logprobs) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const completion = await openai.chat.completions.create({ + messages: [{ role: "user", content: "Hello!" }], + model: "VAR_model_id", + logprobs: true, + top_logprobs: 2, + }); + + console.log(completion.choices[0]); + } + + main(); + response: | + { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1702685778, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + }, + "logprobs": { + "content": [ + { + "token": "Hello", + "logprob": -0.31725305, + "bytes": [72, 101, 108, 108, 111], + "top_logprobs": [ + { + "token": "Hello", + "logprob": -0.31725305, + "bytes": [72, 101, 108, 108, 111] + }, + { + "token": "Hi", + "logprob": -1.3190403, + "bytes": [72, 105] + } + ] + }, + { + "token": "!", + "logprob": -0.02380986, + "bytes": [ + 33 + ], + "top_logprobs": [ + { + "token": "!", + "logprob": -0.02380986, + "bytes": [33] + }, + { + "token": " there", + "logprob": -3.787621, + "bytes": [32, 116, 104, 101, 114, 101] + } + ] + }, + { + "token": " How", + "logprob": -0.000054669687, + "bytes": [32, 72, 111, 119], + "top_logprobs": [ + { + "token": " How", + "logprob": -0.000054669687, + "bytes": [32, 72, 111, 119] + }, + { + "token": "<|end|>", + "logprob": -10.953937, + "bytes": null + } + ] + }, + { + "token": " can", + "logprob": -0.015801601, + "bytes": [32, 99, 97, 110], + "top_logprobs": [ + { + "token": " can", + "logprob": -0.015801601, + "bytes": [32, 99, 97, 110] + }, + { + "token": " may", + "logprob": -4.161023, + "bytes": [32, 109, 97, 121] + } + ] + }, + { + "token": " I", + "logprob": -3.7697225e-6, + "bytes": [ + 32, + 73 + ], + "top_logprobs": [ + { + "token": " I", + "logprob": -3.7697225e-6, + "bytes": [32, 73] + }, + { + "token": " assist", + "logprob": -13.596657, + "bytes": [32, 97, 115, 115, 105, 115, 116] + } + ] + }, + { + "token": " assist", + "logprob": -0.04571125, + "bytes": [32, 97, 115, 115, 105, 115, 116], + "top_logprobs": [ + { + "token": " assist", + "logprob": -0.04571125, + "bytes": [32, 97, 115, 115, 105, 115, 116] + }, + { + "token": " help", + "logprob": -3.1089056, + "bytes": [32, 104, 101, 108, 112] + } + ] + }, + { + "token": " you", + "logprob": -5.4385737e-6, + "bytes": [32, 121, 111, 117], + "top_logprobs": [ + { + "token": " you", + "logprob": -5.4385737e-6, + "bytes": [32, 121, 111, 117] + }, + { + "token": " today", + "logprob": -12.807695, + "bytes": [32, 116, 111, 100, 97, 121] + } + ] + }, + { + "token": " today", + "logprob": -0.0040071653, + "bytes": [32, 116, 111, 100, 97, 121], + "top_logprobs": [ + { + "token": " today", + "logprob": -0.0040071653, + "bytes": [32, 116, 111, 100, 97, 121] + }, + { + "token": "?", + "logprob": -5.5247097, + "bytes": [63] + } + ] + }, + { + "token": "?", + "logprob": -0.0008108172, + "bytes": [63], + "top_logprobs": [ + { + "token": "?", + "logprob": -0.0008108172, + "bytes": [63] + }, + { + "token": "?\n", + "logprob": -7.184561, + "bytes": [63, 10] + } + ] + } + ] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 9, + "total_tokens": 18 + }, + "system_fingerprint": null + } + + /completions: + post: + operationId: createCompletion + tags: + - Completions + summary: Creates a completion for the provided prompt and parameters. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateCompletionRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/CreateCompletionResponse" + x-oaiMeta: + name: Create completion + group: completions + returns: | + Returns a [completion](/docs/api-reference/completions/object) object, or a sequence of completion objects if the request is streamed. + legacy: true + examples: + - title: No streaming + request: + curl: | + curl https://api.openai.com/v1/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "VAR_model_id", + "prompt": "Say this is a test", + "max_tokens": 7, + "temperature": 0 + }' + python: | + from openai import OpenAI + client = OpenAI() + + client.completions.create( + model="VAR_model_id", + prompt="Say this is a test", + max_tokens=7, + temperature=0 + ) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const completion = await openai.completions.create({ + model: "VAR_model_id", + prompt: "Say this is a test.", + max_tokens: 7, + temperature: 0, + }); + + console.log(completion); + } + main(); + response: | + { + "id": "cmpl-uqkvlQyYK7bGYrRHQ0eXlWi7", + "object": "text_completion", + "created": 1589478378, + "model": "VAR_model_id", + "system_fingerprint": "fp_44709d6fcb", + "choices": [ + { + "text": "\n\nThis is indeed a test", + "index": 0, + "logprobs": null, + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 7, + "total_tokens": 12 + } + } + - title: Streaming + request: + curl: | + curl https://api.openai.com/v1/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "VAR_model_id", + "prompt": "Say this is a test", + "max_tokens": 7, + "temperature": 0, + "stream": true + }' + python: | + from openai import OpenAI + client = OpenAI() + + for chunk in client.completions.create( + model="VAR_model_id", + prompt="Say this is a test", + max_tokens=7, + temperature=0, + stream=True + ): + print(chunk.choices[0].text) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const stream = await openai.completions.create({ + model: "VAR_model_id", + prompt: "Say this is a test.", + stream: true, + }); + + for await (const chunk of stream) { + console.log(chunk.choices[0].text) + } + } + main(); + response: | + { + "id": "cmpl-7iA7iJjj8V2zOkCGvWF2hAkDWBQZe", + "object": "text_completion", + "created": 1690759702, + "choices": [ + { + "text": "This", + "index": 0, + "logprobs": null, + "finish_reason": null + } + ], + "model": "gpt-3.5-turbo-instruct" + "system_fingerprint": "fp_44709d6fcb", + } + + /images/generations: + post: + operationId: createImage + tags: + - Images + summary: Creates an image given a prompt. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateImageRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ImagesResponse" + x-oaiMeta: + name: Create image + group: images + returns: Returns a list of [image](/docs/api-reference/images/object) objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/images/generations \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "dall-e-3", + "prompt": "A cute baby sea otter", + "n": 1, + "size": "1024x1024" + }' + python: | + from openai import OpenAI + client = OpenAI() + + client.images.generate( + model="dall-e-3", + prompt="A cute baby sea otter", + n=1, + size="1024x1024" + ) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const image = await openai.images.generate({ model: "dall-e-3", prompt: "A cute baby sea otter" }); + + console.log(image.data); + } + main(); + response: | + { + "created": 1589478378, + "data": [ + { + "url": "https://..." + }, + { + "url": "https://..." + } + ] + } + /images/edits: + post: + operationId: createImageEdit + tags: + - Images + summary: Creates an edited or extended image given an original image and a prompt. + requestBody: + required: true + content: + multipart/form-data: + schema: + $ref: "#/components/schemas/CreateImageEditRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ImagesResponse" + x-oaiMeta: + name: Create image edit + group: images + returns: Returns a list of [image](/docs/api-reference/images/object) objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/images/edits \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -F image="@otter.png" \ + -F mask="@mask.png" \ + -F prompt="A cute baby sea otter wearing a beret" \ + -F n=2 \ + -F size="1024x1024" + python: | + from openai import OpenAI + client = OpenAI() + + client.images.edit( + image=open("otter.png", "rb"), + mask=open("mask.png", "rb"), + prompt="A cute baby sea otter wearing a beret", + n=2, + size="1024x1024" + ) + node.js: |- + import fs from "fs"; + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const image = await openai.images.edit({ + image: fs.createReadStream("otter.png"), + mask: fs.createReadStream("mask.png"), + prompt: "A cute baby sea otter wearing a beret", + }); + + console.log(image.data); + } + main(); + response: | + { + "created": 1589478378, + "data": [ + { + "url": "https://..." + }, + { + "url": "https://..." + } + ] + } + /images/variations: + post: + operationId: createImageVariation + tags: + - Images + summary: Creates a variation of a given image. + requestBody: + required: true + content: + multipart/form-data: + schema: + $ref: "#/components/schemas/CreateImageVariationRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ImagesResponse" + x-oaiMeta: + name: Create image variation + group: images + returns: Returns a list of [image](/docs/api-reference/images/object) objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/images/variations \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -F image="@otter.png" \ + -F n=2 \ + -F size="1024x1024" + python: | + from openai import OpenAI + client = OpenAI() + + response = client.images.create_variation( + image=open("image_edit_original.png", "rb"), + n=2, + size="1024x1024" + ) + node.js: |- + import fs from "fs"; + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const image = await openai.images.createVariation({ + image: fs.createReadStream("otter.png"), + }); + + console.log(image.data); + } + main(); + response: | + { + "created": 1589478378, + "data": [ + { + "url": "https://..." + }, + { + "url": "https://..." + } + ] + } + + /embeddings: + post: + operationId: createEmbedding + tags: + - Embeddings + summary: Creates an embedding vector representing the input text. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateEmbeddingRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/CreateEmbeddingResponse" + x-oaiMeta: + name: Create embeddings + group: embeddings + returns: A list of [embedding](/docs/api-reference/embeddings/object) objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/embeddings \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "input": "The food was delicious and the waiter...", + "model": "text-embedding-ada-002", + "encoding_format": "float" + }' + python: | + from openai import OpenAI + client = OpenAI() + + client.embeddings.create( + model="text-embedding-ada-002", + input="The food was delicious and the waiter...", + encoding_format="float" + ) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const embedding = await openai.embeddings.create({ + model: "text-embedding-ada-002", + input: "The quick brown fox jumped over the lazy dog", + encoding_format: "float", + }); + + console.log(embedding); + } + + main(); + response: | + { + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [ + 0.0023064255, + -0.009327292, + .... (1536 floats total for ada-002) + -0.0028842222, + ], + "index": 0 + } + ], + "model": "text-embedding-ada-002", + "usage": { + "prompt_tokens": 8, + "total_tokens": 8 + } + } + + /audio/speech: + post: + operationId: createSpeech + tags: + - Audio + summary: Generates audio from the input text. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateSpeechRequest" + responses: + "200": + description: OK + headers: + Transfer-Encoding: + schema: + type: string + description: chunked + content: + application/octet-stream: + schema: + type: string + format: binary + x-oaiMeta: + name: Create speech + group: audio + returns: The audio file content. + examples: + request: + curl: | + curl https://api.openai.com/v1/audio/speech \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "tts-1", + "input": "The quick brown fox jumped over the lazy dog.", + "voice": "alloy" + }' \ + --output speech.mp3 + python: | + from pathlib import Path + import openai + + speech_file_path = Path(__file__).parent / "speech.mp3" + response = openai.audio.speech.create( + model="tts-1", + voice="alloy", + input="The quick brown fox jumped over the lazy dog." + ) + response.stream_to_file(speech_file_path) + node: | + import fs from "fs"; + import path from "path"; + import OpenAI from "openai"; + + const openai = new OpenAI(); + + const speechFile = path.resolve("./speech.mp3"); + + async function main() { + const mp3 = await openai.audio.speech.create({ + model: "tts-1", + voice: "alloy", + input: "Today is a wonderful day to build something people love!", + }); + console.log(speechFile); + const buffer = Buffer.from(await mp3.arrayBuffer()); + await fs.promises.writeFile(speechFile, buffer); + } + main(); + /audio/transcriptions: + post: + operationId: createTranscription + tags: + - Audio + summary: Transcribes audio into the input language. + requestBody: + required: true + content: + multipart/form-data: + schema: + $ref: "#/components/schemas/CreateTranscriptionRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/CreateTranscriptionResponseJson" + - $ref: "#/components/schemas/CreateTranscriptionResponseVerboseJson" + x-oaiMeta: + name: Create transcription + group: audio + returns: The [transcription object](/docs/api-reference/audio/json-object) or a [verbose transcription object](/docs/api-reference/audio/verbose-json-object). + examples: + - title: Default + request: + curl: | + curl https://api.openai.com/v1/audio/transcriptions \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: multipart/form-data" \ + -F file="@/path/to/file/audio.mp3" \ + -F model="whisper-1" + python: | + from openai import OpenAI + client = OpenAI() + + audio_file = open("speech.mp3", "rb") + transcript = client.audio.transcriptions.create( + model="whisper-1", + file=audio_file + ) + node: | + import fs from "fs"; + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const transcription = await openai.audio.transcriptions.create({ + file: fs.createReadStream("audio.mp3"), + model: "whisper-1", + }); + + console.log(transcription.text); + } + main(); + response: &basic_transcription_response_example | + { + "text": "Imagine the wildest idea that you've ever had, and you're curious about how it might scale to something that's a 100, a 1,000 times bigger. This is a place where you can get to do that." + } + - title: Word timestamps + request: + curl: | + curl https://api.openai.com/v1/audio/transcriptions \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: multipart/form-data" \ + -F file="@/path/to/file/audio.mp3" \ + -F "timestamp_granularities[]=word" \ + -F model="whisper-1" \ + -F response_format="verbose_json" + python: | + from openai import OpenAI + client = OpenAI() + + audio_file = open("speech.mp3", "rb") + transcript = client.audio.transcriptions.create( + file=audio_file, + model="whisper-1", + response_format="verbose_json", + timestamp_granularities=["word"] + ) + + print(transcript.words) + node: | + import fs from "fs"; + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const transcription = await openai.audio.transcriptions.create({ + file: fs.createReadStream("audio.mp3"), + model: "whisper-1", + response_format: "verbose_json", + timestamp_granularities: ["word"] + }); + + console.log(transcription.text); + } + main(); + response: | + { + "task": "transcribe", + "language": "english", + "duration": 8.470000267028809, + "text": "The beach was a popular spot on a hot summer day. People were swimming in the ocean, building sandcastles, and playing beach volleyball.", + "words": [ + { + "word": "The", + "start": 0.0, + "end": 0.23999999463558197 + }, + ... + { + "word": "volleyball", + "start": 7.400000095367432, + "end": 7.900000095367432 + } + ] + } + - title: Segment timestamps + request: + curl: | + curl https://api.openai.com/v1/audio/transcriptions \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: multipart/form-data" \ + -F file="@/path/to/file/audio.mp3" \ + -F "timestamp_granularities[]=segment" \ + -F model="whisper-1" \ + -F response_format="verbose_json" + python: | + from openai import OpenAI + client = OpenAI() + + audio_file = open("speech.mp3", "rb") + transcript = client.audio.transcriptions.create( + file=audio_file, + model="whisper-1", + response_format="verbose_json", + timestamp_granularities=["segment"] + ) + + print(transcript.words) + node: | + import fs from "fs"; + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const transcription = await openai.audio.transcriptions.create({ + file: fs.createReadStream("audio.mp3"), + model: "whisper-1", + response_format: "verbose_json", + timestamp_granularities: ["segment"] + }); + + console.log(transcription.text); + } + main(); + response: &verbose_transcription_response_example | + { + "task": "transcribe", + "language": "english", + "duration": 8.470000267028809, + "text": "The beach was a popular spot on a hot summer day. People were swimming in the ocean, building sandcastles, and playing beach volleyball.", + "segments": [ + { + "id": 0, + "seek": 0, + "start": 0.0, + "end": 3.319999933242798, + "text": " The beach was a popular spot on a hot summer day.", + "tokens": [ + 50364, 440, 7534, 390, 257, 3743, 4008, 322, 257, 2368, 4266, 786, 13, 50530 + ], + "temperature": 0.0, + "avg_logprob": -0.2860786020755768, + "compression_ratio": 1.2363636493682861, + "no_speech_prob": 0.00985979475080967 + }, + ... + ] + } + /audio/translations: + post: + operationId: createTranslation + tags: + - Audio + summary: Translates audio into English. + requestBody: + required: true + content: + multipart/form-data: + schema: + $ref: "#/components/schemas/CreateTranslationRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/CreateTranslationResponseJson" + - $ref: "#/components/schemas/CreateTranslationResponseVerboseJson" + x-oaiMeta: + name: Create translation + group: audio + returns: The translated text. + examples: + request: + curl: | + curl https://api.openai.com/v1/audio/translations \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: multipart/form-data" \ + -F file="@/path/to/file/german.m4a" \ + -F model="whisper-1" + python: | + from openai import OpenAI + client = OpenAI() + + audio_file = open("speech.mp3", "rb") + transcript = client.audio.translations.create( + model="whisper-1", + file=audio_file + ) + node: | + import fs from "fs"; + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const translation = await openai.audio.translations.create({ + file: fs.createReadStream("speech.mp3"), + model: "whisper-1", + }); + + console.log(translation.text); + } + main(); + response: | + { + "text": "Hello, my name is Wolfgang and I come from Germany. Where are you heading today?" + } + + /files: + get: + operationId: listFiles + tags: + - Files + summary: Returns a list of files that belong to the user's organization. + parameters: + - in: query + name: purpose + required: false + schema: + type: string + description: Only return files with the given purpose. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListFilesResponse" + x-oaiMeta: + name: List files + group: files + returns: A list of [File](/docs/api-reference/files/object) objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/files \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.files.list() + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const list = await openai.files.list(); + + for await (const file of list) { + console.log(file); + } + } + + main(); + response: | + { + "data": [ + { + "id": "file-abc123", + "object": "file", + "bytes": 175, + "created_at": 1613677385, + "filename": "salesOverview.pdf", + "purpose": "assistants", + }, + { + "id": "file-abc123", + "object": "file", + "bytes": 140, + "created_at": 1613779121, + "filename": "puppy.jsonl", + "purpose": "fine-tune", + } + ], + "object": "list" + } + post: + operationId: createFile + tags: + - Files + summary: | + Upload a file that can be used across various endpoints. The size of all the files uploaded by one organization can be up to 100 GB. + + The size of individual files can be a maximum of 512 MB or 2 million tokens for Assistants. See the [Assistants Tools guide](/docs/assistants/tools) to learn more about the types of files supported. The Fine-tuning API only supports `.jsonl` files. + + Please [contact us](https://help.openai.com/) if you need to increase these storage limits. + requestBody: + required: true + content: + multipart/form-data: + schema: + $ref: "#/components/schemas/CreateFileRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/OpenAIFile" + x-oaiMeta: + name: Upload file + group: files + returns: The uploaded [File](/docs/api-reference/files/object) object. + examples: + request: + curl: | + curl https://api.openai.com/v1/files \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -F purpose="fine-tune" \ + -F file="@mydata.jsonl" + python: | + from openai import OpenAI + client = OpenAI() + + client.files.create( + file=open("mydata.jsonl", "rb"), + purpose="fine-tune" + ) + node.js: |- + import fs from "fs"; + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const file = await openai.files.create({ + file: fs.createReadStream("mydata.jsonl"), + purpose: "fine-tune", + }); + + console.log(file); + } + + main(); + response: | + { + "id": "file-abc123", + "object": "file", + "bytes": 120000, + "created_at": 1677610602, + "filename": "mydata.jsonl", + "purpose": "fine-tune", + } + /files/{file_id}: + delete: + operationId: deleteFile + tags: + - Files + summary: Delete a file. + parameters: + - in: path + name: file_id + required: true + schema: + type: string + description: The ID of the file to use for this request. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/DeleteFileResponse" + x-oaiMeta: + name: Delete file + group: files + returns: Deletion status. + examples: + request: + curl: | + curl https://api.openai.com/v1/files/file-abc123 \ + -X DELETE \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.files.delete("file-abc123") + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const file = await openai.files.del("file-abc123"); + + console.log(file); + } + + main(); + response: | + { + "id": "file-abc123", + "object": "file", + "deleted": true + } + get: + operationId: retrieveFile + tags: + - Files + summary: Returns information about a specific file. + parameters: + - in: path + name: file_id + required: true + schema: + type: string + description: The ID of the file to use for this request. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/OpenAIFile" + x-oaiMeta: + name: Retrieve file + group: files + returns: The [File](/docs/api-reference/files/object) object matching the specified ID. + examples: + request: + curl: | + curl https://api.openai.com/v1/files/file-abc123 \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.files.retrieve("file-abc123") + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const file = await openai.files.retrieve("file-abc123"); + + console.log(file); + } + + main(); + response: | + { + "id": "file-abc123", + "object": "file", + "bytes": 120000, + "created_at": 1677610602, + "filename": "mydata.jsonl", + "purpose": "fine-tune", + } + /files/{file_id}/content: + get: + operationId: downloadFile + tags: + - Files + summary: Returns the contents of the specified file. + parameters: + - in: path + name: file_id + required: true + schema: + type: string + description: The ID of the file to use for this request. + responses: + "200": + description: OK + content: + application/json: + schema: + type: string + x-oaiMeta: + name: Retrieve file content + group: files + returns: The file content. + examples: + request: + curl: | + curl https://api.openai.com/v1/files/file-abc123/content \ + -H "Authorization: Bearer $OPENAI_API_KEY" > file.jsonl + python: | + from openai import OpenAI + client = OpenAI() + + content = client.files.content("file-abc123") + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const file = await openai.files.content("file-abc123"); + + console.log(file); + } + + main(); + + /fine_tuning/jobs: + post: + operationId: createFineTuningJob + tags: + - Fine-tuning + summary: | + Creates a fine-tuning job which begins the process of creating a new model from a given dataset. + + Response includes details of the enqueued job including job status and the name of the fine-tuned models once complete. + + [Learn more about fine-tuning](/docs/guides/fine-tuning) + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateFineTuningJobRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/FineTuningJob" + x-oaiMeta: + name: Create fine-tuning job + group: fine-tuning + returns: A [fine-tuning.job](/docs/api-reference/fine-tuning/object) object. + examples: + - title: Default + request: + curl: | + curl https://api.openai.com/v1/fine_tuning/jobs \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "training_file": "file-BK7bzQj3FfZFXr7DbL6xJwfo", + "model": "gpt-3.5-turbo" + }' + python: | + from openai import OpenAI + client = OpenAI() + + client.fine_tuning.jobs.create( + training_file="file-abc123", + model="gpt-3.5-turbo" + ) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const fineTune = await openai.fineTuning.jobs.create({ + training_file: "file-abc123" + }); + + console.log(fineTune); + } + + main(); + response: | + { + "object": "fine_tuning.job", + "id": "ftjob-abc123", + "model": "gpt-3.5-turbo-0125", + "created_at": 1614807352, + "fine_tuned_model": null, + "organization_id": "org-123", + "result_files": [], + "status": "queued", + "validation_file": null, + "training_file": "file-abc123", + } + - title: Epochs + request: + curl: | + curl https://api.openai.com/v1/fine_tuning/jobs \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "training_file": "file-abc123", + "model": "gpt-3.5-turbo", + "hyperparameters": { + "n_epochs": 2 + } + }' + python: | + from openai import OpenAI + client = OpenAI() + + client.fine_tuning.jobs.create( + training_file="file-abc123", + model="gpt-3.5-turbo", + hyperparameters={ + "n_epochs":2 + } + ) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const fineTune = await openai.fineTuning.jobs.create({ + training_file: "file-abc123", + model: "gpt-3.5-turbo", + hyperparameters: { n_epochs: 2 } + }); + + console.log(fineTune); + } + + main(); + response: | + { + "object": "fine_tuning.job", + "id": "ftjob-abc123", + "model": "gpt-3.5-turbo-0125", + "created_at": 1614807352, + "fine_tuned_model": null, + "organization_id": "org-123", + "result_files": [], + "status": "queued", + "validation_file": null, + "training_file": "file-abc123", + "hyperparameters": {"n_epochs": 2}, + } + - title: Validation file + request: + curl: | + curl https://api.openai.com/v1/fine_tuning/jobs \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "training_file": "file-abc123", + "validation_file": "file-abc123", + "model": "gpt-3.5-turbo" + }' + python: | + from openai import OpenAI + client = OpenAI() + + client.fine_tuning.jobs.create( + training_file="file-abc123", + validation_file="file-def456", + model="gpt-3.5-turbo" + ) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const fineTune = await openai.fineTuning.jobs.create({ + training_file: "file-abc123", + validation_file: "file-abc123" + }); + + console.log(fineTune); + } + + main(); + response: | + { + "object": "fine_tuning.job", + "id": "ftjob-abc123", + "model": "gpt-3.5-turbo-0125", + "created_at": 1614807352, + "fine_tuned_model": null, + "organization_id": "org-123", + "result_files": [], + "status": "queued", + "validation_file": "file-abc123", + "training_file": "file-abc123", + } + - title: W&B Integration + request: + curl: | + curl https://api.openai.com/v1/fine_tuning/jobs \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "training_file": "file-abc123", + "validation_file": "file-abc123", + "model": "gpt-3.5-turbo", + "integrations": [ + { + "type": "wandb", + "wandb": { + "project": "my-wandb-project", + "name": "ft-run-display-name" + "tags": [ + "first-experiment", "v2" + ] + } + } + ] + }' + response: | + { + "object": "fine_tuning.job", + "id": "ftjob-abc123", + "model": "gpt-3.5-turbo-0125", + "created_at": 1614807352, + "fine_tuned_model": null, + "organization_id": "org-123", + "result_files": [], + "status": "queued", + "validation_file": "file-abc123", + "training_file": "file-abc123", + "integrations": [ + { + "type": "wandb", + "wandb": { + "project": "my-wandb-project", + "entity": None, + "run_id": "ftjob-abc123" + } + } + ] + } + get: + operationId: listPaginatedFineTuningJobs + tags: + - Fine-tuning + summary: | + List your organization's fine-tuning jobs + parameters: + - name: after + in: query + description: Identifier for the last job from the previous pagination request. + required: false + schema: + type: string + - name: limit + in: query + description: Number of fine-tuning jobs to retrieve. + required: false + schema: + type: integer + default: 20 + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListPaginatedFineTuningJobsResponse" + x-oaiMeta: + name: List fine-tuning jobs + group: fine-tuning + returns: A list of paginated [fine-tuning job](/docs/api-reference/fine-tuning/object) objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/fine_tuning/jobs?limit=2 \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.fine_tuning.jobs.list() + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const list = await openai.fineTuning.jobs.list(); + + for await (const fineTune of list) { + console.log(fineTune); + } + } + + main(); + response: | + { + "object": "list", + "data": [ + { + "object": "fine_tuning.job.event", + "id": "ft-event-TjX0lMfOniCZX64t9PUQT5hn", + "created_at": 1689813489, + "level": "warn", + "message": "Fine tuning process stopping due to job cancellation", + "data": null, + "type": "message" + }, + { ... }, + { ... } + ], "has_more": true + } + /fine_tuning/jobs/{fine_tuning_job_id}: + get: + operationId: retrieveFineTuningJob + tags: + - Fine-tuning + summary: | + Get info about a fine-tuning job. + + [Learn more about fine-tuning](/docs/guides/fine-tuning) + parameters: + - in: path + name: fine_tuning_job_id + required: true + schema: + type: string + example: ft-AF1WoRqd3aJAHsqc9NY7iL8F + description: | + The ID of the fine-tuning job. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/FineTuningJob" + x-oaiMeta: + name: Retrieve fine-tuning job + group: fine-tuning + returns: The [fine-tuning](/docs/api-reference/fine-tuning/object) object with the given ID. + examples: + request: + curl: | + curl https://api.openai.com/v1/fine_tuning/jobs/ft-AF1WoRqd3aJAHsqc9NY7iL8F \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.fine_tuning.jobs.retrieve("ftjob-abc123") + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const fineTune = await openai.fineTuning.jobs.retrieve("ftjob-abc123"); + + console.log(fineTune); + } + + main(); + response: &fine_tuning_example | + { + "object": "fine_tuning.job", + "id": "ftjob-abc123", + "model": "davinci-002", + "created_at": 1692661014, + "finished_at": 1692661190, + "fine_tuned_model": "ft:davinci-002:my-org:custom_suffix:7q8mpxmy", + "organization_id": "org-123", + "result_files": [ + "file-abc123" + ], + "status": "succeeded", + "validation_file": null, + "training_file": "file-abc123", + "hyperparameters": { + "n_epochs": 4, + "batch_size": 1, + "learning_rate_multiplier": 1.0 + }, + "trained_tokens": 5768, + "integrations": [], + "seed": 0, + "estimated_finish": 0 + } + /fine_tuning/jobs/{fine_tuning_job_id}/events: + get: + operationId: listFineTuningEvents + tags: + - Fine-tuning + summary: | + Get status updates for a fine-tuning job. + parameters: + - in: path + name: fine_tuning_job_id + required: true + schema: + type: string + example: ft-AF1WoRqd3aJAHsqc9NY7iL8F + description: | + The ID of the fine-tuning job to get events for. + - name: after + in: query + description: Identifier for the last event from the previous pagination request. + required: false + schema: + type: string + - name: limit + in: query + description: Number of events to retrieve. + required: false + schema: + type: integer + default: 20 + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListFineTuningJobEventsResponse" + x-oaiMeta: + name: List fine-tuning events + group: fine-tuning + returns: A list of fine-tuning event objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/events \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.fine_tuning.jobs.list_events( + fine_tuning_job_id="ftjob-abc123", + limit=2 + ) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const list = await openai.fineTuning.list_events(id="ftjob-abc123", limit=2); + + for await (const fineTune of list) { + console.log(fineTune); + } + } + + main(); + response: | + { + "object": "list", + "data": [ + { + "object": "fine_tuning.job.event", + "id": "ft-event-ddTJfwuMVpfLXseO0Am0Gqjm", + "created_at": 1692407401, + "level": "info", + "message": "Fine tuning job successfully completed", + "data": null, + "type": "message" + }, + { + "object": "fine_tuning.job.event", + "id": "ft-event-tyiGuB72evQncpH87xe505Sv", + "created_at": 1692407400, + "level": "info", + "message": "New fine-tuned model created: ft:gpt-3.5-turbo:openai::7p4lURel", + "data": null, + "type": "message" + } + ], + "has_more": true + } + /fine_tuning/jobs/{fine_tuning_job_id}/cancel: + post: + operationId: cancelFineTuningJob + tags: + - Fine-tuning + summary: | + Immediately cancel a fine-tune job. + parameters: + - in: path + name: fine_tuning_job_id + required: true + schema: + type: string + example: ft-AF1WoRqd3aJAHsqc9NY7iL8F + description: | + The ID of the fine-tuning job to cancel. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/FineTuningJob" + x-oaiMeta: + name: Cancel fine-tuning + group: fine-tuning + returns: The cancelled [fine-tuning](/docs/api-reference/fine-tuning/object) object. + examples: + request: + curl: | + curl -X POST https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/cancel \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.fine_tuning.jobs.cancel("ftjob-abc123") + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const fineTune = await openai.fineTuning.jobs.cancel("ftjob-abc123"); + + console.log(fineTune); + } + main(); + response: | + { + "object": "fine_tuning.job", + "id": "ftjob-abc123", + "model": "gpt-3.5-turbo-0125", + "created_at": 1689376978, + "fine_tuned_model": null, + "organization_id": "org-123", + "result_files": [], + "hyperparameters": { + "n_epochs": "auto" + }, + "status": "cancelled", + "validation_file": "file-abc123", + "training_file": "file-abc123" + } + /fine_tuning/jobs/{fine_tuning_job_id}/checkpoints: + get: + operationId: listFineTuningJobCheckpoints + tags: + - Fine-tuning + summary: | + List checkpoints for a fine-tuning job. + parameters: + - in: path + name: fine_tuning_job_id + required: true + schema: + type: string + example: ft-AF1WoRqd3aJAHsqc9NY7iL8F + description: | + The ID of the fine-tuning job to get checkpoints for. + - name: after + in: query + description: Identifier for the last checkpoint ID from the previous pagination request. + required: false + schema: + type: string + - name: limit + in: query + description: Number of checkpoints to retrieve. + required: false + schema: + type: integer + default: 10 + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListFineTuningJobCheckpointsResponse" + x-oaiMeta: + name: List fine-tuning checkpoints + group: fine-tuning + returns: A list of fine-tuning [checkpoint objects](/docs/api-reference/fine-tuning/checkpoint-object) for a fine-tuning job. + examples: + request: + curl: | + curl https://api.openai.com/v1/fine_tuning/jobs/ftjob-abc123/checkpoints \ + -H "Authorization: Bearer $OPENAI_API_KEY" + response: | + { + "object": "list" + "data": [ + { + "object": "fine_tuning.job.checkpoint", + "id": "ftckpt_zc4Q7MP6XxulcVzj4MZdwsAB", + "created_at": 1519129973, + "fine_tuned_model_checkpoint": "ft:gpt-3.5-turbo-0125:my-org:custom-suffix:96olL566:ckpt-step-2000", + "metrics": { + "full_valid_loss": 0.134, + "full_valid_mean_token_accuracy": 0.874 + }, + "fine_tuning_job_id": "ftjob-abc123", + "step_number": 2000, + }, + { + "object": "fine_tuning.job.checkpoint", + "id": "ftckpt_enQCFmOTGj3syEpYVhBRLTSy", + "created_at": 1519129833, + "fine_tuned_model_checkpoint": "ft:gpt-3.5-turbo-0125:my-org:custom-suffix:7q8mpxmy:ckpt-step-1000", + "metrics": { + "full_valid_loss": 0.167, + "full_valid_mean_token_accuracy": 0.781 + }, + "fine_tuning_job_id": "ftjob-abc123", + "step_number": 1000, + }, + ], + "first_id": "ftckpt_zc4Q7MP6XxulcVzj4MZdwsAB", + "last_id": "ftckpt_enQCFmOTGj3syEpYVhBRLTSy", + "has_more": true + } + + /models: + get: + operationId: listModels + tags: + - Models + summary: Lists the currently available models, and provides basic information about each one such as the owner and availability. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListModelsResponse" + x-oaiMeta: + name: List models + group: models + returns: A list of [model](/docs/api-reference/models/object) objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/models \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.models.list() + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const list = await openai.models.list(); + + for await (const model of list) { + console.log(model); + } + } + main(); + response: | + { + "object": "list", + "data": [ + { + "id": "model-id-0", + "object": "model", + "created": 1686935002, + "owned_by": "organization-owner" + }, + { + "id": "model-id-1", + "object": "model", + "created": 1686935002, + "owned_by": "organization-owner", + }, + { + "id": "model-id-2", + "object": "model", + "created": 1686935002, + "owned_by": "openai" + }, + ], + "object": "list" + } + /models/{model}: + get: + operationId: retrieveModel + tags: + - Models + summary: Retrieves a model instance, providing basic information about the model such as the owner and permissioning. + parameters: + - in: path + name: model + required: true + schema: + type: string + # ideally this will be an actual ID, so this will always work from browser + example: gpt-3.5-turbo + description: The ID of the model to use for this request + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/Model" + x-oaiMeta: + name: Retrieve model + group: models + returns: The [model](/docs/api-reference/models/object) object matching the specified ID. + examples: + request: + curl: | + curl https://api.openai.com/v1/models/VAR_model_id \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.models.retrieve("VAR_model_id") + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const model = await openai.models.retrieve("VAR_model_id"); + + console.log(model); + } + + main(); + response: &retrieve_model_response | + { + "id": "VAR_model_id", + "object": "model", + "created": 1686935002, + "owned_by": "openai" + } + delete: + operationId: deleteModel + tags: + - Models + summary: Delete a fine-tuned model. You must have the Owner role in your organization to delete a model. + parameters: + - in: path + name: model + required: true + schema: + type: string + example: ft:gpt-3.5-turbo:acemeco:suffix:abc123 + description: The model to delete + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/DeleteModelResponse" + x-oaiMeta: + name: Delete a fine-tuned model + group: models + returns: Deletion status. + examples: + request: + curl: | + curl https://api.openai.com/v1/models/ft:gpt-3.5-turbo:acemeco:suffix:abc123 \ + -X DELETE \ + -H "Authorization: Bearer $OPENAI_API_KEY" + python: | + from openai import OpenAI + client = OpenAI() + + client.models.delete("ft:gpt-3.5-turbo:acemeco:suffix:abc123") + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const model = await openai.models.del("ft:gpt-3.5-turbo:acemeco:suffix:abc123"); + + console.log(model); + } + main(); + response: | + { + "id": "ft:gpt-3.5-turbo:acemeco:suffix:abc123", + "object": "model", + "deleted": true + } + + /moderations: + post: + operationId: createModeration + tags: + - Moderations + summary: Classifies if text is potentially harmful. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateModerationRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/CreateModerationResponse" + x-oaiMeta: + name: Create moderation + group: moderations + returns: A [moderation](/docs/api-reference/moderations/object) object. + examples: + request: + curl: | + curl https://api.openai.com/v1/moderations \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "input": "I want to kill them." + }' + python: | + from openai import OpenAI + client = OpenAI() + + moderation = client.moderations.create(input="I want to kill them.") + print(moderation) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const moderation = await openai.moderations.create({ input: "I want to kill them." }); + + console.log(moderation); + } + main(); + response: &moderation_example | + { + "id": "modr-XXXXX", + "model": "text-moderation-005", + "results": [ + { + "flagged": true, + "categories": { + "sexual": false, + "hate": false, + "harassment": false, + "self-harm": false, + "sexual/minors": false, + "hate/threatening": false, + "violence/graphic": false, + "self-harm/intent": false, + "self-harm/instructions": false, + "harassment/threatening": true, + "violence": true, + }, + "category_scores": { + "sexual": 1.2282071e-06, + "hate": 0.010696256, + "harassment": 0.29842457, + "self-harm": 1.5236925e-08, + "sexual/minors": 5.7246268e-08, + "hate/threatening": 0.0060676364, + "violence/graphic": 4.435014e-06, + "self-harm/intent": 8.098441e-10, + "self-harm/instructions": 2.8498655e-11, + "harassment/threatening": 0.63055265, + "violence": 0.99011886, + } + } + ] + } + + /assistants: + get: + operationId: listAssistants + tags: + - Assistants + summary: Returns a list of assistants. + parameters: + - name: limit + in: query + description: &pagination_limit_param_description | + A limit on the number of objects to be returned. Limit can range between 1 and 100, and the default is 20. + required: false + schema: + type: integer + default: 20 + - name: order + in: query + description: &pagination_order_param_description | + Sort order by the `created_at` timestamp of the objects. `asc` for ascending order and `desc` for descending order. + schema: + type: string + default: desc + enum: ["asc", "desc"] + - name: after + in: query + description: &pagination_after_param_description | + A cursor for use in pagination. `after` is an object ID that defines your place in the list. For instance, if you make a list request and receive 100 objects, ending with obj_foo, your subsequent call can include after=obj_foo in order to fetch the next page of the list. + schema: + type: string + - name: before + in: query + description: &pagination_before_param_description | + A cursor for use in pagination. `before` is an object ID that defines your place in the list. For instance, if you make a list request and receive 100 objects, ending with obj_foo, your subsequent call can include before=obj_foo in order to fetch the previous page of the list. + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListAssistantsResponse" + x-oaiMeta: + name: List assistants + group: assistants + beta: true + returns: A list of [assistant](/docs/api-reference/assistants/object) objects. + examples: + request: + curl: | + curl "https://api.openai.com/v1/assistants?order=desc&limit=20" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + my_assistants = client.beta.assistants.list( + order="desc", + limit="20", + ) + print(my_assistants.data) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const myAssistants = await openai.beta.assistants.list({ + order: "desc", + limit: "20", + }); + + console.log(myAssistants.data); + } + + main(); + response: &list_assistants_example | + { + "object": "list", + "data": [ + { + "id": "asst_abc123", + "object": "assistant", + "created_at": 1698982736, + "name": "Coding Tutor", + "description": null, + "model": "gpt-4-turbo", + "instructions": "You are a helpful assistant designed to make me better at coding!", + "tools": [], + "tool_resources": {}, + "metadata": {}, + "top_p": 1.0, + "temperature": 1.0, + "response_format": "auto" + }, + { + "id": "asst_abc456", + "object": "assistant", + "created_at": 1698982718, + "name": "My Assistant", + "description": null, + "model": "gpt-4-turbo", + "instructions": "You are a helpful assistant designed to make me better at coding!", + "tools": [], + "tool_resources": {}, + "metadata": {}, + "top_p": 1.0, + "temperature": 1.0, + "response_format": "auto" + }, + { + "id": "asst_abc789", + "object": "assistant", + "created_at": 1698982643, + "name": null, + "description": null, + "model": "gpt-4-turbo", + "instructions": null, + "tools": [], + "tool_resources": {}, + "metadata": {}, + "top_p": 1.0, + "temperature": 1.0, + "response_format": "auto" + } + ], + "first_id": "asst_abc123", + "last_id": "asst_abc789", + "has_more": false + } + post: + operationId: createAssistant + tags: + - Assistants + summary: Create an assistant with a model and instructions. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateAssistantRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/AssistantObject" + x-oaiMeta: + name: Create assistant + group: assistants + beta: true + returns: An [assistant](/docs/api-reference/assistants/object) object. + examples: + - title: Code Interpreter + request: + curl: | + curl "https://api.openai.com/v1/assistants" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "instructions": "You are a personal math tutor. When asked a question, write and run Python code to answer the question.", + "name": "Math Tutor", + "tools": [{"type": "code_interpreter"}], + "model": "gpt-4-turbo" + }' + + python: | + from openai import OpenAI + client = OpenAI() + + my_assistant = client.beta.assistants.create( + instructions="You are a personal math tutor. When asked a question, write and run Python code to answer the question.", + name="Math Tutor", + tools=[{"type": "code_interpreter"}], + model="gpt-4-turbo", + ) + print(my_assistant) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const myAssistant = await openai.beta.assistants.create({ + instructions: + "You are a personal math tutor. When asked a question, write and run Python code to answer the question.", + name: "Math Tutor", + tools: [{ type: "code_interpreter" }], + model: "gpt-4-turbo", + }); + + console.log(myAssistant); + } + + main(); + response: &create_assistants_example | + { + "id": "asst_abc123", + "object": "assistant", + "created_at": 1698984975, + "name": "Math Tutor", + "description": null, + "model": "gpt-4-turbo", + "instructions": "You are a personal math tutor. When asked a question, write and run Python code to answer the question.", + "tools": [ + { + "type": "code_interpreter" + } + ], + "metadata": {}, + "top_p": 1.0, + "temperature": 1.0, + "response_format": "auto" + } + - title: Files + request: + curl: | + curl https://api.openai.com/v1/assistants \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "instructions": "You are an HR bot, and you have access to files to answer employee questions about company policies.", + "tools": [{"type": "file_search"}], + "tool_resources": {"file_search": {"vector_store_ids": ["vs_123"]}}, + "model": "gpt-4-turbo" + }' + python: | + from openai import OpenAI + client = OpenAI() + + my_assistant = client.beta.assistants.create( + instructions="You are an HR bot, and you have access to files to answer employee questions about company policies.", + name="HR Helper", + tools=[{"type": "file_search"}], + tool_resources={"file_search": {"vector_store_ids": ["vs_123"]}}, + model="gpt-4-turbo" + ) + print(my_assistant) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const myAssistant = await openai.beta.assistants.create({ + instructions: + "You are an HR bot, and you have access to files to answer employee questions about company policies.", + name: "HR Helper", + tools: [{ type: "file_search" }], + tool_resources: { + file_search: { + vector_store_ids: ["vs_123"] + } + }, + model: "gpt-4-turbo" + }); + + console.log(myAssistant); + } + + main(); + response: | + { + "id": "asst_abc123", + "object": "assistant", + "created_at": 1699009403, + "name": "HR Helper", + "description": null, + "model": "gpt-4-turbo", + "instructions": "You are an HR bot, and you have access to files to answer employee questions about company policies.", + "tools": [ + { + "type": "file_search" + } + ], + "tool_resources": { + "file_search": { + "vector_store_ids": ["vs_123"] + } + }, + "metadata": {}, + "top_p": 1.0, + "temperature": 1.0, + "response_format": "auto" + } + + /assistants/{assistant_id}: + get: + operationId: getAssistant + tags: + - Assistants + summary: Retrieves an assistant. + parameters: + - in: path + name: assistant_id + required: true + schema: + type: string + description: The ID of the assistant to retrieve. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/AssistantObject" + x-oaiMeta: + name: Retrieve assistant + group: assistants + beta: true + returns: The [assistant](/docs/api-reference/assistants/object) object matching the specified ID. + examples: + request: + curl: | + curl https://api.openai.com/v1/assistants/asst_abc123 \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + my_assistant = client.beta.assistants.retrieve("asst_abc123") + print(my_assistant) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const myAssistant = await openai.beta.assistants.retrieve( + "asst_abc123" + ); + + console.log(myAssistant); + } + + main(); + response: | + { + "id": "asst_abc123", + "object": "assistant", + "created_at": 1699009709, + "name": "HR Helper", + "description": null, + "model": "gpt-4-turbo", + "instructions": "You are an HR bot, and you have access to files to answer employee questions about company policies.", + "tools": [ + { + "type": "file_search" + } + ], + "metadata": {}, + "top_p": 1.0, + "temperature": 1.0, + "response_format": "auto" + } + post: + operationId: modifyAssistant + tags: + - Assistants + summary: Modifies an assistant. + parameters: + - in: path + name: assistant_id + required: true + schema: + type: string + description: The ID of the assistant to modify. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ModifyAssistantRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/AssistantObject" + x-oaiMeta: + name: Modify assistant + group: assistants + beta: true + returns: The modified [assistant](/docs/api-reference/assistants/object) object. + examples: + request: + curl: | + curl https://api.openai.com/v1/assistants/asst_abc123 \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "instructions": "You are an HR bot, and you have access to files to answer employee questions about company policies. Always response with info from either of the files.", + "tools": [{"type": "file_search"}], + "model": "gpt-4-turbo" + }' + python: | + from openai import OpenAI + client = OpenAI() + + my_updated_assistant = client.beta.assistants.update( + "asst_abc123", + instructions="You are an HR bot, and you have access to files to answer employee questions about company policies. Always response with info from either of the files.", + name="HR Helper", + tools=[{"type": "file_search"}], + model="gpt-4-turbo" + ) + + print(my_updated_assistant) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const myUpdatedAssistant = await openai.beta.assistants.update( + "asst_abc123", + { + instructions: + "You are an HR bot, and you have access to files to answer employee questions about company policies. Always response with info from either of the files.", + name: "HR Helper", + tools: [{ type: "file_search" }], + model: "gpt-4-turbo" + } + ); + + console.log(myUpdatedAssistant); + } + + main(); + response: | + { + "id": "asst_123", + "object": "assistant", + "created_at": 1699009709, + "name": "HR Helper", + "description": null, + "model": "gpt-4-turbo", + "instructions": "You are an HR bot, and you have access to files to answer employee questions about company policies. Always response with info from either of the files.", + "tools": [ + { + "type": "file_search" + } + ], + "tool_resources": { + "file_search": { + "vector_store_ids": [] + } + }, + "metadata": {}, + "top_p": 1.0, + "temperature": 1.0, + "response_format": "auto" + } + delete: + operationId: deleteAssistant + tags: + - Assistants + summary: Delete an assistant. + parameters: + - in: path + name: assistant_id + required: true + schema: + type: string + description: The ID of the assistant to delete. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/DeleteAssistantResponse" + x-oaiMeta: + name: Delete assistant + group: assistants + beta: true + returns: Deletion status + examples: + request: + curl: | + curl https://api.openai.com/v1/assistants/asst_abc123 \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" \ + -X DELETE + python: | + from openai import OpenAI + client = OpenAI() + + response = client.beta.assistants.delete("asst_abc123") + print(response) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const response = await openai.beta.assistants.del("asst_abc123"); + + console.log(response); + } + main(); + response: | + { + "id": "asst_abc123", + "object": "assistant.deleted", + "deleted": true + } + + /threads: + post: + operationId: createThread + tags: + - Assistants + summary: Create a thread. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CreateThreadRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ThreadObject" + x-oaiMeta: + name: Create thread + group: threads + beta: true + returns: A [thread](/docs/api-reference/threads) object. + examples: + - title: Empty + request: + curl: | + curl https://api.openai.com/v1/threads \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '' + python: | + from openai import OpenAI + client = OpenAI() + + empty_thread = client.beta.threads.create() + print(empty_thread) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const emptyThread = await openai.beta.threads.create(); + + console.log(emptyThread); + } + + main(); + response: | + { + "id": "thread_abc123", + "object": "thread", + "created_at": 1699012949, + "metadata": {}, + "tool_resources": {} + } + - title: Messages + request: + curl: | + curl https://api.openai.com/v1/threads \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "messages": [{ + "role": "user", + "content": "Hello, what is AI?" + }, { + "role": "user", + "content": "How does AI work? Explain it in simple terms." + }] + }' + python: | + from openai import OpenAI + client = OpenAI() + + message_thread = client.beta.threads.create( + messages=[ + { + "role": "user", + "content": "Hello, what is AI?" + }, + { + "role": "user", + "content": "How does AI work? Explain it in simple terms." + }, + ] + ) + + print(message_thread) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const messageThread = await openai.beta.threads.create({ + messages: [ + { + role: "user", + content: "Hello, what is AI?" + }, + { + role: "user", + content: "How does AI work? Explain it in simple terms.", + }, + ], + }); + + console.log(messageThread); + } + + main(); + response: | + { + "id": "thread_abc123", + "object": "thread", + "created_at": 1699014083, + "metadata": {}, + "tool_resources": {} + } + + /threads/{thread_id}: + get: + operationId: getThread + tags: + - Assistants + summary: Retrieves a thread. + parameters: + - in: path + name: thread_id + required: true + schema: + type: string + description: The ID of the thread to retrieve. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ThreadObject" + x-oaiMeta: + name: Retrieve thread + group: threads + beta: true + returns: The [thread](/docs/api-reference/threads/object) object matching the specified ID. + examples: + request: + curl: | + curl https://api.openai.com/v1/threads/thread_abc123 \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + my_thread = client.beta.threads.retrieve("thread_abc123") + print(my_thread) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const myThread = await openai.beta.threads.retrieve( + "thread_abc123" + ); + + console.log(myThread); + } + + main(); + response: | + { + "id": "thread_abc123", + "object": "thread", + "created_at": 1699014083, + "metadata": {}, + "tool_resources": { + "code_interpreter": { + "file_ids": [] + } + } + } + post: + operationId: modifyThread + tags: + - Assistants + summary: Modifies a thread. + parameters: + - in: path + name: thread_id + required: true + schema: + type: string + description: The ID of the thread to modify. Only the `metadata` can be modified. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ModifyThreadRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ThreadObject" + x-oaiMeta: + name: Modify thread + group: threads + beta: true + returns: The modified [thread](/docs/api-reference/threads/object) object matching the specified ID. + examples: + request: + curl: | + curl https://api.openai.com/v1/threads/thread_abc123 \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "metadata": { + "modified": "true", + "user": "abc123" + } + }' + python: | + from openai import OpenAI + client = OpenAI() + + my_updated_thread = client.beta.threads.update( + "thread_abc123", + metadata={ + "modified": "true", + "user": "abc123" + } + ) + print(my_updated_thread) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const updatedThread = await openai.beta.threads.update( + "thread_abc123", + { + metadata: { modified: "true", user: "abc123" }, + } + ); + + console.log(updatedThread); + } + + main(); + response: | + { + "id": "thread_abc123", + "object": "thread", + "created_at": 1699014083, + "metadata": { + "modified": "true", + "user": "abc123" + }, + "tool_resources": {} + } + delete: + operationId: deleteThread + tags: + - Assistants + summary: Delete a thread. + parameters: + - in: path + name: thread_id + required: true + schema: + type: string + description: The ID of the thread to delete. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/DeleteThreadResponse" + x-oaiMeta: + name: Delete thread + group: threads + beta: true + returns: Deletion status + examples: + request: + curl: | + curl https://api.openai.com/v1/threads/thread_abc123 \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" \ + -X DELETE + python: | + from openai import OpenAI + client = OpenAI() + + response = client.beta.threads.delete("thread_abc123") + print(response) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const response = await openai.beta.threads.del("thread_abc123"); + + console.log(response); + } + main(); + response: | + { + "id": "thread_abc123", + "object": "thread.deleted", + "deleted": true + } + + /threads/{thread_id}/messages: + get: + operationId: listMessages + tags: + - Assistants + summary: Returns a list of messages for a given thread. + parameters: + - in: path + name: thread_id + required: true + schema: + type: string + description: The ID of the [thread](/docs/api-reference/threads) the messages belong to. + - name: limit + in: query + description: *pagination_limit_param_description + required: false + schema: + type: integer + default: 20 + - name: order + in: query + description: *pagination_order_param_description + schema: + type: string + default: desc + enum: ["asc", "desc"] + - name: after + in: query + description: *pagination_after_param_description + schema: + type: string + - name: before + in: query + description: *pagination_before_param_description + schema: + type: string + - name: run_id + in: query + description: | + Filter messages by the run ID that generated them. + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListMessagesResponse" + x-oaiMeta: + name: List messages + group: threads + beta: true + returns: A list of [message](/docs/api-reference/messages) objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/threads/thread_abc123/messages \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + thread_messages = client.beta.threads.messages.list("thread_abc123") + print(thread_messages.data) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const threadMessages = await openai.beta.threads.messages.list( + "thread_abc123" + ); + + console.log(threadMessages.data); + } + + main(); + response: | + { + "object": "list", + "data": [ + { + "id": "msg_abc123", + "object": "thread.message", + "created_at": 1699016383, + "assistant_id": null, + "thread_id": "thread_abc123", + "run_id": null, + "role": "user", + "content": [ + { + "type": "text", + "text": { + "value": "How does AI work? Explain it in simple terms.", + "annotations": [] + } + } + ], + "attachments": [], + "metadata": {} + }, + { + "id": "msg_abc456", + "object": "thread.message", + "created_at": 1699016383, + "assistant_id": null, + "thread_id": "thread_abc123", + "run_id": null, + "role": "user", + "content": [ + { + "type": "text", + "text": { + "value": "Hello, what is AI?", + "annotations": [] + } + } + ], + "attachments": [], + "metadata": {} + } + ], + "first_id": "msg_abc123", + "last_id": "msg_abc456", + "has_more": false + } + post: + operationId: createMessage + tags: + - Assistants + summary: Create a message. + parameters: + - in: path + name: thread_id + required: true + schema: + type: string + description: The ID of the [thread](/docs/api-reference/threads) to create a message for. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateMessageRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/MessageObject" + x-oaiMeta: + name: Create message + group: threads + beta: true + returns: A [message](/docs/api-reference/messages/object) object. + examples: + request: + curl: | + curl https://api.openai.com/v1/threads/thread_abc123/messages \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "role": "user", + "content": "How does AI work? Explain it in simple terms." + }' + python: | + from openai import OpenAI + client = OpenAI() + + thread_message = client.beta.threads.messages.create( + "thread_abc123", + role="user", + content="How does AI work? Explain it in simple terms.", + ) + print(thread_message) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const threadMessages = await openai.beta.threads.messages.create( + "thread_abc123", + { role: "user", content: "How does AI work? Explain it in simple terms." } + ); + + console.log(threadMessages); + } + + main(); + response: | + { + "id": "msg_abc123", + "object": "thread.message", + "created_at": 1713226573, + "assistant_id": null, + "thread_id": "thread_abc123", + "run_id": null, + "role": "user", + "content": [ + { + "type": "text", + "text": { + "value": "How does AI work? Explain it in simple terms.", + "annotations": [] + } + } + ], + "attachments": [], + "metadata": {} + } + + /threads/{thread_id}/messages/{message_id}: + get: + operationId: getMessage + tags: + - Assistants + summary: Retrieve a message. + parameters: + - in: path + name: thread_id + required: true + schema: + type: string + description: The ID of the [thread](/docs/api-reference/threads) to which this message belongs. + - in: path + name: message_id + required: true + schema: + type: string + description: The ID of the message to retrieve. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/MessageObject" + x-oaiMeta: + name: Retrieve message + group: threads + beta: true + returns: The [message](/docs/api-reference/threads/messages/object) object matching the specified ID. + examples: + request: + curl: | + curl https://api.openai.com/v1/threads/thread_abc123/messages/msg_abc123 \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + message = client.beta.threads.messages.retrieve( + message_id="msg_abc123", + thread_id="thread_abc123", + ) + print(message) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const message = await openai.beta.threads.messages.retrieve( + "thread_abc123", + "msg_abc123" + ); + + console.log(message); + } + + main(); + response: | + { + "id": "msg_abc123", + "object": "thread.message", + "created_at": 1699017614, + "assistant_id": null, + "thread_id": "thread_abc123", + "run_id": null, + "role": "user", + "content": [ + { + "type": "text", + "text": { + "value": "How does AI work? Explain it in simple terms.", + "annotations": [] + } + } + ], + "attachments": [], + "metadata": {} + } + post: + operationId: modifyMessage + tags: + - Assistants + summary: Modifies a message. + parameters: + - in: path + name: thread_id + required: true + schema: + type: string + description: The ID of the thread to which this message belongs. + - in: path + name: message_id + required: true + schema: + type: string + description: The ID of the message to modify. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ModifyMessageRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/MessageObject" + x-oaiMeta: + name: Modify message + group: threads + beta: true + returns: The modified [message](/docs/api-reference/threads/messages/object) object. + examples: + request: + curl: | + curl https://api.openai.com/v1/threads/thread_abc123/messages/msg_abc123 \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "metadata": { + "modified": "true", + "user": "abc123" + } + }' + python: | + from openai import OpenAI + client = OpenAI() + + message = client.beta.threads.messages.update( + message_id="msg_abc12", + thread_id="thread_abc123", + metadata={ + "modified": "true", + "user": "abc123", + }, + ) + print(message) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const message = await openai.beta.threads.messages.update( + "thread_abc123", + "msg_abc123", + { + metadata: { + modified: "true", + user: "abc123", + }, + } + }' + response: | + { + "id": "msg_abc123", + "object": "thread.message", + "created_at": 1699017614, + "assistant_id": null, + "thread_id": "thread_abc123", + "run_id": null, + "role": "user", + "content": [ + { + "type": "text", + "text": { + "value": "How does AI work? Explain it in simple terms.", + "annotations": [] + } + } + ], + "file_ids": [], + "metadata": { + "modified": "true", + "user": "abc123" + } + } + delete: + operationId: deleteMessage + tags: + - Assistants + summary: Deletes a message. + parameters: + - in: path + name: thread_id + required: true + schema: + type: string + description: The ID of the thread to which this message belongs. + - in: path + name: message_id + required: true + schema: + type: string + description: The ID of the message to delete. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/DeleteMessageResponse" + x-oaiMeta: + name: Delete message + group: threads + beta: true + returns: Deletion status + examples: + request: + curl: | + curl -X DELETE https://api.openai.com/v1/threads/thread_abc123/messages/msg_abc123 \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + deleted_message = client.beta.threads.messages.delete( + message_id="msg_abc12", + thread_id="thread_abc123", + ) + print(deleted_message) + node.js: |- + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const deletedMessage = await openai.beta.threads.messages.del( + "thread_abc123", + "msg_abc123" + ); + + console.log(deletedMessage); + } + response: | + { + "id": "msg_abc123", + "object": "thread.message.deleted", + "deleted": true + } + + + /threads/runs: + post: + operationId: createThreadAndRun + tags: + - Assistants + summary: Create a thread and run it in one request. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateThreadAndRunRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/RunObject" + x-oaiMeta: + name: Create thread and run + group: threads + beta: true + returns: A [run](/docs/api-reference/runs/object) object. + examples: + - title: Default + request: + curl: | + curl https://api.openai.com/v1/threads/runs \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "assistant_id": "asst_abc123", + "thread": { + "messages": [ + {"role": "user", "content": "Explain deep learning to a 5 year old."} + ] + } + }' + python: | + from openai import OpenAI + client = OpenAI() + + run = client.beta.threads.create_and_run( + assistant_id="asst_abc123", + thread={ + "messages": [ + {"role": "user", "content": "Explain deep learning to a 5 year old."} + ] + } + ) + + print(run) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const run = await openai.beta.threads.createAndRun({ + assistant_id: "asst_abc123", + thread: { + messages: [ + { role: "user", content: "Explain deep learning to a 5 year old." }, + ], + }, + }); + + console.log(run); + } + + main(); + response: | + { + "id": "run_abc123", + "object": "thread.run", + "created_at": 1699076792, + "assistant_id": "asst_abc123", + "thread_id": "thread_abc123", + "status": "queued", + "started_at": null, + "expires_at": 1699077392, + "cancelled_at": null, + "failed_at": null, + "completed_at": null, + "required_action": null, + "last_error": null, + "model": "gpt-4-turbo", + "instructions": "You are a helpful assistant.", + "tools": [], + "tool_resources": {}, + "metadata": {}, + "temperature": 1.0, + "top_p": 1.0, + "max_completion_tokens": null, + "max_prompt_tokens": null, + "truncation_strategy": { + "type": "auto", + "last_messages": null + }, + "incomplete_details": null, + "usage": null, + "response_format": "auto", + "tool_choice": "auto" + } + + - title: Streaming + request: + curl: | + curl https://api.openai.com/v1/threads/runs \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "assistant_id": "asst_123", + "thread": { + "messages": [ + {"role": "user", "content": "Hello"} + ] + }, + "stream": true + }' + python: | + from openai import OpenAI + client = OpenAI() + + stream = client.beta.threads.create_and_run( + assistant_id="asst_123", + thread={ + "messages": [ + {"role": "user", "content": "Hello"} + ] + }, + stream=True + ) + + for event in stream: + print(event) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const stream = await openai.beta.threads.createAndRun({ + assistant_id: "asst_123", + thread: { + messages: [ + { role: "user", content: "Hello" }, + ], + }, + stream: true + }); + + for await (const event of stream) { + console.log(event); + } + } + + main(); + response: | + event: thread.created + data: {"id":"thread_123","object":"thread","created_at":1710348075,"metadata":{}} + + event: thread.run.created + data: {"id":"run_123","object":"thread.run","created_at":1710348075,"assistant_id":"asst_123","thread_id":"thread_123","status":"queued","started_at":null,"expires_at":1710348675,"cancelled_at":null,"failed_at":null,"completed_at":null,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[],"tool_resources":{},"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":null,"response_format":"auto","tool_choice":"auto"} + + event: thread.run.queued + data: {"id":"run_123","object":"thread.run","created_at":1710348075,"assistant_id":"asst_123","thread_id":"thread_123","status":"queued","started_at":null,"expires_at":1710348675,"cancelled_at":null,"failed_at":null,"completed_at":null,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[],"tool_resources":{},"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":null,"response_format":"auto","tool_choice":"auto"} + + event: thread.run.in_progress + data: {"id":"run_123","object":"thread.run","created_at":1710348075,"assistant_id":"asst_123","thread_id":"thread_123","status":"in_progress","started_at":null,"expires_at":1710348675,"cancelled_at":null,"failed_at":null,"completed_at":null,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[],"tool_resources":{},"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":null,"response_format":"auto","tool_choice":"auto"} + + event: thread.run.step.created + data: {"id":"step_001","object":"thread.run.step","created_at":1710348076,"run_id":"run_123","assistant_id":"asst_123","thread_id":"thread_123","type":"message_creation","status":"in_progress","cancelled_at":null,"completed_at":null,"expires_at":1710348675,"failed_at":null,"last_error":null,"step_details":{"type":"message_creation","message_creation":{"message_id":"msg_001"}},"usage":null} + + event: thread.run.step.in_progress + data: {"id":"step_001","object":"thread.run.step","created_at":1710348076,"run_id":"run_123","assistant_id":"asst_123","thread_id":"thread_123","type":"message_creation","status":"in_progress","cancelled_at":null,"completed_at":null,"expires_at":1710348675,"failed_at":null,"last_error":null,"step_details":{"type":"message_creation","message_creation":{"message_id":"msg_001"}},"usage":null} + + event: thread.message.created + data: {"id":"msg_001","object":"thread.message","created_at":1710348076,"assistant_id":"asst_123","thread_id":"thread_123","run_id":"run_123","status":"in_progress","incomplete_details":null,"incomplete_at":null,"completed_at":null,"role":"assistant","content":[], "metadata":{}} + + event: thread.message.in_progress + data: {"id":"msg_001","object":"thread.message","created_at":1710348076,"assistant_id":"asst_123","thread_id":"thread_123","run_id":"run_123","status":"in_progress","incomplete_details":null,"incomplete_at":null,"completed_at":null,"role":"assistant","content":[], "metadata":{}} + + event: thread.message.delta + data: {"id":"msg_001","object":"thread.message.delta","delta":{"content":[{"index":0,"type":"text","text":{"value":"Hello","annotations":[]}}]}} + + ... + + event: thread.message.delta + data: {"id":"msg_001","object":"thread.message.delta","delta":{"content":[{"index":0,"type":"text","text":{"value":" today"}}]}} + + event: thread.message.delta + data: {"id":"msg_001","object":"thread.message.delta","delta":{"content":[{"index":0,"type":"text","text":{"value":"?"}}]}} + + event: thread.message.completed + data: {"id":"msg_001","object":"thread.message","created_at":1710348076,"assistant_id":"asst_123","thread_id":"thread_123","run_id":"run_123","status":"completed","incomplete_details":null,"incomplete_at":null,"completed_at":1710348077,"role":"assistant","content":[{"type":"text","text":{"value":"Hello! How can I assist you today?","annotations":[]}}], "metadata":{}} + + event: thread.run.step.completed + data: {"id":"step_001","object":"thread.run.step","created_at":1710348076,"run_id":"run_123","assistant_id":"asst_123","thread_id":"thread_123","type":"message_creation","status":"completed","cancelled_at":null,"completed_at":1710348077,"expires_at":1710348675,"failed_at":null,"last_error":null,"step_details":{"type":"message_creation","message_creation":{"message_id":"msg_001"}},"usage":{"prompt_tokens":20,"completion_tokens":11,"total_tokens":31}} + + event: thread.run.completed + {"id":"run_123","object":"thread.run","created_at":1710348076,"assistant_id":"asst_123","thread_id":"thread_123","status":"completed","started_at":1713226836,"expires_at":null,"cancelled_at":null,"failed_at":null,"completed_at":1713226837,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":{"prompt_tokens":345,"completion_tokens":11,"total_tokens":356},"response_format":"auto","tool_choice":"auto"} + + event: done + data: [DONE] + + - title: Streaming with Functions + request: + curl: | + curl https://api.openai.com/v1/threads/runs \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "assistant_id": "asst_abc123", + "thread": { + "messages": [ + {"role": "user", "content": "What is the weather like in San Francisco?"} + ] + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"] + } + }, + "required": ["location"] + } + } + } + ], + "stream": true + }' + python: | + from openai import OpenAI + client = OpenAI() + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + } + ] + + stream = client.beta.threads.create_and_run( + thread={ + "messages": [ + {"role": "user", "content": "What is the weather like in San Francisco?"} + ] + }, + assistant_id="asst_abc123", + tools=tools, + stream=True + ) + + for event in stream: + print(event) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + const tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + } + ]; + + async function main() { + const stream = await openai.beta.threads.createAndRun({ + assistant_id: "asst_123", + thread: { + messages: [ + { role: "user", content: "What is the weather like in San Francisco?" }, + ], + }, + tools: tools, + stream: true + }); + + for await (const event of stream) { + console.log(event); + } + } + + main(); + response: | + event: thread.created + data: {"id":"thread_123","object":"thread","created_at":1710351818,"metadata":{}} + + event: thread.run.created + data: {"id":"run_123","object":"thread.run","created_at":1710351818,"assistant_id":"asst_123","thread_id":"thread_123","status":"queued","started_at":null,"expires_at":1710352418,"cancelled_at":null,"failed_at":null,"completed_at":null,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather in a given location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["location"]}}}],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":null,"response_format":"auto","tool_choice":"auto"}} + + event: thread.run.queued + data: {"id":"run_123","object":"thread.run","created_at":1710351818,"assistant_id":"asst_123","thread_id":"thread_123","status":"queued","started_at":null,"expires_at":1710352418,"cancelled_at":null,"failed_at":null,"completed_at":null,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather in a given location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["location"]}}}],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":null,"response_format":"auto","tool_choice":"auto"}} + + event: thread.run.in_progress + data: {"id":"run_123","object":"thread.run","created_at":1710351818,"assistant_id":"asst_123","thread_id":"thread_123","status":"in_progress","started_at":1710351818,"expires_at":1710352418,"cancelled_at":null,"failed_at":null,"completed_at":null,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather in a given location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["location"]}}}],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":null,"response_format":"auto","tool_choice":"auto"}} + + event: thread.run.step.created + data: {"id":"step_001","object":"thread.run.step","created_at":1710351819,"run_id":"run_123","assistant_id":"asst_123","thread_id":"thread_123","type":"tool_calls","status":"in_progress","cancelled_at":null,"completed_at":null,"expires_at":1710352418,"failed_at":null,"last_error":null,"step_details":{"type":"tool_calls","tool_calls":[]},"usage":null} + + event: thread.run.step.in_progress + data: {"id":"step_001","object":"thread.run.step","created_at":1710351819,"run_id":"run_123","assistant_id":"asst_123","thread_id":"thread_123","type":"tool_calls","status":"in_progress","cancelled_at":null,"completed_at":null,"expires_at":1710352418,"failed_at":null,"last_error":null,"step_details":{"type":"tool_calls","tool_calls":[]},"usage":null} + + event: thread.run.step.delta + data: {"id":"step_001","object":"thread.run.step.delta","delta":{"step_details":{"type":"tool_calls","tool_calls":[{"index":0,"id":"call_XXNp8YGaFrjrSjgqxtC8JJ1B","type":"function","function":{"name":"get_current_weather","arguments":"","output":null}}]}}} + + event: thread.run.step.delta + data: {"id":"step_001","object":"thread.run.step.delta","delta":{"step_details":{"type":"tool_calls","tool_calls":[{"index":0,"type":"function","function":{"arguments":"{\""}}]}}} + + event: thread.run.step.delta + data: {"id":"step_001","object":"thread.run.step.delta","delta":{"step_details":{"type":"tool_calls","tool_calls":[{"index":0,"type":"function","function":{"arguments":"location"}}]}}} + + ... + + event: thread.run.step.delta + data: {"id":"step_001","object":"thread.run.step.delta","delta":{"step_details":{"type":"tool_calls","tool_calls":[{"index":0,"type":"function","function":{"arguments":"ahrenheit"}}]}}} + + event: thread.run.step.delta + data: {"id":"step_001","object":"thread.run.step.delta","delta":{"step_details":{"type":"tool_calls","tool_calls":[{"index":0,"type":"function","function":{"arguments":"\"}"}}]}}} + + event: thread.run.requires_action + data: {"id":"run_123","object":"thread.run","created_at":1710351818,"assistant_id":"asst_123","thread_id":"thread_123","status":"requires_action","started_at":1710351818,"expires_at":1710352418,"cancelled_at":null,"failed_at":null,"completed_at":null,"required_action":{"type":"submit_tool_outputs","submit_tool_outputs":{"tool_calls":[{"id":"call_XXNp8YGaFrjrSjgqxtC8JJ1B","type":"function","function":{"name":"get_current_weather","arguments":"{\"location\":\"San Francisco, CA\",\"unit\":\"fahrenheit\"}"}}]}},"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather in a given location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["location"]}}}],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":{"prompt_tokens":345,"completion_tokens":11,"total_tokens":356},"response_format":"auto","tool_choice":"auto"}} + + event: done + data: [DONE] + + /threads/{thread_id}/runs: + get: + operationId: listRuns + tags: + - Assistants + summary: Returns a list of runs belonging to a thread. + parameters: + - name: thread_id + in: path + required: true + schema: + type: string + description: The ID of the thread the run belongs to. + - name: limit + in: query + description: *pagination_limit_param_description + required: false + schema: + type: integer + default: 20 + - name: order + in: query + description: *pagination_order_param_description + schema: + type: string + default: desc + enum: ["asc", "desc"] + - name: after + in: query + description: *pagination_after_param_description + schema: + type: string + - name: before + in: query + description: *pagination_before_param_description + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListRunsResponse" + x-oaiMeta: + name: List runs + group: threads + beta: true + returns: A list of [run](/docs/api-reference/runs/object) objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/threads/thread_abc123/runs \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + runs = client.beta.threads.runs.list( + "thread_abc123" + ) + + print(runs) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const runs = await openai.beta.threads.runs.list( + "thread_abc123" + ); + + console.log(runs); + } + + main(); + response: | + { + "object": "list", + "data": [ + { + "id": "run_abc123", + "object": "thread.run", + "created_at": 1699075072, + "assistant_id": "asst_abc123", + "thread_id": "thread_abc123", + "status": "completed", + "started_at": 1699075072, + "expires_at": null, + "cancelled_at": null, + "failed_at": null, + "completed_at": 1699075073, + "last_error": null, + "model": "gpt-4-turbo", + "instructions": null, + "incomplete_details": null, + "tools": [ + { + "type": "code_interpreter" + } + ], + "tool_resources": { + "code_interpreter": { + "file_ids": [ + "file-abc123", + "file-abc456" + ] + } + }, + "metadata": {}, + "usage": { + "prompt_tokens": 123, + "completion_tokens": 456, + "total_tokens": 579 + }, + "temperature": 1.0, + "top_p": 1.0, + "max_prompt_tokens": 1000, + "max_completion_tokens": 1000, + "truncation_strategy": { + "type": "auto", + "last_messages": null + }, + "response_format": "auto", + "tool_choice": "auto" + }, + { + "id": "run_abc456", + "object": "thread.run", + "created_at": 1699063290, + "assistant_id": "asst_abc123", + "thread_id": "thread_abc123", + "status": "completed", + "started_at": 1699063290, + "expires_at": null, + "cancelled_at": null, + "failed_at": null, + "completed_at": 1699063291, + "last_error": null, + "model": "gpt-4-turbo", + "instructions": null, + "incomplete_details": null, + "tools": [ + { + "type": "code_interpreter" + } + ], + "tool_resources": { + "code_interpreter": { + "file_ids": [ + "file-abc123", + "file-abc456" + ] + } + }, + "metadata": {}, + "usage": { + "prompt_tokens": 123, + "completion_tokens": 456, + "total_tokens": 579 + }, + "temperature": 1.0, + "top_p": 1.0, + "max_prompt_tokens": 1000, + "max_completion_tokens": 1000, + "truncation_strategy": { + "type": "auto", + "last_messages": null + }, + "response_format": "auto", + "tool_choice": "auto" + } + ], + "first_id": "run_abc123", + "last_id": "run_abc456", + "has_more": false + } + post: + operationId: createRun + tags: + - Assistants + summary: Create a run. + parameters: + - in: path + name: thread_id + required: true + schema: + type: string + description: The ID of the thread to run. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateRunRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/RunObject" + x-oaiMeta: + name: Create run + group: threads + beta: true + returns: A [run](/docs/api-reference/runs/object) object. + examples: + - title: Default + request: + curl: | + curl https://api.openai.com/v1/threads/thread_abc123/runs \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "assistant_id": "asst_abc123" + }' + python: | + from openai import OpenAI + client = OpenAI() + + run = client.beta.threads.runs.create( + thread_id="thread_abc123", + assistant_id="asst_abc123" + ) + + print(run) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const run = await openai.beta.threads.runs.create( + "thread_abc123", + { assistant_id: "asst_abc123" } + ); + + console.log(run); + } + + main(); + response: &run_object_example | + { + "id": "run_abc123", + "object": "thread.run", + "created_at": 1699063290, + "assistant_id": "asst_abc123", + "thread_id": "thread_abc123", + "status": "queued", + "started_at": 1699063290, + "expires_at": null, + "cancelled_at": null, + "failed_at": null, + "completed_at": 1699063291, + "last_error": null, + "model": "gpt-4-turbo", + "instructions": null, + "incomplete_details": null, + "tools": [ + { + "type": "code_interpreter" + } + ], + "metadata": {}, + "usage": null, + "temperature": 1.0, + "top_p": 1.0, + "max_prompt_tokens": 1000, + "max_completion_tokens": 1000, + "truncation_strategy": { + "type": "auto", + "last_messages": null + }, + "response_format": "auto", + "tool_choice": "auto" + } + - title: Streaming + request: + curl: | + curl https://api.openai.com/v1/threads/thread_123/runs \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "assistant_id": "asst_123", + "stream": true + }' + python: | + from openai import OpenAI + client = OpenAI() + + stream = client.beta.threads.runs.create( + thread_id="thread_123", + assistant_id="asst_123", + stream=True + ) + + for event in stream: + print(event) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const stream = await openai.beta.threads.runs.create( + "thread_123", + { assistant_id: "asst_123", stream: true } + ); + + for await (const event of stream) { + console.log(event); + } + } + + main(); + response: | + event: thread.run.created + data: {"id":"run_123","object":"thread.run","created_at":1710330640,"assistant_id":"asst_123","thread_id":"thread_123","status":"queued","started_at":null,"expires_at":1710331240,"cancelled_at":null,"failed_at":null,"completed_at":null,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":null,"response_format":"auto","tool_choice":"auto"}} + + event: thread.run.queued + data: {"id":"run_123","object":"thread.run","created_at":1710330640,"assistant_id":"asst_123","thread_id":"thread_123","status":"queued","started_at":null,"expires_at":1710331240,"cancelled_at":null,"failed_at":null,"completed_at":null,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":null,"response_format":"auto","tool_choice":"auto"}} + + event: thread.run.in_progress + data: {"id":"run_123","object":"thread.run","created_at":1710330640,"assistant_id":"asst_123","thread_id":"thread_123","status":"in_progress","started_at":1710330641,"expires_at":1710331240,"cancelled_at":null,"failed_at":null,"completed_at":null,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":null,"response_format":"auto","tool_choice":"auto"}} + + event: thread.run.step.created + data: {"id":"step_001","object":"thread.run.step","created_at":1710330641,"run_id":"run_123","assistant_id":"asst_123","thread_id":"thread_123","type":"message_creation","status":"in_progress","cancelled_at":null,"completed_at":null,"expires_at":1710331240,"failed_at":null,"last_error":null,"step_details":{"type":"message_creation","message_creation":{"message_id":"msg_001"}},"usage":null} + + event: thread.run.step.in_progress + data: {"id":"step_001","object":"thread.run.step","created_at":1710330641,"run_id":"run_123","assistant_id":"asst_123","thread_id":"thread_123","type":"message_creation","status":"in_progress","cancelled_at":null,"completed_at":null,"expires_at":1710331240,"failed_at":null,"last_error":null,"step_details":{"type":"message_creation","message_creation":{"message_id":"msg_001"}},"usage":null} + + event: thread.message.created + data: {"id":"msg_001","object":"thread.message","created_at":1710330641,"assistant_id":"asst_123","thread_id":"thread_123","run_id":"run_123","status":"in_progress","incomplete_details":null,"incomplete_at":null,"completed_at":null,"role":"assistant","content":[],"metadata":{}} + + event: thread.message.in_progress + data: {"id":"msg_001","object":"thread.message","created_at":1710330641,"assistant_id":"asst_123","thread_id":"thread_123","run_id":"run_123","status":"in_progress","incomplete_details":null,"incomplete_at":null,"completed_at":null,"role":"assistant","content":[],"metadata":{}} + + event: thread.message.delta + data: {"id":"msg_001","object":"thread.message.delta","delta":{"content":[{"index":0,"type":"text","text":{"value":"Hello","annotations":[]}}]}} + + ... + + event: thread.message.delta + data: {"id":"msg_001","object":"thread.message.delta","delta":{"content":[{"index":0,"type":"text","text":{"value":" today"}}]}} + + event: thread.message.delta + data: {"id":"msg_001","object":"thread.message.delta","delta":{"content":[{"index":0,"type":"text","text":{"value":"?"}}]}} + + event: thread.message.completed + data: {"id":"msg_001","object":"thread.message","created_at":1710330641,"assistant_id":"asst_123","thread_id":"thread_123","run_id":"run_123","status":"completed","incomplete_details":null,"incomplete_at":null,"completed_at":1710330642,"role":"assistant","content":[{"type":"text","text":{"value":"Hello! How can I assist you today?","annotations":[]}}],"metadata":{}} + + event: thread.run.step.completed + data: {"id":"step_001","object":"thread.run.step","created_at":1710330641,"run_id":"run_123","assistant_id":"asst_123","thread_id":"thread_123","type":"message_creation","status":"completed","cancelled_at":null,"completed_at":1710330642,"expires_at":1710331240,"failed_at":null,"last_error":null,"step_details":{"type":"message_creation","message_creation":{"message_id":"msg_001"}},"usage":{"prompt_tokens":20,"completion_tokens":11,"total_tokens":31}} + + event: thread.run.completed + data: {"id":"run_123","object":"thread.run","created_at":1710330640,"assistant_id":"asst_123","thread_id":"thread_123","status":"completed","started_at":1710330641,"expires_at":null,"cancelled_at":null,"failed_at":null,"completed_at":1710330642,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":{"prompt_tokens":20,"completion_tokens":11,"total_tokens":31},"response_format":"auto","tool_choice":"auto"}} + + event: done + data: [DONE] + + - title: Streaming with Functions + request: + curl: | + curl https://api.openai.com/v1/threads/thread_abc123/runs \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "assistant_id": "asst_abc123", + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"] + } + }, + "required": ["location"] + } + } + } + ], + "stream": true + }' + python: | + from openai import OpenAI + client = OpenAI() + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + } + ] + + stream = client.beta.threads.runs.create( + thread_id="thread_abc123", + assistant_id="asst_abc123", + tools=tools, + stream=True + ) + + for event in stream: + print(event) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + const tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + } + ]; + + async function main() { + const stream = await openai.beta.threads.runs.create( + "thread_abc123", + { + assistant_id: "asst_abc123", + tools: tools, + stream: true + } + ); + + for await (const event of stream) { + console.log(event); + } + } + + main(); + response: | + event: thread.run.created + data: {"id":"run_123","object":"thread.run","created_at":1710348075,"assistant_id":"asst_123","thread_id":"thread_123","status":"queued","started_at":null,"expires_at":1710348675,"cancelled_at":null,"failed_at":null,"completed_at":null,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":null,"response_format":"auto","tool_choice":"auto"}} + + event: thread.run.queued + data: {"id":"run_123","object":"thread.run","created_at":1710348075,"assistant_id":"asst_123","thread_id":"thread_123","status":"queued","started_at":null,"expires_at":1710348675,"cancelled_at":null,"failed_at":null,"completed_at":null,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":null,"response_format":"auto","tool_choice":"auto"}} + + event: thread.run.in_progress + data: {"id":"run_123","object":"thread.run","created_at":1710348075,"assistant_id":"asst_123","thread_id":"thread_123","status":"in_progress","started_at":1710348075,"expires_at":1710348675,"cancelled_at":null,"failed_at":null,"completed_at":null,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":null,"response_format":"auto","tool_choice":"auto"}} + + event: thread.run.step.created + data: {"id":"step_001","object":"thread.run.step","created_at":1710348076,"run_id":"run_123","assistant_id":"asst_123","thread_id":"thread_123","type":"message_creation","status":"in_progress","cancelled_at":null,"completed_at":null,"expires_at":1710348675,"failed_at":null,"last_error":null,"step_details":{"type":"message_creation","message_creation":{"message_id":"msg_001"}},"usage":null} + + event: thread.run.step.in_progress + data: {"id":"step_001","object":"thread.run.step","created_at":1710348076,"run_id":"run_123","assistant_id":"asst_123","thread_id":"thread_123","type":"message_creation","status":"in_progress","cancelled_at":null,"completed_at":null,"expires_at":1710348675,"failed_at":null,"last_error":null,"step_details":{"type":"message_creation","message_creation":{"message_id":"msg_001"}},"usage":null} + + event: thread.message.created + data: {"id":"msg_001","object":"thread.message","created_at":1710348076,"assistant_id":"asst_123","thread_id":"thread_123","run_id":"run_123","status":"in_progress","incomplete_details":null,"incomplete_at":null,"completed_at":null,"role":"assistant","content":[],"metadata":{}} + + event: thread.message.in_progress + data: {"id":"msg_001","object":"thread.message","created_at":1710348076,"assistant_id":"asst_123","thread_id":"thread_123","run_id":"run_123","status":"in_progress","incomplete_details":null,"incomplete_at":null,"completed_at":null,"role":"assistant","content":[],"metadata":{}} + + event: thread.message.delta + data: {"id":"msg_001","object":"thread.message.delta","delta":{"content":[{"index":0,"type":"text","text":{"value":"Hello","annotations":[]}}]}} + + ... + + event: thread.message.delta + data: {"id":"msg_001","object":"thread.message.delta","delta":{"content":[{"index":0,"type":"text","text":{"value":" today"}}]}} + + event: thread.message.delta + data: {"id":"msg_001","object":"thread.message.delta","delta":{"content":[{"index":0,"type":"text","text":{"value":"?"}}]}} + + event: thread.message.completed + data: {"id":"msg_001","object":"thread.message","created_at":1710348076,"assistant_id":"asst_123","thread_id":"thread_123","run_id":"run_123","status":"completed","incomplete_details":null,"incomplete_at":null,"completed_at":1710348077,"role":"assistant","content":[{"type":"text","text":{"value":"Hello! How can I assist you today?","annotations":[]}}],"metadata":{}} + + event: thread.run.step.completed + data: {"id":"step_001","object":"thread.run.step","created_at":1710348076,"run_id":"run_123","assistant_id":"asst_123","thread_id":"thread_123","type":"message_creation","status":"completed","cancelled_at":null,"completed_at":1710348077,"expires_at":1710348675,"failed_at":null,"last_error":null,"step_details":{"type":"message_creation","message_creation":{"message_id":"msg_001"}},"usage":{"prompt_tokens":20,"completion_tokens":11,"total_tokens":31}} + + event: thread.run.completed + data: {"id":"run_123","object":"thread.run","created_at":1710348075,"assistant_id":"asst_123","thread_id":"thread_123","status":"completed","started_at":1710348075,"expires_at":null,"cancelled_at":null,"failed_at":null,"completed_at":1710348077,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":{"prompt_tokens":20,"completion_tokens":11,"total_tokens":31},"response_format":"auto","tool_choice":"auto"}} + + event: done + data: [DONE] + + /threads/{thread_id}/runs/{run_id}: + get: + operationId: getRun + tags: + - Assistants + summary: Retrieves a run. + parameters: + - in: path + name: thread_id + required: true + schema: + type: string + description: The ID of the [thread](/docs/api-reference/threads) that was run. + - in: path + name: run_id + required: true + schema: + type: string + description: The ID of the run to retrieve. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/RunObject" + x-oaiMeta: + name: Retrieve run + group: threads + beta: true + returns: The [run](/docs/api-reference/runs/object) object matching the specified ID. + examples: + request: + curl: | + curl https://api.openai.com/v1/threads/thread_abc123/runs/run_abc123 \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + run = client.beta.threads.runs.retrieve( + thread_id="thread_abc123", + run_id="run_abc123" + ) + + print(run) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const run = await openai.beta.threads.runs.retrieve( + "thread_abc123", + "run_abc123" + ); + + console.log(run); + } + + main(); + response: | + { + "id": "run_abc123", + "object": "thread.run", + "created_at": 1699075072, + "assistant_id": "asst_abc123", + "thread_id": "thread_abc123", + "status": "completed", + "started_at": 1699075072, + "expires_at": null, + "cancelled_at": null, + "failed_at": null, + "completed_at": 1699075073, + "last_error": null, + "model": "gpt-4-turbo", + "instructions": null, + "incomplete_details": null, + "tools": [ + { + "type": "code_interpreter" + } + ], + "metadata": {}, + "usage": { + "prompt_tokens": 123, + "completion_tokens": 456, + "total_tokens": 579 + }, + "temperature": 1.0, + "top_p": 1.0, + "max_prompt_tokens": 1000, + "max_completion_tokens": 1000, + "truncation_strategy": { + "type": "auto", + "last_messages": null + }, + "response_format": "auto", + "tool_choice": "auto" + } + post: + operationId: modifyRun + tags: + - Assistants + summary: Modifies a run. + parameters: + - in: path + name: thread_id + required: true + schema: + type: string + description: The ID of the [thread](/docs/api-reference/threads) that was run. + - in: path + name: run_id + required: true + schema: + type: string + description: The ID of the run to modify. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ModifyRunRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/RunObject" + x-oaiMeta: + name: Modify run + group: threads + beta: true + returns: The modified [run](/docs/api-reference/runs/object) object matching the specified ID. + examples: + request: + curl: | + curl https://api.openai.com/v1/threads/thread_abc123/runs/run_abc123 \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "metadata": { + "user_id": "user_abc123" + } + }' + python: | + from openai import OpenAI + client = OpenAI() + + run = client.beta.threads.runs.update( + thread_id="thread_abc123", + run_id="run_abc123", + metadata={"user_id": "user_abc123"}, + ) + + print(run) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const run = await openai.beta.threads.runs.update( + "thread_abc123", + "run_abc123", + { + metadata: { + user_id: "user_abc123", + }, + } + ); + + console.log(run); + } + + main(); + response: | + { + "id": "run_abc123", + "object": "thread.run", + "created_at": 1699075072, + "assistant_id": "asst_abc123", + "thread_id": "thread_abc123", + "status": "completed", + "started_at": 1699075072, + "expires_at": null, + "cancelled_at": null, + "failed_at": null, + "completed_at": 1699075073, + "last_error": null, + "model": "gpt-4-turbo", + "instructions": null, + "incomplete_details": null, + "tools": [ + { + "type": "code_interpreter" + } + ], + "tool_resources": { + "code_interpreter": { + "file_ids": [ + "file-abc123", + "file-abc456" + ] + } + }, + "metadata": { + "user_id": "user_abc123" + }, + "usage": { + "prompt_tokens": 123, + "completion_tokens": 456, + "total_tokens": 579 + }, + "temperature": 1.0, + "top_p": 1.0, + "max_prompt_tokens": 1000, + "max_completion_tokens": 1000, + "truncation_strategy": { + "type": "auto", + "last_messages": null + }, + "response_format": "auto", + "tool_choice": "auto" + } + + /threads/{thread_id}/runs/{run_id}/submit_tool_outputs: + post: + operationId: submitToolOuputsToRun + tags: + - Assistants + summary: | + When a run has the `status: "requires_action"` and `required_action.type` is `submit_tool_outputs`, this endpoint can be used to submit the outputs from the tool calls once they're all completed. All outputs must be submitted in a single request. + parameters: + - in: path + name: thread_id + required: true + schema: + type: string + description: The ID of the [thread](/docs/api-reference/threads) to which this run belongs. + - in: path + name: run_id + required: true + schema: + type: string + description: The ID of the run that requires the tool output submission. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/SubmitToolOutputsRunRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/RunObject" + x-oaiMeta: + name: Submit tool outputs to run + group: threads + beta: true + returns: The modified [run](/docs/api-reference/runs/object) object matching the specified ID. + examples: + - title: Default + request: + curl: | + curl https://api.openai.com/v1/threads/thread_123/runs/run_123/submit_tool_outputs \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "tool_outputs": [ + { + "tool_call_id": "call_001", + "output": "70 degrees and sunny." + } + ] + }' + python: | + from openai import OpenAI + client = OpenAI() + + run = client.beta.threads.runs.submit_tool_outputs( + thread_id="thread_123", + run_id="run_123", + tool_outputs=[ + { + "tool_call_id": "call_001", + "output": "70 degrees and sunny." + } + ] + ) + + print(run) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const run = await openai.beta.threads.runs.submitToolOutputs( + "thread_123", + "run_123", + { + tool_outputs: [ + { + tool_call_id: "call_001", + output: "70 degrees and sunny.", + }, + ], + } + ); + + console.log(run); + } + + main(); + response: | + { + "id": "run_123", + "object": "thread.run", + "created_at": 1699075592, + "assistant_id": "asst_123", + "thread_id": "thread_123", + "status": "queued", + "started_at": 1699075592, + "expires_at": 1699076192, + "cancelled_at": null, + "failed_at": null, + "completed_at": null, + "last_error": null, + "model": "gpt-4-turbo", + "instructions": null, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"] + } + }, + "required": ["location"] + } + } + } + ], + "metadata": {}, + "usage": null, + "temperature": 1.0, + "top_p": 1.0, + "max_prompt_tokens": 1000, + "max_completion_tokens": 1000, + "truncation_strategy": { + "type": "auto", + "last_messages": null + }, + "response_format": "auto", + "tool_choice": "auto" + } + + - title: Streaming + request: + curl: | + curl https://api.openai.com/v1/threads/thread_123/runs/run_123/submit_tool_outputs \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "tool_outputs": [ + { + "tool_call_id": "call_001", + "output": "70 degrees and sunny." + } + ], + "stream": true + }' + python: | + from openai import OpenAI + client = OpenAI() + + stream = client.beta.threads.runs.submit_tool_outputs( + thread_id="thread_123", + run_id="run_123", + tool_outputs=[ + { + "tool_call_id": "call_001", + "output": "70 degrees and sunny." + } + ], + stream=True + ) + + for event in stream: + print(event) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const stream = await openai.beta.threads.runs.submitToolOutputs( + "thread_123", + "run_123", + { + tool_outputs: [ + { + tool_call_id: "call_001", + output: "70 degrees and sunny.", + }, + ], + } + ); + + for await (const event of stream) { + console.log(event); + } + } + + main(); + response: | + event: thread.run.step.completed + data: {"id":"step_001","object":"thread.run.step","created_at":1710352449,"run_id":"run_123","assistant_id":"asst_123","thread_id":"thread_123","type":"tool_calls","status":"completed","cancelled_at":null,"completed_at":1710352475,"expires_at":1710353047,"failed_at":null,"last_error":null,"step_details":{"type":"tool_calls","tool_calls":[{"id":"call_iWr0kQ2EaYMaxNdl0v3KYkx7","type":"function","function":{"name":"get_current_weather","arguments":"{\"location\":\"San Francisco, CA\",\"unit\":\"fahrenheit\"}","output":"70 degrees and sunny."}}]},"usage":{"prompt_tokens":291,"completion_tokens":24,"total_tokens":315}} + + event: thread.run.queued + data: {"id":"run_123","object":"thread.run","created_at":1710352447,"assistant_id":"asst_123","thread_id":"thread_123","status":"queued","started_at":1710352448,"expires_at":1710353047,"cancelled_at":null,"failed_at":null,"completed_at":null,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather in a given location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["location"]}}}],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":null,"response_format":"auto","tool_choice":"auto"}} + + event: thread.run.in_progress + data: {"id":"run_123","object":"thread.run","created_at":1710352447,"assistant_id":"asst_123","thread_id":"thread_123","status":"in_progress","started_at":1710352475,"expires_at":1710353047,"cancelled_at":null,"failed_at":null,"completed_at":null,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather in a given location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["location"]}}}],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":null,"response_format":"auto","tool_choice":"auto"}} + + event: thread.run.step.created + data: {"id":"step_002","object":"thread.run.step","created_at":1710352476,"run_id":"run_123","assistant_id":"asst_123","thread_id":"thread_123","type":"message_creation","status":"in_progress","cancelled_at":null,"completed_at":null,"expires_at":1710353047,"failed_at":null,"last_error":null,"step_details":{"type":"message_creation","message_creation":{"message_id":"msg_002"}},"usage":null} + + event: thread.run.step.in_progress + data: {"id":"step_002","object":"thread.run.step","created_at":1710352476,"run_id":"run_123","assistant_id":"asst_123","thread_id":"thread_123","type":"message_creation","status":"in_progress","cancelled_at":null,"completed_at":null,"expires_at":1710353047,"failed_at":null,"last_error":null,"step_details":{"type":"message_creation","message_creation":{"message_id":"msg_002"}},"usage":null} + + event: thread.message.created + data: {"id":"msg_002","object":"thread.message","created_at":1710352476,"assistant_id":"asst_123","thread_id":"thread_123","run_id":"run_123","status":"in_progress","incomplete_details":null,"incomplete_at":null,"completed_at":null,"role":"assistant","content":[],"metadata":{}} + + event: thread.message.in_progress + data: {"id":"msg_002","object":"thread.message","created_at":1710352476,"assistant_id":"asst_123","thread_id":"thread_123","run_id":"run_123","status":"in_progress","incomplete_details":null,"incomplete_at":null,"completed_at":null,"role":"assistant","content":[],"metadata":{}} + + event: thread.message.delta + data: {"id":"msg_002","object":"thread.message.delta","delta":{"content":[{"index":0,"type":"text","text":{"value":"The","annotations":[]}}]}} + + event: thread.message.delta + data: {"id":"msg_002","object":"thread.message.delta","delta":{"content":[{"index":0,"type":"text","text":{"value":" current"}}]}} + + event: thread.message.delta + data: {"id":"msg_002","object":"thread.message.delta","delta":{"content":[{"index":0,"type":"text","text":{"value":" weather"}}]}} + + ... + + event: thread.message.delta + data: {"id":"msg_002","object":"thread.message.delta","delta":{"content":[{"index":0,"type":"text","text":{"value":" sunny"}}]}} + + event: thread.message.delta + data: {"id":"msg_002","object":"thread.message.delta","delta":{"content":[{"index":0,"type":"text","text":{"value":"."}}]}} + + event: thread.message.completed + data: {"id":"msg_002","object":"thread.message","created_at":1710352476,"assistant_id":"asst_123","thread_id":"thread_123","run_id":"run_123","status":"completed","incomplete_details":null,"incomplete_at":null,"completed_at":1710352477,"role":"assistant","content":[{"type":"text","text":{"value":"The current weather in San Francisco, CA is 70 degrees Fahrenheit and sunny.","annotations":[]}}],"metadata":{}} + + event: thread.run.step.completed + data: {"id":"step_002","object":"thread.run.step","created_at":1710352476,"run_id":"run_123","assistant_id":"asst_123","thread_id":"thread_123","type":"message_creation","status":"completed","cancelled_at":null,"completed_at":1710352477,"expires_at":1710353047,"failed_at":null,"last_error":null,"step_details":{"type":"message_creation","message_creation":{"message_id":"msg_002"}},"usage":{"prompt_tokens":329,"completion_tokens":18,"total_tokens":347}} + + event: thread.run.completed + data: {"id":"run_123","object":"thread.run","created_at":1710352447,"assistant_id":"asst_123","thread_id":"thread_123","status":"completed","started_at":1710352475,"expires_at":null,"cancelled_at":null,"failed_at":null,"completed_at":1710352477,"required_action":null,"last_error":null,"model":"gpt-4-turbo","instructions":null,"tools":[{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather in a given location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["location"]}}}],"metadata":{},"temperature":1.0,"top_p":1.0,"max_completion_tokens":null,"max_prompt_tokens":null,"truncation_strategy":{"type":"auto","last_messages":null},"incomplete_details":null,"usage":{"prompt_tokens":20,"completion_tokens":11,"total_tokens":31},"response_format":"auto","tool_choice":"auto"}} + + event: done + data: [DONE] + + /threads/{thread_id}/runs/{run_id}/cancel: + post: + operationId: cancelRun + tags: + - Assistants + summary: Cancels a run that is `in_progress`. + parameters: + - in: path + name: thread_id + required: true + schema: + type: string + description: The ID of the thread to which this run belongs. + - in: path + name: run_id + required: true + schema: + type: string + description: The ID of the run to cancel. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/RunObject" + x-oaiMeta: + name: Cancel a run + group: threads + beta: true + returns: The modified [run](/docs/api-reference/runs/object) object matching the specified ID. + examples: + request: + curl: | + curl https://api.openai.com/v1/threads/thread_abc123/runs/run_abc123/cancel \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "OpenAI-Beta: assistants=v2" \ + -X POST + python: | + from openai import OpenAI + client = OpenAI() + + run = client.beta.threads.runs.cancel( + thread_id="thread_abc123", + run_id="run_abc123" + ) + + print(run) + node.js: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const run = await openai.beta.threads.runs.cancel( + "thread_abc123", + "run_abc123" + ); + + console.log(run); + } + + main(); + response: | + { + "id": "run_abc123", + "object": "thread.run", + "created_at": 1699076126, + "assistant_id": "asst_abc123", + "thread_id": "thread_abc123", + "status": "cancelling", + "started_at": 1699076126, + "expires_at": 1699076726, + "cancelled_at": null, + "failed_at": null, + "completed_at": null, + "last_error": null, + "model": "gpt-4-turbo", + "instructions": "You summarize books.", + "tools": [ + { + "type": "file_search" + } + ], + "tool_resources": { + "file_search": { + "vector_store_ids": ["vs_123"] + } + }, + "metadata": {}, + "usage": null, + "temperature": 1.0, + "top_p": 1.0, + "response_format": "auto" + } + + /threads/{thread_id}/runs/{run_id}/steps: + get: + operationId: listRunSteps + tags: + - Assistants + summary: Returns a list of run steps belonging to a run. + parameters: + - name: thread_id + in: path + required: true + schema: + type: string + description: The ID of the thread the run and run steps belong to. + - name: run_id + in: path + required: true + schema: + type: string + description: The ID of the run the run steps belong to. + - name: limit + in: query + description: *pagination_limit_param_description + required: false + schema: + type: integer + default: 20 + - name: order + in: query + description: *pagination_order_param_description + schema: + type: string + default: desc + enum: ["asc", "desc"] + - name: after + in: query + description: *pagination_after_param_description + schema: + type: string + - name: before + in: query + description: *pagination_before_param_description + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListRunStepsResponse" + x-oaiMeta: + name: List run steps + group: threads + beta: true + returns: A list of [run step](/docs/api-reference/runs/step-object) objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/threads/thread_abc123/runs/run_abc123/steps \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + run_steps = client.beta.threads.runs.steps.list( + thread_id="thread_abc123", + run_id="run_abc123" + ) + + print(run_steps) + node.js: | + import OpenAI from "openai"; + const openai = new OpenAI(); + + async function main() { + const runStep = await openai.beta.threads.runs.steps.list( + "thread_abc123", + "run_abc123" + ); + console.log(runStep); + } + + main(); + response: | + { + "object": "list", + "data": [ + { + "id": "step_abc123", + "object": "thread.run.step", + "created_at": 1699063291, + "run_id": "run_abc123", + "assistant_id": "asst_abc123", + "thread_id": "thread_abc123", + "type": "message_creation", + "status": "completed", + "cancelled_at": null, + "completed_at": 1699063291, + "expired_at": null, + "failed_at": null, + "last_error": null, + "step_details": { + "type": "message_creation", + "message_creation": { + "message_id": "msg_abc123" + } + }, + "usage": { + "prompt_tokens": 123, + "completion_tokens": 456, + "total_tokens": 579 + } + } + ], + "first_id": "step_abc123", + "last_id": "step_abc456", + "has_more": false + } + + /threads/{thread_id}/runs/{run_id}/steps/{step_id}: + get: + operationId: getRunStep + tags: + - Assistants + summary: Retrieves a run step. + parameters: + - in: path + name: thread_id + required: true + schema: + type: string + description: The ID of the thread to which the run and run step belongs. + - in: path + name: run_id + required: true + schema: + type: string + description: The ID of the run to which the run step belongs. + - in: path + name: step_id + required: true + schema: + type: string + description: The ID of the run step to retrieve. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/RunStepObject" + x-oaiMeta: + name: Retrieve run step + group: threads + beta: true + returns: The [run step](/docs/api-reference/runs/step-object) object matching the specified ID. + examples: + request: + curl: | + curl https://api.openai.com/v1/threads/thread_abc123/runs/run_abc123/steps/step_abc123 \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + run_step = client.beta.threads.runs.steps.retrieve( + thread_id="thread_abc123", + run_id="run_abc123", + step_id="step_abc123" + ) + + print(run_step) + node.js: | + import OpenAI from "openai"; + const openai = new OpenAI(); + + async function main() { + const runStep = await openai.beta.threads.runs.steps.retrieve( + "thread_abc123", + "run_abc123", + "step_abc123" + ); + console.log(runStep); + } + + main(); + response: &run_step_object_example | + { + "id": "step_abc123", + "object": "thread.run.step", + "created_at": 1699063291, + "run_id": "run_abc123", + "assistant_id": "asst_abc123", + "thread_id": "thread_abc123", + "type": "message_creation", + "status": "completed", + "cancelled_at": null, + "completed_at": 1699063291, + "expired_at": null, + "failed_at": null, + "last_error": null, + "step_details": { + "type": "message_creation", + "message_creation": { + "message_id": "msg_abc123" + } + }, + "usage": { + "prompt_tokens": 123, + "completion_tokens": 456, + "total_tokens": 579 + } + } + + /vector_stores: + get: + operationId: listVectorStores + tags: + - Vector Stores + summary: Returns a list of vector stores. + parameters: + - name: limit + in: query + description: *pagination_limit_param_description + required: false + schema: + type: integer + default: 20 + - name: order + in: query + description: *pagination_order_param_description + schema: + type: string + default: desc + enum: ["asc", "desc"] + - name: after + in: query + description: *pagination_after_param_description + schema: + type: string + - name: before + in: query + description: *pagination_before_param_description + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListVectorStoresResponse" + x-oaiMeta: + name: List vector stores + group: vector_stores + beta: true + returns: A list of [vector store](/docs/api-reference/vector-stores/object) objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/vector_stores \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + vector_stores = client.beta.vector_stores.list() + print(vector_stores) + node.js: | + import OpenAI from "openai"; + const openai = new OpenAI(); + + async function main() { + const vectorStores = await openai.beta.vectorStores.list(); + console.log(vectorStores); + } + + main(); + response: | + { + "object": "list", + "data": [ + { + "id": "vs_abc123", + "object": "vector_store", + "created_at": 1699061776, + "name": "Support FAQ", + "bytes": 139920, + "file_counts": { + "in_progress": 0, + "completed": 3, + "failed": 0, + "cancelled": 0, + "total": 3 + } + }, + { + "id": "vs_abc456", + "object": "vector_store", + "created_at": 1699061776, + "name": "Support FAQ v2", + "bytes": 139920, + "file_counts": { + "in_progress": 0, + "completed": 3, + "failed": 0, + "cancelled": 0, + "total": 3 + } + } + ], + "first_id": "vs_abc123", + "last_id": "vs_abc456", + "has_more": false + } + post: + operationId: createVectorStore + tags: + - Vector Stores + summary: Create a vector store. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateVectorStoreRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/VectorStoreObject" + x-oaiMeta: + name: Create vector store + group: vector_stores + beta: true + returns: A [vector store](/docs/api-reference/vector-stores/object) object. + examples: + request: + curl: | + curl https://api.openai.com/v1/vector_stores \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" + -d '{ + "name": "Support FAQ" + }' + python: | + from openai import OpenAI + client = OpenAI() + + vector_store = client.beta.vector_stores.create( + name="Support FAQ" + ) + print(vector_store) + node.js: | + import OpenAI from "openai"; + const openai = new OpenAI(); + + async function main() { + const vectorStore = await openai.beta.vectorStores.create({ + name: "Support FAQ" + }); + console.log(vectorStore); + } + + main(); + response: | + { + "id": "vs_abc123", + "object": "vector_store", + "created_at": 1699061776, + "name": "Support FAQ", + "bytes": 139920, + "file_counts": { + "in_progress": 0, + "completed": 3, + "failed": 0, + "cancelled": 0, + "total": 3 + } + } + + /vector_stores/{vector_store_id}: + get: + operationId: getVectorStore + tags: + - Vector Stores + summary: Retrieves a vector store. + parameters: + - in: path + name: vector_store_id + required: true + schema: + type: string + description: The ID of the vector store to retrieve. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/VectorStoreObject" + x-oaiMeta: + name: Retrieve vector store + group: vector_stores + beta: true + returns: The [vector store](/docs/api-reference/vector-stores/object) object matching the specified ID. + examples: + request: + curl: | + curl https://api.openai.com/v1/vector_stores/vs_abc123 \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + vector_store = client.beta.vector_stores.retrieve( + vector_store_id="vs_abc123" + ) + print(vector_store) + node.js: | + import OpenAI from "openai"; + const openai = new OpenAI(); + + async function main() { + const vectorStore = await openai.beta.vectorStores.retrieve( + "vs_abc123" + ); + console.log(vectorStore); + } + + main(); + response: | + { + "id": "vs_abc123", + "object": "vector_store", + "created_at": 1699061776 + } + post: + operationId: modifyVectorStore + tags: + - Vector Stores + summary: Modifies a vector store. + parameters: + - in: path + name: vector_store_id + required: true + schema: + type: string + description: The ID of the vector store to modify. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/UpdateVectorStoreRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/VectorStoreObject" + x-oaiMeta: + name: Modify vector store + group: vector_stores + beta: true + returns: The modified [vector store](/docs/api-reference/vector-stores/object) object. + examples: + request: + curl: | + curl https://api.openai.com/v1/vector_stores/vs_abc123 \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" + -d '{ + "name": "Support FAQ" + }' + python: | + from openai import OpenAI + client = OpenAI() + + vector_store = client.beta.vector_stores.update( + vector_store_id="vs_abc123", + name="Support FAQ" + ) + print(vector_store) + node.js: | + import OpenAI from "openai"; + const openai = new OpenAI(); + + async function main() { + const vectorStore = await openai.beta.vectorStores.update( + "vs_abc123", + { + name: "Support FAQ" + } + ); + console.log(vectorStore); + } + + main(); + response: | + { + "id": "vs_abc123", + "object": "vector_store", + "created_at": 1699061776, + "name": "Support FAQ", + "bytes": 139920, + "file_counts": { + "in_progress": 0, + "completed": 3, + "failed": 0, + "cancelled": 0, + "total": 3 + } + } + + delete: + operationId: deleteVectorStore + tags: + - Vector Stores + summary: Delete a vector store. + parameters: + - in: path + name: vector_store_id + required: true + schema: + type: string + description: The ID of the vector store to delete. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/DeleteVectorStoreResponse" + x-oaiMeta: + name: Delete vector store + group: vector_stores + beta: true + returns: Deletion status + examples: + request: + curl: | + curl https://api.openai.com/v1/vector_stores/vs_abc123 \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" \ + -X DELETE + python: | + from openai import OpenAI + client = OpenAI() + + deleted_vector_store = client.beta.vector_stores.delete( + vector_store_id="vs_abc123" + ) + print(deleted_vector_store) + node.js: | + import OpenAI from "openai"; + const openai = new OpenAI(); + + async function main() { + const deletedVectorStore = await openai.beta.vectorStores.del( + "vs_abc123" + ); + console.log(deletedVectorStore); + } + + main(); + response: | + { + id: "vs_abc123", + object: "vector_store.deleted", + deleted: true + } + + /vector_stores/{vector_store_id}/files: + get: + operationId: listVectorStoreFiles + tags: + - Vector Stores + summary: Returns a list of vector store files. + parameters: + - name: vector_store_id + in: path + description: The ID of the vector store that the files belong to. + required: true + schema: + type: string + - name: limit + in: query + description: *pagination_limit_param_description + required: false + schema: + type: integer + default: 20 + - name: order + in: query + description: *pagination_order_param_description + schema: + type: string + default: desc + enum: ["asc", "desc"] + - name: after + in: query + description: *pagination_after_param_description + schema: + type: string + - name: before + in: query + description: *pagination_before_param_description + schema: + type: string + - name: filter + in: query + description: "Filter by file status. One of `in_progress`, `completed`, `failed`, `cancelled`." + schema: + type: string + enum: ["in_progress", "completed", "failed", "cancelled"] + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListVectorStoreFilesResponse" + x-oaiMeta: + name: List vector store files + group: vector_stores + beta: true + returns: A list of [vector store file](/docs/api-reference/vector-stores-files/file-object) objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/vector_stores/vs_abc123/files \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + vector_store_files = client.beta.vector_stores.files.list( + vector_store_id="vs_abc123" + ) + print(vector_store_files) + node.js: | + import OpenAI from "openai"; + const openai = new OpenAI(); + + async function main() { + const vectorStoreFiles = await openai.beta.vectorStores.files.list( + "vs_abc123" + ); + console.log(vectorStoreFiles); + } + + main(); + response: | + { + "object": "list", + "data": [ + { + "id": "file-abc123", + "object": "vector_store.file", + "created_at": 1699061776, + "vector_store_id": "vs_abc123" + }, + { + "id": "file-abc456", + "object": "vector_store.file", + "created_at": 1699061776, + "vector_store_id": "vs_abc123" + } + ], + "first_id": "file-abc123", + "last_id": "file-abc456", + "has_more": false + } + post: + operationId: createVectorStoreFile + tags: + - Vector Stores + summary: Create a vector store file by attaching a [File](/docs/api-reference/files) to a [vector store](/docs/api-reference/vector-stores/object). + parameters: + - in: path + name: vector_store_id + required: true + schema: + type: string + example: vs_abc123 + description: | + The ID of the vector store for which to create a File. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateVectorStoreFileRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/VectorStoreFileObject" + x-oaiMeta: + name: Create vector store file + group: vector_stores + beta: true + returns: A [vector store file](/docs/api-reference/vector-stores-files/file-object) object. + examples: + request: + curl: | + curl https://api.openai.com/v1/vector_stores/vs_abc123/files \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "file_id": "file-abc123" + }' + python: | + from openai import OpenAI + client = OpenAI() + + vector_store_file = client.beta.vector_stores.files.create( + vector_store_id="vs_abc123", + file_id="file-abc123" + ) + print(vector_store_file) + node.js: | + import OpenAI from "openai"; + const openai = new OpenAI(); + + async function main() { + const myVectorStoreFile = await openai.beta.vectorStores.files.create( + "vs_abc123", + { + file_id: "file-abc123" + } + ); + console.log(myVectorStoreFile); + } + + main(); + response: | + { + "id": "file-abc123", + "object": "vector_store.file", + "created_at": 1699061776, + "usage_bytes": 1234, + "vector_store_id": "vs_abcd", + "status": "completed", + "last_error": null + } + + /vector_stores/{vector_store_id}/files/{file_id}: + get: + operationId: getVectorStoreFile + tags: + - Vector Stores + summary: Retrieves a vector store file. + parameters: + - in: path + name: vector_store_id + required: true + schema: + type: string + example: vs_abc123 + description: The ID of the vector store that the file belongs to. + - in: path + name: file_id + required: true + schema: + type: string + example: file-abc123 + description: The ID of the file being retrieved. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/VectorStoreFileObject" + x-oaiMeta: + name: Retrieve vector store file + group: vector_stores + beta: true + returns: The [vector store file](/docs/api-reference/vector-stores-files/file-object) object. + examples: + request: + curl: | + curl https://api.openai.com/v1/vector_stores/vs_abc123/files/file-abc123 \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + vector_store_file = client.beta.vector_stores.files.retrieve( + vector_store_id="vs_abc123", + file_id="file-abc123" + ) + print(vector_store_file) + node.js: | + import OpenAI from "openai"; + const openai = new OpenAI(); + + async function main() { + const vectorStoreFile = await openai.beta.vectorStores.files.retrieve( + "vs_abc123", + "file-abc123" + ); + console.log(vectorStoreFile); + } + + main(); + response: | + { + "id": "file-abc123", + "object": "vector_store.file", + "created_at": 1699061776, + "vector_store_id": "vs_abcd", + "status": "completed", + "last_error": null + } + delete: + operationId: deleteVectorStoreFile + tags: + - Vector Stores + summary: Delete a vector store file. This will remove the file from the vector store but the file itself will not be deleted. To delete the file, use the [delete file](/docs/api-reference/files/delete) endpoint. + parameters: + - in: path + name: vector_store_id + required: true + schema: + type: string + description: The ID of the vector store that the file belongs to. + - in: path + name: file_id + required: true + schema: + type: string + description: The ID of the file to delete. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/DeleteVectorStoreFileResponse" + x-oaiMeta: + name: Delete vector store file + group: vector_stores + beta: true + returns: Deletion status + examples: + request: + curl: | + curl https://api.openai.com/v1/vector_stores/vs_abc123/files/file-abc123 \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" \ + -X DELETE + python: | + from openai import OpenAI + client = OpenAI() + + deleted_vector_store_file = client.beta.vector_stores.files.delete( + vector_store_id="vs_abc123", + file_id="file-abc123" + ) + print(deleted_vector_store_file) + node.js: | + import OpenAI from "openai"; + const openai = new OpenAI(); + + async function main() { + const deletedVectorStoreFile = await openai.beta.vectorStores.files.del( + "vs_abc123", + "file-abc123" + ); + console.log(deletedVectorStoreFile); + } + + main(); + response: | + { + id: "file-abc123", + object: "vector_store.file.deleted", + deleted: true + } + + /vector_stores/{vector_store_id}/file_batches: + post: + operationId: createVectorStoreFileBatch + tags: + - Vector Stores + summary: Create a vector store file batch. + parameters: + - in: path + name: vector_store_id + required: true + schema: + type: string + example: vs_abc123 + description: | + The ID of the vector store for which to create a File Batch. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateVectorStoreFileBatchRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/VectorStoreFileBatchObject" + x-oaiMeta: + name: Create vector store file batch + group: vector_stores + beta: true + returns: A [vector store file batch](/docs/api-reference/vector-stores-file-batches/batch-object) object. + examples: + request: + curl: | + curl https://api.openai.com/v1/vector_stores/vs_abc123/file_batches \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json \ + -H "OpenAI-Beta: assistants=v2" \ + -d '{ + "file_ids": ["file-abc123", "file-abc456"] + }' + python: | + from openai import OpenAI + client = OpenAI() + + vector_store_file_batch = client.beta.vector_stores.file_batches.create( + vector_store_id="vs_abc123", + file_ids=["file-abc123", "file-abc456"] + ) + print(vector_store_file_batch) + node.js: | + import OpenAI from "openai"; + const openai = new OpenAI(); + + async function main() { + const myVectorStoreFileBatch = await openai.beta.vectorStores.fileBatches.create( + "vs_abc123", + { + file_ids: ["file-abc123", "file-abc456"] + } + ); + console.log(myVectorStoreFileBatch); + } + + main(); + response: | + { + "id": "vsfb_abc123", + "object": "vector_store.file_batch", + "created_at": 1699061776, + "vector_store_id": "vs_abc123", + "status": "in_progress", + "file_counts": { + "in_progress": 1, + "completed": 1, + "failed": 0, + "cancelled": 0, + "total": 0, + } + } + + /vector_stores/{vector_store_id}/file_batches/{batch_id}: + get: + operationId: getVectorStoreFileBatch + tags: + - Vector Stores + summary: Retrieves a vector store file batch. + parameters: + - in: path + name: vector_store_id + required: true + schema: + type: string + example: vs_abc123 + description: The ID of the vector store that the file batch belongs to. + - in: path + name: batch_id + required: true + schema: + type: string + example: vsfb_abc123 + description: The ID of the file batch being retrieved. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/VectorStoreFileBatchObject" + x-oaiMeta: + name: Retrieve vector store file batch + group: vector_stores + beta: true + returns: The [vector store file batch](/docs/api-reference/vector-stores-file-batches/batch-object) object. + examples: + request: + curl: | + curl https://api.openai.com/v1/vector_stores/vs_abc123/files_batches/vsfb_abc123 \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + vector_store_file_batch = client.beta.vector_stores.file_batches.retrieve( + vector_store_id="vs_abc123", + batch_id="vsfb_abc123" + ) + print(vector_store_file_batch) + node.js: | + import OpenAI from "openai"; + const openai = new OpenAI(); + + async function main() { + const vectorStoreFileBatch = await openai.beta.vectorStores.fileBatches.retrieve( + "vs_abc123", + "vsfb_abc123" + ); + console.log(vectorStoreFileBatch); + } + + main(); + response: | + { + "id": "vsfb_abc123", + "object": "vector_store.file_batch", + "created_at": 1699061776, + "vector_store_id": "vs_abc123", + "status": "in_progress", + "file_counts": { + "in_progress": 1, + "completed": 1, + "failed": 0, + "cancelled": 0, + "total": 0, + } + } + + /vector_stores/{vector_store_id}/file_batches/{batch_id}/cancel: + post: + operationId: cancelVectorStoreFileBatch + tags: + - Vector Stores + summary: Cancel a vector store file batch. This attempts to cancel the processing of files in this batch as soon as possible. + parameters: + - in: path + name: vector_store_id + required: true + schema: + type: string + description: The ID of the vector store that the file batch belongs to. + - in: path + name: batch_id + required: true + schema: + type: string + description: The ID of the file batch to cancel. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/VectorStoreFileBatchObject" + x-oaiMeta: + name: Cancel vector store file batch + group: vector_stores + beta: true + returns: The modified vector store file batch object. + examples: + request: + curl: | + curl https://api.openai.com/v1/vector_stores/vs_abc123/files_batches/vsfb_abc123/cancel \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" \ + -X POST + python: | + from openai import OpenAI + client = OpenAI() + + deleted_vector_store_file_batch = client.beta.vector_stores.file_batches.cancel( + vector_store_id="vs_abc123", + file_batch_id="vsfb_abc123" + ) + print(deleted_vector_store_file_batch) + node.js: | + import OpenAI from "openai"; + const openai = new OpenAI(); + + async function main() { + const deletedVectorStoreFileBatch = await openai.vector_stores.fileBatches.cancel( + "vs_abc123", + "vsfb_abc123" + ); + console.log(deletedVectorStoreFileBatch); + } + + main(); + response: | + { + "id": "vsfb_abc123", + "object": "vector_store.file_batch", + "created_at": 1699061776, + "vector_store_id": "vs_abc123", + "status": "cancelling", + "file_counts": { + "in_progress": 12, + "completed": 3, + "failed": 0, + "cancelled": 0, + "total": 15, + } + } + + /vector_stores/{vector_store_id}/file_batches/{batch_id}/files: + get: + operationId: listFilesInVectorStoreBatch + tags: + - Vector Stores + summary: Returns a list of vector store files in a batch. + parameters: + - name: vector_store_id + in: path + description: The ID of the vector store that the files belong to. + required: true + schema: + type: string + - name: batch_id + in: path + description: The ID of the file batch that the files belong to. + required: true + schema: + type: string + - name: limit + in: query + description: *pagination_limit_param_description + required: false + schema: + type: integer + default: 20 + - name: order + in: query + description: *pagination_order_param_description + schema: + type: string + default: desc + enum: ["asc", "desc"] + - name: after + in: query + description: *pagination_after_param_description + schema: + type: string + - name: before + in: query + description: *pagination_before_param_description + schema: + type: string + - name: filter + in: query + description: "Filter by file status. One of `in_progress`, `completed`, `failed`, `cancelled`." + schema: + type: string + enum: ["in_progress", "completed", "failed", "cancelled"] + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListVectorStoreFilesResponse" + x-oaiMeta: + name: List vector store files in a batch + group: vector_stores + beta: true + returns: A list of [vector store file](/docs/api-reference/vector-stores-files/file-object) objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/vector_stores/vs_abc123/files_batches/vsfb_abc123/files \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -H "OpenAI-Beta: assistants=v2" + python: | + from openai import OpenAI + client = OpenAI() + + vector_store_files = client.beta.vector_stores.file_batches.list_files( + vector_store_id="vs_abc123", + batch_id="vsfb_abc123" + ) + print(vector_store_files) + node.js: | + import OpenAI from "openai"; + const openai = new OpenAI(); + + async function main() { + const vectorStoreFiles = await openai.beta.vectorStores.fileBatches.listFiles( + "vs_abc123", + "vsfb_abc123" + ); + console.log(vectorStoreFiles); + } + + main(); + response: | + { + "object": "list", + "data": [ + { + "id": "file-abc123", + "object": "vector_store.file", + "created_at": 1699061776, + "vector_store_id": "vs_abc123" + }, + { + "id": "file-abc456", + "object": "vector_store.file", + "created_at": 1699061776, + "vector_store_id": "vs_abc123" + } + ], + "first_id": "file-abc123", + "last_id": "file-abc456", + "has_more": false + } + + /batches: + post: + summary: Creates and executes a batch from an uploaded file of requests + operationId: createBatch + tags: + - Batch + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - input_file_id + - endpoint + - completion_window + properties: + input_file_id: + type: string + description: | + The ID of an uploaded file that contains requests for the new batch. + + See [upload file](/docs/api-reference/files/create) for how to upload a file. + + Your input file must be formatted as a [JSONL file](/docs/api-reference/batch/requestInput), and must be uploaded with the purpose `batch`. + endpoint: + type: string + enum: ["/v1/chat/completions", "/v1/embeddings"] + description: The endpoint to be used for all requests in the batch. Currently `/v1/chat/completions` and `/v1/embeddings` are supported. + completion_window: + type: string + enum: ["24h"] + description: The time frame within which the batch should be processed. Currently only `24h` is supported. + metadata: + type: object + additionalProperties: + type: string + description: Optional custom metadata for the batch. + nullable: true + responses: + "200": + description: Batch created successfully. + content: + application/json: + schema: + $ref: "#/components/schemas/Batch" + x-oaiMeta: + name: Create batch + group: batch + returns: The created [Batch](/docs/api-reference/batch/object) object. + examples: + request: + curl: | + curl https://api.openai.com/v1/batches \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "input_file_id": "file-abc123", + "endpoint": "/v1/chat/completions", + "completion_window": "24h" + }' + python: | + from openai import OpenAI + client = OpenAI() + + client.batches.create( + input_file_id="file-abc123", + endpoint="/v1/chat/completions", + completion_window="24h" + ) + node: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const batch = await openai.batches.create({ + input_file_id: "file-abc123", + endpoint: "/v1/chat/completions", + completion_window: "24h" + }); + + console.log(batch); + } + + main(); + response: | + { + "id": "batch_abc123", + "object": "batch", + "endpoint": "/v1/completions", + "errors": null, + "input_file_id": "file-abc123", + "completion_window": "24h", + "status": "validating", + "output_file_id": null, + "error_file_id": null, + "created_at": 1711471533, + "in_progress_at": null, + "expires_at": null, + "finalizing_at": null, + "completed_at": null, + "failed_at": null, + "expired_at": null, + "cancelling_at": null, + "cancelled_at": null, + "request_counts": { + "total": 0, + "completed": 0, + "failed": 0 + }, + "metadata": { + "customer_id": "user_123456789", + "batch_description": "Nightly eval job", + } + } + get: + operationId: listBatches + tags: + - Batch + summary: List your organization's batches. + parameters: + - in: query + name: after + required: false + schema: + type: string + description: *pagination_after_param_description + - name: limit + in: query + description: *pagination_limit_param_description + required: false + schema: + type: integer + default: 20 + responses: + "200": + description: Batch listed successfully. + content: + application/json: + schema: + $ref: "#/components/schemas/ListBatchesResponse" + x-oaiMeta: + name: List batch + group: batch + returns: A list of paginated [Batch](/docs/api-reference/batch/object) objects. + examples: + request: + curl: | + curl https://api.openai.com/v1/batches?limit=2 \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" + python: | + from openai import OpenAI + client = OpenAI() + + client.batches.list() + node: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const list = await openai.batches.list(); + + for await (const batch of list) { + console.log(batch); + } + } + + main(); + response: | + { + "object": "list", + "data": [ + { + "id": "batch_abc123", + "object": "batch", + "endpoint": "/v1/completions", + "errors": null, + "input_file_id": "file-abc123", + "completion_window": "24h", + "status": "completed", + "output_file_id": "file-cvaTdG", + "error_file_id": "file-HOWS94", + "created_at": 1711471533, + "in_progress_at": 1711471538, + "expires_at": 1711557933, + "finalizing_at": 1711493133, + "completed_at": 1711493163, + "failed_at": null, + "expired_at": null, + "cancelling_at": null, + "cancelled_at": null, + "request_counts": { + "total": 100, + "completed": 95, + "failed": 5 + }, + "metadata": { + "customer_id": "user_123456789", + "batch_description": "Nightly job", + } + }, + { ... }, + ], + "first_id": "batch_abc123", + "last_id": "batch_abc456", + "has_more": true + } + + /batches/{batch_id}: + get: + operationId: retrieveBatch + tags: + - Batch + summary: Retrieves a batch. + parameters: + - in: path + name: batch_id + required: true + schema: + type: string + description: The ID of the batch to retrieve. + responses: + "200": + description: Batch retrieved successfully. + content: + application/json: + schema: + $ref: "#/components/schemas/Batch" + x-oaiMeta: + name: Retrieve batch + group: batch + returns: The [Batch](/docs/api-reference/batch/object) object matching the specified ID. + examples: + request: + curl: | + curl https://api.openai.com/v1/batches/batch_abc123 \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + python: | + from openai import OpenAI + client = OpenAI() + + client.batches.retrieve("batch_abc123") + node: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const batch = await openai.batches.retrieve("batch_abc123"); + + console.log(batch); + } + + main(); + response: &batch_object | + { + "id": "batch_abc123", + "object": "batch", + "endpoint": "/v1/completions", + "errors": null, + "input_file_id": "file-abc123", + "completion_window": "24h", + "status": "completed", + "output_file_id": "file-cvaTdG", + "error_file_id": "file-HOWS94", + "created_at": 1711471533, + "in_progress_at": 1711471538, + "expires_at": 1711557933, + "finalizing_at": 1711493133, + "completed_at": 1711493163, + "failed_at": null, + "expired_at": null, + "cancelling_at": null, + "cancelled_at": null, + "request_counts": { + "total": 100, + "completed": 95, + "failed": 5 + }, + "metadata": { + "customer_id": "user_123456789", + "batch_description": "Nightly eval job", + } + } + + /batches/{batch_id}/cancel: + post: + operationId: cancelBatch + tags: + - Batch + summary: Cancels an in-progress batch. + parameters: + - in: path + name: batch_id + required: true + schema: + type: string + description: The ID of the batch to cancel. + responses: + "200": + description: Batch is cancelling. Returns the cancelling batch's details. + content: + application/json: + schema: + $ref: "#/components/schemas/Batch" + x-oaiMeta: + name: Cancel batch + group: batch + returns: The [Batch](/docs/api-reference/batch/object) object matching the specified ID. + examples: + request: + curl: | + curl https://api.openai.com/v1/batches/batch_abc123/cancel \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -X POST + python: | + from openai import OpenAI + client = OpenAI() + + client.batches.cancel("batch_abc123") + node: | + import OpenAI from "openai"; + + const openai = new OpenAI(); + + async function main() { + const batch = await openai.batches.cancel("batch_abc123"); + + console.log(batch); + } + + main(); + response: | + { + "id": "batch_abc123", + "object": "batch", + "endpoint": "/v1/completions", + "errors": null, + "input_file_id": "file-abc123", + "completion_window": "24h", + "status": "cancelling", + "output_file_id": null, + "error_file_id": null, + "created_at": 1711471533, + "in_progress_at": 1711471538, + "expires_at": 1711557933, + "finalizing_at": null, + "completed_at": null, + "failed_at": null, + "expired_at": null, + "cancelling_at": 1711475133, + "cancelled_at": null, + "request_counts": { + "total": 100, + "completed": 23, + "failed": 1 + }, + "metadata": { + "customer_id": "user_123456789", + "batch_description": "Nightly eval job", + } + } + +components: + securitySchemes: + ApiKeyAuth: + type: http + scheme: "bearer" + + schemas: + Error: + type: object + properties: + code: + type: string + nullable: true + message: + type: string + nullable: false + param: + type: string + nullable: true + type: + type: string + nullable: false + required: + - type + - message + - param + - code + ErrorResponse: + type: object + properties: + error: + $ref: "#/components/schemas/Error" + required: + - error + + ListModelsResponse: + type: object + properties: + object: + type: string + enum: [list] + data: + type: array + items: + $ref: "#/components/schemas/Model" + required: + - object + - data + DeleteModelResponse: + type: object + properties: + id: + type: string + deleted: + type: boolean + object: + type: string + required: + - id + - object + - deleted + + CreateCompletionRequest: + type: object + properties: + model: + description: &model_description | + ID of the model to use. You can use the [List models](/docs/api-reference/models/list) API to see all of your available models, or see our [Model overview](/docs/models/overview) for descriptions of them. + anyOf: + - type: string + - type: string + enum: ["gpt-3.5-turbo-instruct", "davinci-002", "babbage-002"] + x-oaiTypeLabel: string + prompt: + description: &completions_prompt_description | + The prompt(s) to generate completions for, encoded as a string, array of strings, array of tokens, or array of token arrays. + + Note that <|endoftext|> is the document separator that the model sees during training, so if a prompt is not specified the model will generate as if from the beginning of a new document. + default: "<|endoftext|>" + nullable: true + oneOf: + - type: string + default: "" + example: "This is a test." + - type: array + items: + type: string + default: "" + example: "This is a test." + - type: array + minItems: 1 + items: + type: integer + example: "[1212, 318, 257, 1332, 13]" + - type: array + minItems: 1 + items: + type: array + minItems: 1 + items: + type: integer + example: "[[1212, 318, 257, 1332, 13]]" + best_of: + type: integer + default: 1 + minimum: 0 + maximum: 20 + nullable: true + description: &completions_best_of_description | + Generates `best_of` completions server-side and returns the "best" (the one with the highest log probability per token). Results cannot be streamed. + + When used with `n`, `best_of` controls the number of candidate completions and `n` specifies how many to return – `best_of` must be greater than `n`. + + **Note:** Because this parameter generates many completions, it can quickly consume your token quota. Use carefully and ensure that you have reasonable settings for `max_tokens` and `stop`. + echo: + type: boolean + default: false + nullable: true + description: &completions_echo_description > + Echo back the prompt in addition to the completion + frequency_penalty: + type: number + default: 0 + minimum: -2 + maximum: 2 + nullable: true + description: &completions_frequency_penalty_description | + Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. + + [See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details) + logit_bias: &completions_logit_bias + type: object + x-oaiTypeLabel: map + default: null + nullable: true + additionalProperties: + type: integer + description: &completions_logit_bias_description | + Modify the likelihood of specified tokens appearing in the completion. + + Accepts a JSON object that maps tokens (specified by their token ID in the GPT tokenizer) to an associated bias value from -100 to 100. You can use this [tokenizer tool](/tokenizer?view=bpe) to convert text to token IDs. Mathematically, the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should result in a ban or exclusive selection of the relevant token. + + As an example, you can pass `{"50256": -100}` to prevent the <|endoftext|> token from being generated. + logprobs: &completions_logprobs_configuration + type: integer + minimum: 0 + maximum: 5 + default: null + nullable: true + description: &completions_logprobs_description | + Include the log probabilities on the `logprobs` most likely output tokens, as well the chosen tokens. For example, if `logprobs` is 5, the API will return a list of the 5 most likely tokens. The API will always return the `logprob` of the sampled token, so there may be up to `logprobs+1` elements in the response. + + The maximum value for `logprobs` is 5. + max_tokens: + type: integer + minimum: 0 + default: 16 + example: 16 + nullable: true + description: &completions_max_tokens_description | + The maximum number of [tokens](/tokenizer) that can be generated in the completion. + + The token count of your prompt plus `max_tokens` cannot exceed the model's context length. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens. + n: + type: integer + minimum: 1 + maximum: 128 + default: 1 + example: 1 + nullable: true + description: &completions_completions_description | + How many completions to generate for each prompt. + + **Note:** Because this parameter generates many completions, it can quickly consume your token quota. Use carefully and ensure that you have reasonable settings for `max_tokens` and `stop`. + presence_penalty: + type: number + default: 0 + minimum: -2 + maximum: 2 + nullable: true + description: &completions_presence_penalty_description | + Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. + + [See more information about frequency and presence penalties.](/docs/guides/text-generation/parameter-details) + seed: &completions_seed_param + type: integer + minimum: -9223372036854775808 + maximum: 9223372036854775807 + nullable: true + description: | + If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same `seed` and parameters should return the same result. + + Determinism is not guaranteed, and you should refer to the `system_fingerprint` response parameter to monitor changes in the backend. + stop: + description: &completions_stop_description > + Up to 4 sequences where the API will stop generating further tokens. The returned text will not contain the stop sequence. + default: null + nullable: true + oneOf: + - type: string + default: <|endoftext|> + example: "\n" + nullable: true + - type: array + minItems: 1 + maxItems: 4 + items: + type: string + example: '["\n"]' + stream: + description: > + Whether to stream back partial progress. If set, tokens will be sent as data-only [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) + as they become available, with the stream terminated by a `data: [DONE]` message. [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions). + type: boolean + nullable: true + default: false + suffix: + description: | + The suffix that comes after a completion of inserted text. + + This parameter is only supported for `gpt-3.5-turbo-instruct`. + default: null + nullable: true + type: string + example: "test." + temperature: + type: number + minimum: 0 + maximum: 2 + default: 1 + example: 1 + nullable: true + description: &completions_temperature_description | + What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. + + We generally recommend altering this or `top_p` but not both. + top_p: + type: number + minimum: 0 + maximum: 1 + default: 1 + example: 1 + nullable: true + description: &completions_top_p_description | + An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. + + We generally recommend altering this or `temperature` but not both. + user: &end_user_param_configuration + type: string + example: user-1234 + description: | + A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](/docs/guides/safety-best-practices/end-user-ids). + required: + - model + - prompt + + CreateCompletionResponse: + type: object + description: | + Represents a completion response from the API. Note: both the streamed and non-streamed response objects share the same shape (unlike the chat endpoint). + properties: + id: + type: string + description: A unique identifier for the completion. + choices: + type: array + description: The list of completion choices the model generated for the input prompt. + items: + type: object + required: + - finish_reason + - index + - logprobs + - text + properties: + finish_reason: + type: string + description: &completion_finish_reason_description | + The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, + `length` if the maximum number of tokens specified in the request was reached, + or `content_filter` if content was omitted due to a flag from our content filters. + enum: ["stop", "length", "content_filter"] + index: + type: integer + logprobs: + type: object + nullable: true + properties: + text_offset: + type: array + items: + type: integer + token_logprobs: + type: array + items: + type: number + tokens: + type: array + items: + type: string + top_logprobs: + type: array + items: + type: object + additionalProperties: + type: number + text: + type: string + created: + type: integer + description: The Unix timestamp (in seconds) of when the completion was created. + model: + type: string + description: The model used for completion. + system_fingerprint: + type: string + description: | + This fingerprint represents the backend configuration that the model runs with. + + Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. + object: + type: string + description: The object type, which is always "text_completion" + enum: [text_completion] + usage: + $ref: "#/components/schemas/CompletionUsage" + required: + - id + - object + - created + - model + - choices + x-oaiMeta: + name: The completion object + legacy: true + example: | + { + "id": "cmpl-uqkvlQyYK7bGYrRHQ0eXlWi7", + "object": "text_completion", + "created": 1589478378, + "model": "gpt-4-turbo", + "choices": [ + { + "text": "\n\nThis is indeed a test", + "index": 0, + "logprobs": null, + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 7, + "total_tokens": 12 + } + } + + ChatCompletionRequestMessageContentPart: + oneOf: + - $ref: "#/components/schemas/ChatCompletionRequestMessageContentPartText" + - $ref: "#/components/schemas/ChatCompletionRequestMessageContentPartImage" + x-oaiExpandable: true + + ChatCompletionRequestMessageContentPartImage: + type: object + title: Image content part + properties: + type: + type: string + enum: ["image_url"] + description: The type of the content part. + image_url: + type: object + properties: + url: + type: string + description: Either a URL of the image or the base64 encoded image data. + format: uri + detail: + type: string + description: Specifies the detail level of the image. Learn more in the [Vision guide](/docs/guides/vision/low-or-high-fidelity-image-understanding). + enum: ["auto", "low", "high"] + default: "auto" + required: + - url + required: + - type + - image_url + + ChatCompletionRequestMessageContentPartText: + type: object + title: Text content part + properties: + type: + type: string + enum: ["text"] + description: The type of the content part. + text: + type: string + description: The text content. + required: + - type + - text + + ChatCompletionRequestMessage: + oneOf: + - $ref: "#/components/schemas/ChatCompletionRequestSystemMessage" + - $ref: "#/components/schemas/ChatCompletionRequestUserMessage" + - $ref: "#/components/schemas/ChatCompletionRequestAssistantMessage" + - $ref: "#/components/schemas/ChatCompletionRequestToolMessage" + - $ref: "#/components/schemas/ChatCompletionRequestFunctionMessage" + x-oaiExpandable: true + + ChatCompletionRequestSystemMessage: + type: object + title: System message + properties: + content: + description: The contents of the system message. + type: string + role: + type: string + enum: ["system"] + description: The role of the messages author, in this case `system`. + name: + type: string + description: An optional name for the participant. Provides the model information to differentiate between participants of the same role. + required: + - content + - role + + ChatCompletionRequestUserMessage: + type: object + title: User message + properties: + content: + description: | + The contents of the user message. + oneOf: + - type: string + description: The text contents of the message. + title: Text content + - type: array + description: An array of content parts with a defined type, each can be of type `text` or `image_url` when passing in images. You can pass multiple images by adding multiple `image_url` content parts. Image input is only supported when using the `gpt-4-visual-preview` model. + title: Array of content parts + items: + $ref: "#/components/schemas/ChatCompletionRequestMessageContentPart" + minItems: 1 + x-oaiExpandable: true + role: + type: string + enum: ["user"] + description: The role of the messages author, in this case `user`. + name: + type: string + description: An optional name for the participant. Provides the model information to differentiate between participants of the same role. + required: + - content + - role + + ChatCompletionRequestAssistantMessage: + type: object + title: Assistant message + properties: + content: + nullable: true + type: string + description: | + The contents of the assistant message. Required unless `tool_calls` or `function_call` is specified. + role: + type: string + enum: ["assistant"] + description: The role of the messages author, in this case `assistant`. + name: + type: string + description: An optional name for the participant. Provides the model information to differentiate between participants of the same role. + tool_calls: + $ref: "#/components/schemas/ChatCompletionMessageToolCalls" + function_call: + type: object + deprecated: true + description: "Deprecated and replaced by `tool_calls`. The name and arguments of a function that should be called, as generated by the model." + properties: + arguments: + type: string + description: The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function. + name: + type: string + description: The name of the function to call. + required: + - arguments + - name + required: + - role + + ChatCompletionRequestToolMessage: + type: object + title: Tool message + properties: + role: + type: string + enum: ["tool"] + description: The role of the messages author, in this case `tool`. + content: + type: string + description: The contents of the tool message. + tool_call_id: + type: string + description: Tool call that this message is responding to. + required: + - role + - content + - tool_call_id + + ChatCompletionRequestFunctionMessage: + type: object + title: Function message + deprecated: true + properties: + role: + type: string + enum: ["function"] + description: The role of the messages author, in this case `function`. + content: + nullable: true + type: string + description: The contents of the function message. + name: + type: string + description: The name of the function to call. + required: + - role + - content + - name + + FunctionParameters: + type: object + description: "The parameters the functions accepts, described as a JSON Schema object. See the [guide](/docs/guides/text-generation/function-calling) for examples, and the [JSON Schema reference](https://json-schema.org/understanding-json-schema/) for documentation about the format. \n\nOmitting `parameters` defines a function with an empty parameter list." + additionalProperties: true + + ChatCompletionFunctions: + type: object + deprecated: true + properties: + description: + type: string + description: A description of what the function does, used by the model to choose when and how to call the function. + name: + type: string + description: The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64. + parameters: + $ref: "#/components/schemas/FunctionParameters" + required: + - name + + ChatCompletionFunctionCallOption: + type: object + description: > + Specifying a particular function via `{"name": "my_function"}` forces the model to call that function. + properties: + name: + type: string + description: The name of the function to call. + required: + - name + + ChatCompletionTool: + type: object + properties: + type: + type: string + enum: ["function"] + description: The type of the tool. Currently, only `function` is supported. + function: + $ref: "#/components/schemas/FunctionObject" + required: + - type + - function + + FunctionObject: + type: object + properties: + description: + type: string + description: A description of what the function does, used by the model to choose when and how to call the function. + name: + type: string + description: The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64. + parameters: + $ref: "#/components/schemas/FunctionParameters" + required: + - name + + ChatCompletionToolChoiceOption: + description: | + Controls which (if any) tool is called by the model. + `none` means the model will not call any tool and instead generates a message. + `auto` means the model can pick between generating a message or calling one or more tools. + `required` means the model must call one or more tools. + Specifying a particular tool via `{"type": "function", "function": {"name": "my_function"}}` forces the model to call that tool. + + `none` is the default when no tools are present. `auto` is the default if tools are present. + oneOf: + - type: string + description: > + `none` means the model will not call any tool and instead generates a message. + `auto` means the model can pick between generating a message or calling one or more tools. + `required` means the model must call one or more tools. + enum: [none, auto, required] + - $ref: "#/components/schemas/ChatCompletionNamedToolChoice" + x-oaiExpandable: true + + ChatCompletionNamedToolChoice: + type: object + description: Specifies a tool the model should use. Use to force the model to call a specific function. + properties: + type: + type: string + enum: ["function"] + description: The type of the tool. Currently, only `function` is supported. + function: + type: object + properties: + name: + type: string + description: The name of the function to call. + required: + - name + required: + - type + - function + + ChatCompletionMessageToolCalls: + type: array + description: The tool calls generated by the model, such as function calls. + items: + $ref: "#/components/schemas/ChatCompletionMessageToolCall" + + ChatCompletionMessageToolCall: + type: object + properties: + # TODO: index included when streaming + id: + type: string + description: The ID of the tool call. + type: + type: string + enum: ["function"] + description: The type of the tool. Currently, only `function` is supported. + function: + type: object + description: The function that the model called. + properties: + name: + type: string + description: The name of the function to call. + arguments: + type: string + description: The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function. + required: + - name + - arguments + required: + - id + - type + - function + + ChatCompletionMessageToolCallChunk: + type: object + properties: + index: + type: integer + id: + type: string + description: The ID of the tool call. + type: + type: string + enum: ["function"] + description: The type of the tool. Currently, only `function` is supported. + function: + type: object + properties: + name: + type: string + description: The name of the function to call. + arguments: + type: string + description: The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function. + required: + - index + + # Note, this isn't referenced anywhere, but is kept as a convenience to record all possible roles in one place. + ChatCompletionRole: + type: string + description: The role of the author of a message + enum: + - system + - user + - assistant + - tool + - function + + ChatCompletionResponseMessage: + type: object + description: A chat completion message generated by the model. + properties: + content: + type: string + description: The contents of the message. + nullable: true + tool_calls: + $ref: "#/components/schemas/ChatCompletionMessageToolCalls" + role: + type: string + enum: ["assistant"] + description: The role of the author of this message. + function_call: + type: object + deprecated: true + description: "Deprecated and replaced by `tool_calls`. The name and arguments of a function that should be called, as generated by the model." + properties: + arguments: + type: string + description: The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function. + name: + type: string + description: The name of the function to call. + required: + - name + - arguments + required: + - role + - content + + ChatCompletionStreamResponseDelta: + type: object + description: A chat completion delta generated by streamed model responses. + properties: + content: + type: string + description: The contents of the chunk message. + nullable: true + function_call: + deprecated: true + type: object + description: "Deprecated and replaced by `tool_calls`. The name and arguments of a function that should be called, as generated by the model." + properties: + arguments: + type: string + description: The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function. + name: + type: string + description: The name of the function to call. + tool_calls: + type: array + items: + $ref: "#/components/schemas/ChatCompletionMessageToolCallChunk" + role: + type: string + enum: ["system", "user", "assistant", "tool"] + description: The role of the author of this message. + + CreateChatCompletionRequest: + type: object + properties: + messages: + description: A list of messages comprising the conversation so far. [Example Python code](https://cookbook.openai.com/examples/how_to_format_inputs_to_chatgpt_models). + type: array + minItems: 1 + items: + $ref: "#/components/schemas/ChatCompletionRequestMessage" + model: + description: ID of the model to use. See the [model endpoint compatibility](/docs/models/model-endpoint-compatibility) table for details on which models work with the Chat API. + example: "gpt-4-turbo" + anyOf: + - type: string + - type: string + enum: + [ + "gpt-4-turbo", + "gpt-4-turbo-2024-04-09", + "gpt-4-0125-preview", + "gpt-4-turbo-preview", + "gpt-4-1106-preview", + "gpt-4-vision-preview", + "gpt-4", + "gpt-4-0314", + "gpt-4-0613", + "gpt-4-32k", + "gpt-4-32k-0314", + "gpt-4-32k-0613", + "gpt-3.5-turbo", + "gpt-3.5-turbo-16k", + "gpt-3.5-turbo-0301", + "gpt-3.5-turbo-0613", + "gpt-3.5-turbo-1106", + "gpt-3.5-turbo-0125", + "gpt-3.5-turbo-16k-0613", + ] + x-oaiTypeLabel: string + frequency_penalty: + type: number + default: 0 + minimum: -2 + maximum: 2 + nullable: true + description: *completions_frequency_penalty_description + logit_bias: + type: object + x-oaiTypeLabel: map + default: null + nullable: true + additionalProperties: + type: integer + description: | + Modify the likelihood of specified tokens appearing in the completion. + + Accepts a JSON object that maps tokens (specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically, the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should result in a ban or exclusive selection of the relevant token. + logprobs: + description: Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the `content` of `message`. + type: boolean + default: false + nullable: true + top_logprobs: + description: An integer between 0 and 20 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to `true` if this parameter is used. + type: integer + minimum: 0 + maximum: 20 + nullable: true + max_tokens: + description: | + The maximum number of [tokens](/tokenizer) that can be generated in the chat completion. + + The total length of input tokens and generated tokens is limited by the model's context length. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens. + type: integer + nullable: true + n: + type: integer + minimum: 1 + maximum: 128 + default: 1 + example: 1 + nullable: true + description: How many chat completion choices to generate for each input message. Note that you will be charged based on the number of generated tokens across all of the choices. Keep `n` as `1` to minimize costs. + presence_penalty: + type: number + default: 0 + minimum: -2 + maximum: 2 + nullable: true + description: *completions_presence_penalty_description + response_format: + type: object + description: | + An object specifying the format that the model must output. Compatible with [GPT-4 Turbo](/docs/models/gpt-4-and-gpt-4-turbo) and all GPT-3.5 Turbo models newer than `gpt-3.5-turbo-1106`. + + Setting to `{ "type": "json_object" }` enables JSON mode, which guarantees the message the model generates is valid JSON. + + **Important:** when using JSON mode, you **must** also instruct the model to produce JSON yourself via a system or user message. Without this, the model may generate an unending stream of whitespace until the generation reaches the token limit, resulting in a long-running and seemingly "stuck" request. Also note that the message content may be partially cut off if `finish_reason="length"`, which indicates the generation exceeded `max_tokens` or the conversation exceeded the max context length. + properties: + type: + type: string + enum: ["text", "json_object"] + example: "json_object" + default: "text" + description: Must be one of `text` or `json_object`. + seed: + type: integer + minimum: -9223372036854775808 + maximum: 9223372036854775807 + nullable: true + description: | + This feature is in Beta. + If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same `seed` and parameters should return the same result. + Determinism is not guaranteed, and you should refer to the `system_fingerprint` response parameter to monitor changes in the backend. + x-oaiMeta: + beta: true + stop: + description: | + Up to 4 sequences where the API will stop generating further tokens. + default: null + oneOf: + - type: string + nullable: true + - type: array + minItems: 1 + maxItems: 4 + items: + type: string + stream: + description: > + If set, partial message deltas will be sent, like in ChatGPT. Tokens will be sent as data-only [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) + as they become available, with the stream terminated by a `data: [DONE]` message. [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions). + type: boolean + nullable: true + default: false + temperature: + type: number + minimum: 0 + maximum: 2 + default: 1 + example: 1 + nullable: true + description: *completions_temperature_description + top_p: + type: number + minimum: 0 + maximum: 1 + default: 1 + example: 1 + nullable: true + description: *completions_top_p_description + tools: + type: array + description: > + A list of tools the model may call. Currently, only functions are supported as a tool. + Use this to provide a list of functions the model may generate JSON inputs for. A max of 128 functions are supported. + items: + $ref: "#/components/schemas/ChatCompletionTool" + tool_choice: + $ref: "#/components/schemas/ChatCompletionToolChoiceOption" + user: *end_user_param_configuration + function_call: + deprecated: true + description: | + Deprecated in favor of `tool_choice`. + + Controls which (if any) function is called by the model. + `none` means the model will not call a function and instead generates a message. + `auto` means the model can pick between generating a message or calling a function. + Specifying a particular function via `{"name": "my_function"}` forces the model to call that function. + + `none` is the default when no functions are present. `auto` is the default if functions are present. + oneOf: + - type: string + description: > + `none` means the model will not call a function and instead generates a message. + `auto` means the model can pick between generating a message or calling a function. + enum: [none, auto] + - $ref: "#/components/schemas/ChatCompletionFunctionCallOption" + x-oaiExpandable: true + functions: + deprecated: true + description: | + Deprecated in favor of `tools`. + + A list of functions the model may generate JSON inputs for. + type: array + minItems: 1 + maxItems: 128 + items: + $ref: "#/components/schemas/ChatCompletionFunctions" + + required: + - model + - messages + + CreateChatCompletionResponse: + type: object + description: Represents a chat completion response returned by model, based on the provided input. + properties: + id: + type: string + description: A unique identifier for the chat completion. + choices: + type: array + description: A list of chat completion choices. Can be more than one if `n` is greater than 1. + items: + type: object + required: + - finish_reason + - index + - message + - logprobs + properties: + finish_reason: + type: string + description: &chat_completion_finish_reason_description | + The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, + `length` if the maximum number of tokens specified in the request was reached, + `content_filter` if content was omitted due to a flag from our content filters, + `tool_calls` if the model called a tool, or `function_call` (deprecated) if the model called a function. + enum: + [ + "stop", + "length", + "tool_calls", + "content_filter", + "function_call", + ] + index: + type: integer + description: The index of the choice in the list of choices. + message: + $ref: "#/components/schemas/ChatCompletionResponseMessage" + logprobs: &chat_completion_response_logprobs + description: Log probability information for the choice. + type: object + nullable: true + properties: + content: + description: A list of message content tokens with log probability information. + type: array + items: + $ref: "#/components/schemas/ChatCompletionTokenLogprob" + nullable: true + required: + - content + created: + type: integer + description: The Unix timestamp (in seconds) of when the chat completion was created. + model: + type: string + description: The model used for the chat completion. + system_fingerprint: + type: string + description: | + This fingerprint represents the backend configuration that the model runs with. + + Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. + object: + type: string + description: The object type, which is always `chat.completion`. + enum: [chat.completion] + usage: + $ref: "#/components/schemas/CompletionUsage" + required: + - choices + - created + - id + - model + - object + x-oaiMeta: + name: The chat completion object + group: chat + example: *chat_completion_example + + CreateChatCompletionFunctionResponse: + type: object + description: Represents a chat completion response returned by model, based on the provided input. + properties: + id: + type: string + description: A unique identifier for the chat completion. + choices: + type: array + description: A list of chat completion choices. Can be more than one if `n` is greater than 1. + items: + type: object + required: + - finish_reason + - index + - message + - logprobs + properties: + finish_reason: + type: string + description: + &chat_completion_function_finish_reason_description | + The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, `length` if the maximum number of tokens specified in the request was reached, `content_filter` if content was omitted due to a flag from our content filters, or `function_call` if the model called a function. + enum: ["stop", "length", "function_call", "content_filter"] + index: + type: integer + description: The index of the choice in the list of choices. + message: + $ref: "#/components/schemas/ChatCompletionResponseMessage" + created: + type: integer + description: The Unix timestamp (in seconds) of when the chat completion was created. + model: + type: string + description: The model used for the chat completion. + system_fingerprint: + type: string + description: | + This fingerprint represents the backend configuration that the model runs with. + + Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. + object: + type: string + description: The object type, which is always `chat.completion`. + enum: [chat.completion] + usage: + $ref: "#/components/schemas/CompletionUsage" + required: + - choices + - created + - id + - model + - object + x-oaiMeta: + name: The chat completion object + group: chat + example: *chat_completion_function_example + + ChatCompletionTokenLogprob: + type: object + properties: + token: &chat_completion_response_logprobs_token + description: The token. + type: string + logprob: &chat_completion_response_logprobs_token_logprob + description: The log probability of this token, if it is within the top 20 most likely tokens. Otherwise, the value `-9999.0` is used to signify that the token is very unlikely. + type: number + bytes: &chat_completion_response_logprobs_bytes + description: A list of integers representing the UTF-8 bytes representation of the token. Useful in instances where characters are represented by multiple tokens and their byte representations must be combined to generate the correct text representation. Can be `null` if there is no bytes representation for the token. + type: array + items: + type: integer + nullable: true + top_logprobs: + description: List of the most likely tokens and their log probability, at this token position. In rare cases, there may be fewer than the number of requested `top_logprobs` returned. + type: array + items: + type: object + properties: + token: *chat_completion_response_logprobs_token + logprob: *chat_completion_response_logprobs_token_logprob + bytes: *chat_completion_response_logprobs_bytes + required: + - token + - logprob + - bytes + required: + - token + - logprob + - bytes + - top_logprobs + + ListPaginatedFineTuningJobsResponse: + type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/FineTuningJob" + has_more: + type: boolean + object: + type: string + enum: [list] + required: + - object + - data + - has_more + + CreateChatCompletionStreamResponse: + type: object + description: Represents a streamed chunk of a chat completion response returned by model, based on the provided input. + properties: + id: + type: string + description: A unique identifier for the chat completion. Each chunk has the same ID. + choices: + type: array + description: A list of chat completion choices. Can be more than one if `n` is greater than 1. + items: + type: object + required: + - delta + - finish_reason + - index + properties: + delta: + $ref: "#/components/schemas/ChatCompletionStreamResponseDelta" + logprobs: *chat_completion_response_logprobs + finish_reason: + type: string + description: *chat_completion_finish_reason_description + enum: + [ + "stop", + "length", + "tool_calls", + "content_filter", + "function_call", + ] + nullable: true + index: + type: integer + description: The index of the choice in the list of choices. + created: + type: integer + description: The Unix timestamp (in seconds) of when the chat completion was created. Each chunk has the same timestamp. + model: + type: string + description: The model to generate the completion. + system_fingerprint: + type: string + description: | + This fingerprint represents the backend configuration that the model runs with. + Can be used in conjunction with the `seed` request parameter to understand when backend changes have been made that might impact determinism. + object: + type: string + description: The object type, which is always `chat.completion.chunk`. + enum: [chat.completion.chunk] + required: + - choices + - created + - id + - model + - object + x-oaiMeta: + name: The chat completion chunk object + group: chat + example: *chat_completion_chunk_example + + CreateChatCompletionImageResponse: + type: object + description: Represents a streamed chunk of a chat completion response returned by model, based on the provided input. + x-oaiMeta: + name: The chat completion chunk object + group: chat + example: *chat_completion_image_example + + CreateImageRequest: + type: object + properties: + prompt: + description: A text description of the desired image(s). The maximum length is 1000 characters for `dall-e-2` and 4000 characters for `dall-e-3`. + type: string + example: "A cute baby sea otter" + model: + anyOf: + - type: string + - type: string + enum: ["dall-e-2", "dall-e-3"] + x-oaiTypeLabel: string + default: "dall-e-2" + example: "dall-e-3" + nullable: true + description: The model to use for image generation. + n: &images_n + type: integer + minimum: 1 + maximum: 10 + default: 1 + example: 1 + nullable: true + description: The number of images to generate. Must be between 1 and 10. For `dall-e-3`, only `n=1` is supported. + quality: + type: string + enum: ["standard", "hd"] + default: "standard" + example: "standard" + description: The quality of the image that will be generated. `hd` creates images with finer details and greater consistency across the image. This param is only supported for `dall-e-3`. + response_format: &images_response_format + type: string + enum: ["url", "b64_json"] + default: "url" + example: "url" + nullable: true + description: The format in which the generated images are returned. Must be one of `url` or `b64_json`. URLs are only valid for 60 minutes after the image has been generated. + size: &images_size + type: string + enum: ["256x256", "512x512", "1024x1024", "1792x1024", "1024x1792"] + default: "1024x1024" + example: "1024x1024" + nullable: true + description: The size of the generated images. Must be one of `256x256`, `512x512`, or `1024x1024` for `dall-e-2`. Must be one of `1024x1024`, `1792x1024`, or `1024x1792` for `dall-e-3` models. + style: + type: string + enum: ["vivid", "natural"] + default: "vivid" + example: "vivid" + nullable: true + description: The style of the generated images. Must be one of `vivid` or `natural`. Vivid causes the model to lean towards generating hyper-real and dramatic images. Natural causes the model to produce more natural, less hyper-real looking images. This param is only supported for `dall-e-3`. + user: *end_user_param_configuration + required: + - prompt + + ImagesResponse: + properties: + created: + type: integer + data: + type: array + items: + $ref: "#/components/schemas/Image" + required: + - created + - data + + Image: + type: object + description: Represents the url or the content of an image generated by the OpenAI API. + properties: + b64_json: + type: string + description: The base64-encoded JSON of the generated image, if `response_format` is `b64_json`. + url: + type: string + description: The URL of the generated image, if `response_format` is `url` (default). + revised_prompt: + type: string + description: The prompt that was used to generate the image, if there was any revision to the prompt. + x-oaiMeta: + name: The image object + example: | + { + "url": "...", + "revised_prompt": "..." + } + + CreateImageEditRequest: + type: object + properties: + image: + description: The image to edit. Must be a valid PNG file, less than 4MB, and square. If mask is not provided, image must have transparency, which will be used as the mask. + type: string + format: binary + prompt: + description: A text description of the desired image(s). The maximum length is 1000 characters. + type: string + example: "A cute baby sea otter wearing a beret" + mask: + description: An additional image whose fully transparent areas (e.g. where alpha is zero) indicate where `image` should be edited. Must be a valid PNG file, less than 4MB, and have the same dimensions as `image`. + type: string + format: binary + model: + anyOf: + - type: string + - type: string + enum: ["dall-e-2"] + x-oaiTypeLabel: string + default: "dall-e-2" + example: "dall-e-2" + nullable: true + description: The model to use for image generation. Only `dall-e-2` is supported at this time. + n: + type: integer + minimum: 1 + maximum: 10 + default: 1 + example: 1 + nullable: true + description: The number of images to generate. Must be between 1 and 10. + size: &dalle2_images_size + type: string + enum: ["256x256", "512x512", "1024x1024"] + default: "1024x1024" + example: "1024x1024" + nullable: true + description: The size of the generated images. Must be one of `256x256`, `512x512`, or `1024x1024`. + response_format: *images_response_format + user: *end_user_param_configuration + required: + - prompt + - image + + CreateImageVariationRequest: + type: object + properties: + image: + description: The image to use as the basis for the variation(s). Must be a valid PNG file, less than 4MB, and square. + type: string + format: binary + model: + anyOf: + - type: string + - type: string + enum: ["dall-e-2"] + x-oaiTypeLabel: string + default: "dall-e-2" + example: "dall-e-2" + nullable: true + description: The model to use for image generation. Only `dall-e-2` is supported at this time. + n: *images_n + response_format: *images_response_format + size: *dalle2_images_size + user: *end_user_param_configuration + required: + - image + + CreateModerationRequest: + type: object + properties: + input: + description: The input text to classify + oneOf: + - type: string + default: "" + example: "I want to kill them." + - type: array + items: + type: string + default: "" + example: "I want to kill them." + model: + description: | + Two content moderations models are available: `text-moderation-stable` and `text-moderation-latest`. + + The default is `text-moderation-latest` which will be automatically upgraded over time. This ensures you are always using our most accurate model. If you use `text-moderation-stable`, we will provide advanced notice before updating the model. Accuracy of `text-moderation-stable` may be slightly lower than for `text-moderation-latest`. + nullable: false + default: "text-moderation-latest" + example: "text-moderation-stable" + anyOf: + - type: string + - type: string + enum: ["text-moderation-latest", "text-moderation-stable"] + x-oaiTypeLabel: string + required: + - input + + CreateModerationResponse: + type: object + description: Represents if a given text input is potentially harmful. + properties: + id: + type: string + description: The unique identifier for the moderation request. + model: + type: string + description: The model used to generate the moderation results. + results: + type: array + description: A list of moderation objects. + items: + type: object + properties: + flagged: + type: boolean + description: Whether any of the below categories are flagged. + categories: + type: object + description: A list of the categories, and whether they are flagged or not. + properties: + hate: + type: boolean + description: Content that expresses, incites, or promotes hate based on race, gender, ethnicity, religion, nationality, sexual orientation, disability status, or caste. Hateful content aimed at non-protected groups (e.g., chess players) is harassment. + hate/threatening: + type: boolean + description: Hateful content that also includes violence or serious harm towards the targeted group based on race, gender, ethnicity, religion, nationality, sexual orientation, disability status, or caste. + harassment: + type: boolean + description: Content that expresses, incites, or promotes harassing language towards any target. + harassment/threatening: + type: boolean + description: Harassment content that also includes violence or serious harm towards any target. + self-harm: + type: boolean + description: Content that promotes, encourages, or depicts acts of self-harm, such as suicide, cutting, and eating disorders. + self-harm/intent: + type: boolean + description: Content where the speaker expresses that they are engaging or intend to engage in acts of self-harm, such as suicide, cutting, and eating disorders. + self-harm/instructions: + type: boolean + description: Content that encourages performing acts of self-harm, such as suicide, cutting, and eating disorders, or that gives instructions or advice on how to commit such acts. + sexual: + type: boolean + description: Content meant to arouse sexual excitement, such as the description of sexual activity, or that promotes sexual services (excluding sex education and wellness). + sexual/minors: + type: boolean + description: Sexual content that includes an individual who is under 18 years old. + violence: + type: boolean + description: Content that depicts death, violence, or physical injury. + violence/graphic: + type: boolean + description: Content that depicts death, violence, or physical injury in graphic detail. + required: + - hate + - hate/threatening + - harassment + - harassment/threatening + - self-harm + - self-harm/intent + - self-harm/instructions + - sexual + - sexual/minors + - violence + - violence/graphic + category_scores: + type: object + description: A list of the categories along with their scores as predicted by model. + properties: + hate: + type: number + description: The score for the category 'hate'. + hate/threatening: + type: number + description: The score for the category 'hate/threatening'. + harassment: + type: number + description: The score for the category 'harassment'. + harassment/threatening: + type: number + description: The score for the category 'harassment/threatening'. + self-harm: + type: number + description: The score for the category 'self-harm'. + self-harm/intent: + type: number + description: The score for the category 'self-harm/intent'. + self-harm/instructions: + type: number + description: The score for the category 'self-harm/instructions'. + sexual: + type: number + description: The score for the category 'sexual'. + sexual/minors: + type: number + description: The score for the category 'sexual/minors'. + violence: + type: number + description: The score for the category 'violence'. + violence/graphic: + type: number + description: The score for the category 'violence/graphic'. + required: + - hate + - hate/threatening + - harassment + - harassment/threatening + - self-harm + - self-harm/intent + - self-harm/instructions + - sexual + - sexual/minors + - violence + - violence/graphic + required: + - flagged + - categories + - category_scores + required: + - id + - model + - results + x-oaiMeta: + name: The moderation object + example: *moderation_example + + ListFilesResponse: + type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/OpenAIFile" + object: + type: string + enum: [list] + required: + - object + - data + + CreateFileRequest: + type: object + additionalProperties: false + properties: + file: + description: | + The File object (not file name) to be uploaded. + type: string + format: binary + purpose: + description: | + The intended purpose of the uploaded file. + + Use "fine-tune" for [Fine-tuning](/docs/api-reference/fine-tuning) and "assistants" for [Assistants](/docs/api-reference/assistants) and [Messages](/docs/api-reference/messages). This allows us to validate the format of the uploaded file is correct for fine-tuning. + type: string + enum: ["fine-tune", "assistants"] + required: + - file + - purpose + + DeleteFileResponse: + type: object + properties: + id: + type: string + object: + type: string + enum: [file] + deleted: + type: boolean + required: + - id + - object + - deleted + + CreateFineTuningJobRequest: + type: object + properties: + model: + description: | + The name of the model to fine-tune. You can select one of the + [supported models](/docs/guides/fine-tuning/what-models-can-be-fine-tuned). + example: "gpt-3.5-turbo" + anyOf: + - type: string + - type: string + enum: ["babbage-002", "davinci-002", "gpt-3.5-turbo"] + x-oaiTypeLabel: string + training_file: + description: | + The ID of an uploaded file that contains training data. + + See [upload file](/docs/api-reference/files/create) for how to upload a file. + + Your dataset must be formatted as a JSONL file. Additionally, you must upload your file with the purpose `fine-tune`. + + See the [fine-tuning guide](/docs/guides/fine-tuning) for more details. + type: string + example: "file-abc123" + hyperparameters: + type: object + description: The hyperparameters used for the fine-tuning job. + properties: + batch_size: + description: | + Number of examples in each batch. A larger batch size means that model parameters + are updated less frequently, but with lower variance. + oneOf: + - type: string + enum: [auto] + - type: integer + minimum: 1 + maximum: 256 + default: auto + learning_rate_multiplier: + description: | + Scaling factor for the learning rate. A smaller learning rate may be useful to avoid + overfitting. + oneOf: + - type: string + enum: [auto] + - type: number + minimum: 0 + exclusiveMinimum: true + default: auto + n_epochs: + description: | + The number of epochs to train the model for. An epoch refers to one full cycle + through the training dataset. + oneOf: + - type: string + enum: [auto] + - type: integer + minimum: 1 + maximum: 50 + default: auto + suffix: + description: | + A string of up to 18 characters that will be added to your fine-tuned model name. + + For example, a `suffix` of "custom-model-name" would produce a model name like `ft:gpt-3.5-turbo:openai:custom-model-name:7p4lURel`. + type: string + minLength: 1 + maxLength: 40 + default: null + nullable: true + validation_file: + description: | + The ID of an uploaded file that contains validation data. + + If you provide this file, the data is used to generate validation + metrics periodically during fine-tuning. These metrics can be viewed in + the fine-tuning results file. + The same data should not be present in both train and validation files. + + Your dataset must be formatted as a JSONL file. You must upload your file with the purpose `fine-tune`. + + See the [fine-tuning guide](/docs/guides/fine-tuning) for more details. + type: string + nullable: true + example: "file-abc123" + integrations: + type: array + description: A list of integrations to enable for your fine-tuning job. + nullable: true + items: + type: object + required: + - type + - wandb + properties: + type: + description: | + The type of integration to enable. Currently, only "wandb" (Weights and Biases) is supported. + oneOf: + - type: string + enum: [wandb] + wandb: + type: object + description: | + The settings for your integration with Weights and Biases. This payload specifies the project that + metrics will be sent to. Optionally, you can set an explicit display name for your run, add tags + to your run, and set a default entity (team, username, etc) to be associated with your run. + required: + - project + properties: + project: + description: | + The name of the project that the new run will be created under. + type: string + example: "my-wandb-project" + name: + description: | + A display name to set for the run. If not set, we will use the Job ID as the name. + nullable: true + type: string + entity: + description: | + The entity to use for the run. This allows you to set the team or username of the WandB user that you would + like associated with the run. If not set, the default entity for the registered WandB API key is used. + nullable: true + type: string + tags: + description: | + A list of tags to be attached to the newly created run. These tags are passed through directly to WandB. Some + default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". + type: array + items: + type: string + example: "custom-tag" + + seed: + description: | + The seed controls the reproducibility of the job. Passing in the same seed and job parameters should produce the same results, but may differ in rare cases. + If a seed is not specified, one will be generated for you. + type: integer + nullable: true + minimum: 0 + maximum: 2147483647 + example: 42 + required: + - model + - training_file + + ListFineTuningJobEventsResponse: + type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/FineTuningJobEvent" + object: + type: string + enum: [list] + required: + - object + - data + + ListFineTuningJobCheckpointsResponse: + type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/FineTuningJobCheckpoint" + object: + type: string + enum: [list] + first_id: + type: string + nullable: true + last_id: + type: string + nullable: true + has_more: + type: boolean + required: + - object + - data + - has_more + + CreateEmbeddingRequest: + type: object + additionalProperties: false + properties: + input: + description: | + Input text to embed, encoded as a string or array of tokens. To embed multiple inputs in a single request, pass an array of strings or array of token arrays. The input must not exceed the max input tokens for the model (8192 tokens for `text-embedding-ada-002`), cannot be an empty string, and any array must be 2048 dimensions or less. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens. + example: "The quick brown fox jumped over the lazy dog" + oneOf: + - type: string + title: string + description: The string that will be turned into an embedding. + default: "" + example: "This is a test." + - type: array + title: array + description: The array of strings that will be turned into an embedding. + minItems: 1 + maxItems: 2048 + items: + type: string + default: "" + example: "['This is a test.']" + - type: array + title: array + description: The array of integers that will be turned into an embedding. + minItems: 1 + maxItems: 2048 + items: + type: integer + example: "[1212, 318, 257, 1332, 13]" + - type: array + title: array + description: The array of arrays containing integers that will be turned into an embedding. + minItems: 1 + maxItems: 2048 + items: + type: array + minItems: 1 + items: + type: integer + example: "[[1212, 318, 257, 1332, 13]]" + x-oaiExpandable: true + model: + description: *model_description + example: "text-embedding-3-small" + anyOf: + - type: string + - type: string + enum: + [ + "text-embedding-ada-002", + "text-embedding-3-small", + "text-embedding-3-large", + ] + x-oaiTypeLabel: string + encoding_format: + description: "The format to return the embeddings in. Can be either `float` or [`base64`](https://pypi.org/project/pybase64/)." + example: "float" + default: "float" + type: string + enum: ["float", "base64"] + dimensions: + description: | + The number of dimensions the resulting output embeddings should have. Only supported in `text-embedding-3` and later models. + type: integer + minimum: 1 + user: *end_user_param_configuration + required: + - model + - input + + CreateEmbeddingResponse: + type: object + properties: + data: + type: array + description: The list of embeddings generated by the model. + items: + $ref: "#/components/schemas/Embedding" + model: + type: string + description: The name of the model used to generate the embedding. + object: + type: string + description: The object type, which is always "list". + enum: [list] + usage: + type: object + description: The usage information for the request. + properties: + prompt_tokens: + type: integer + description: The number of tokens used by the prompt. + total_tokens: + type: integer + description: The total number of tokens used by the request. + required: + - prompt_tokens + - total_tokens + required: + - object + - model + - data + - usage + + CreateTranscriptionRequest: + type: object + additionalProperties: false + properties: + file: + description: | + The audio file object (not file name) to transcribe, in one of these formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm. + type: string + x-oaiTypeLabel: file + format: binary + model: + description: | + ID of the model to use. Only `whisper-1` (which is powered by our open source Whisper V2 model) is currently available. + example: whisper-1 + anyOf: + - type: string + - type: string + enum: ["whisper-1"] + x-oaiTypeLabel: string + language: + description: | + The language of the input audio. Supplying the input language in [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format will improve accuracy and latency. + type: string + prompt: + description: | + An optional text to guide the model's style or continue a previous audio segment. The [prompt](/docs/guides/speech-to-text/prompting) should match the audio language. + type: string + response_format: + description: | + The format of the transcript output, in one of these options: `json`, `text`, `srt`, `verbose_json`, or `vtt`. + type: string + enum: + - json + - text + - srt + - verbose_json + - vtt + default: json + temperature: + description: | + The sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use [log probability](https://en.wikipedia.org/wiki/Log_probability) to automatically increase the temperature until certain thresholds are hit. + type: number + default: 0 + timestamp_granularities[]: + description: | + The timestamp granularities to populate for this transcription. `response_format` must be set `verbose_json` to use timestamp granularities. Either or both of these options are supported: `word`, or `segment`. Note: There is no additional latency for segment timestamps, but generating word timestamps incurs additional latency. + type: array + items: + type: string + enum: + - word + - segment + default: [segment] + required: + - file + - model + + # Note: This does not currently support the non-default response format types. + CreateTranscriptionResponseJson: + type: object + description: Represents a transcription response returned by model, based on the provided input. + properties: + text: + type: string + description: The transcribed text. + required: + - text + x-oaiMeta: + name: The transcription object + group: audio + example: *basic_transcription_response_example + + TranscriptionSegment: + type: object + properties: + id: + type: integer + description: Unique identifier of the segment. + seek: + type: integer + description: Seek offset of the segment. + start: + type: number + format: float + description: Start time of the segment in seconds. + end: + type: number + format: float + description: End time of the segment in seconds. + text: + type: string + description: Text content of the segment. + tokens: + type: array + items: + type: integer + description: Array of token IDs for the text content. + temperature: + type: number + format: float + description: Temperature parameter used for generating the segment. + avg_logprob: + type: number + format: float + description: Average logprob of the segment. If the value is lower than -1, consider the logprobs failed. + compression_ratio: + type: number + format: float + description: Compression ratio of the segment. If the value is greater than 2.4, consider the compression failed. + no_speech_prob: + type: number + format: float + description: Probability of no speech in the segment. If the value is higher than 1.0 and the `avg_logprob` is below -1, consider this segment silent. + required: + - id + - seek + - start + - end + - text + - tokens + - temperature + - avg_logprob + - compression_ratio + - no_speech_prob + + TranscriptionWord: + type: object + properties: + word: + type: string + description: The text content of the word. + start: + type: number + format: float + description: Start time of the word in seconds. + end: + type: number + format: float + description: End time of the word in seconds. + required: [word, start, end] + + CreateTranscriptionResponseVerboseJson: + type: object + description: Represents a verbose json transcription response returned by model, based on the provided input. + properties: + language: + type: string + description: The language of the input audio. + duration: + type: string + description: The duration of the input audio. + text: + type: string + description: The transcribed text. + words: + type: array + description: Extracted words and their corresponding timestamps. + items: + $ref: "#/components/schemas/TranscriptionWord" + segments: + type: array + description: Segments of the transcribed text and their corresponding details. + items: + $ref: "#/components/schemas/TranscriptionSegment" + required: [language, duration, text] + x-oaiMeta: + name: The transcription object + group: audio + example: *verbose_transcription_response_example + + CreateTranslationRequest: + type: object + additionalProperties: false + properties: + file: + description: | + The audio file object (not file name) translate, in one of these formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm. + type: string + x-oaiTypeLabel: file + format: binary + model: + description: | + ID of the model to use. Only `whisper-1` (which is powered by our open source Whisper V2 model) is currently available. + example: whisper-1 + anyOf: + - type: string + - type: string + enum: ["whisper-1"] + x-oaiTypeLabel: string + prompt: + description: | + An optional text to guide the model's style or continue a previous audio segment. The [prompt](/docs/guides/speech-to-text/prompting) should be in English. + type: string + response_format: + description: | + The format of the transcript output, in one of these options: `json`, `text`, `srt`, `verbose_json`, or `vtt`. + type: string + default: json + temperature: + description: | + The sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use [log probability](https://en.wikipedia.org/wiki/Log_probability) to automatically increase the temperature until certain thresholds are hit. + type: number + default: 0 + required: + - file + - model + + # Note: This does not currently support the non-default response format types. + CreateTranslationResponseJson: + type: object + properties: + text: + type: string + required: + - text + + CreateTranslationResponseVerboseJson: + type: object + properties: + language: + type: string + description: The language of the output translation (always `english`). + duration: + type: string + description: The duration of the input audio. + text: + type: string + description: The translated text. + segments: + type: array + description: Segments of the translated text and their corresponding details. + items: + $ref: "#/components/schemas/TranscriptionSegment" + required: [language, duration, text] + + CreateSpeechRequest: + type: object + additionalProperties: false + properties: + model: + description: | + One of the available [TTS models](/docs/models/tts): `tts-1` or `tts-1-hd` + anyOf: + - type: string + - type: string + enum: ["tts-1", "tts-1-hd"] + x-oaiTypeLabel: string + input: + type: string + description: The text to generate audio for. The maximum length is 4096 characters. + maxLength: 4096 + voice: + description: The voice to use when generating the audio. Supported voices are `alloy`, `echo`, `fable`, `onyx`, `nova`, and `shimmer`. Previews of the voices are available in the [Text to speech guide](/docs/guides/text-to-speech/voice-options). + type: string + enum: ["alloy", "echo", "fable", "onyx", "nova", "shimmer"] + response_format: + description: "The format to audio in. Supported formats are `mp3`, `opus`, `aac`, `flac`, `wav`, and `pcm`." + default: "mp3" + type: string + enum: ["mp3", "opus", "aac", "flac", "wav", "pcm"] + speed: + description: "The speed of the generated audio. Select a value from `0.25` to `4.0`. `1.0` is the default." + type: number + default: 1.0 + minimum: 0.25 + maximum: 4.0 + required: + - model + - input + - voice + + Model: + title: Model + description: Describes an OpenAI model offering that can be used with the API. + properties: + id: + type: string + description: The model identifier, which can be referenced in the API endpoints. + created: + type: integer + description: The Unix timestamp (in seconds) when the model was created. + object: + type: string + description: The object type, which is always "model". + enum: [model] + owned_by: + type: string + description: The organization that owns the model. + required: + - id + - object + - created + - owned_by + x-oaiMeta: + name: The model object + example: *retrieve_model_response + + OpenAIFile: + title: OpenAIFile + description: The `File` object represents a document that has been uploaded to OpenAI. + properties: + id: + type: string + description: The file identifier, which can be referenced in the API endpoints. + bytes: + type: integer + description: The size of the file, in bytes. + created_at: + type: integer + description: The Unix timestamp (in seconds) for when the file was created. + filename: + type: string + description: The name of the file. + object: + type: string + description: The object type, which is always `file`. + enum: ["file"] + purpose: + type: string + description: The intended purpose of the file. Supported values are `fine-tune`, `fine-tune-results`, `assistants`, and `assistants_output`. + enum: + [ + "fine-tune", + "fine-tune-results", + "assistants", + "assistants_output", + ] + status: + type: string + deprecated: true + description: Deprecated. The current status of the file, which can be either `uploaded`, `processed`, or `error`. + enum: ["uploaded", "processed", "error"] + status_details: + type: string + deprecated: true + description: Deprecated. For details on why a fine-tuning training file failed validation, see the `error` field on `fine_tuning.job`. + required: + - id + - object + - bytes + - created_at + - filename + - purpose + - status + x-oaiMeta: + name: The file object + example: | + { + "id": "file-abc123", + "object": "file", + "bytes": 120000, + "created_at": 1677610602, + "filename": "salesOverview.pdf", + "purpose": "assistants", + } + Embedding: + type: object + description: | + Represents an embedding vector returned by embedding endpoint. + properties: + index: + type: integer + description: The index of the embedding in the list of embeddings. + embedding: + type: array + description: | + The embedding vector, which is a list of floats. The length of vector depends on the model as listed in the [embedding guide](/docs/guides/embeddings). + items: + type: number + object: + type: string + description: The object type, which is always "embedding". + enum: [embedding] + required: + - index + - object + - embedding + x-oaiMeta: + name: The embedding object + example: | + { + "object": "embedding", + "embedding": [ + 0.0023064255, + -0.009327292, + .... (1536 floats total for ada-002) + -0.0028842222, + ], + "index": 0 + } + + FineTuningJob: + type: object + title: FineTuningJob + description: | + The `fine_tuning.job` object represents a fine-tuning job that has been created through the API. + properties: + id: + type: string + description: The object identifier, which can be referenced in the API endpoints. + created_at: + type: integer + description: The Unix timestamp (in seconds) for when the fine-tuning job was created. + error: + type: object + nullable: true + description: For fine-tuning jobs that have `failed`, this will contain more information on the cause of the failure. + properties: + code: + type: string + description: A machine-readable error code. + message: + type: string + description: A human-readable error message. + param: + type: string + description: The parameter that was invalid, usually `training_file` or `validation_file`. This field will be null if the failure was not parameter-specific. + nullable: true + required: + - code + - message + - param + fine_tuned_model: + type: string + nullable: true + description: The name of the fine-tuned model that is being created. The value will be null if the fine-tuning job is still running. + finished_at: + type: integer + nullable: true + description: The Unix timestamp (in seconds) for when the fine-tuning job was finished. The value will be null if the fine-tuning job is still running. + hyperparameters: + type: object + description: The hyperparameters used for the fine-tuning job. See the [fine-tuning guide](/docs/guides/fine-tuning) for more details. + properties: + n_epochs: + oneOf: + - type: string + enum: [auto] + - type: integer + minimum: 1 + maximum: 50 + default: auto + description: + The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. + + "auto" decides the optimal number of epochs based on the size of the dataset. If setting the number manually, we support any number between 1 and 50 epochs. + required: + - n_epochs + model: + type: string + description: The base model that is being fine-tuned. + object: + type: string + description: The object type, which is always "fine_tuning.job". + enum: [fine_tuning.job] + organization_id: + type: string + description: The organization that owns the fine-tuning job. + result_files: + type: array + description: The compiled results file ID(s) for the fine-tuning job. You can retrieve the results with the [Files API](/docs/api-reference/files/retrieve-contents). + items: + type: string + example: file-abc123 + status: + type: string + description: The current status of the fine-tuning job, which can be either `validating_files`, `queued`, `running`, `succeeded`, `failed`, or `cancelled`. + enum: + [ + "validating_files", + "queued", + "running", + "succeeded", + "failed", + "cancelled", + ] + trained_tokens: + type: integer + nullable: true + description: The total number of billable tokens processed by this fine-tuning job. The value will be null if the fine-tuning job is still running. + training_file: + type: string + description: The file ID used for training. You can retrieve the training data with the [Files API](/docs/api-reference/files/retrieve-contents). + validation_file: + type: string + nullable: true + description: The file ID used for validation. You can retrieve the validation results with the [Files API](/docs/api-reference/files/retrieve-contents). + integrations: + type: array + nullable: true + description: A list of integrations to enable for this fine-tuning job. + maxItems: 5 + items: + oneOf: + - $ref: "#/components/schemas/FineTuningIntegration" + x-oaiExpandable: true + seed: + type: integer + description: The seed used for the fine-tuning job. + estimated_finish: + type: integer + nullable: true + description: The Unix timestamp (in seconds) for when the fine-tuning job is estimated to finish. The value will be null if the fine-tuning job is not running. + required: + - created_at + - error + - finished_at + - fine_tuned_model + - hyperparameters + - id + - model + - object + - organization_id + - result_files + - status + - trained_tokens + - training_file + - validation_file + - seed + x-oaiMeta: + name: The fine-tuning job object + example: *fine_tuning_example + + FineTuningIntegration: + type: object + title: Fine-Tuning Job Integration + required: + - type + - wandb + properties: + type: + type: string + description: "The type of the integration being enabled for the fine-tuning job" + enum: ["wandb"] + wandb: + type: object + description: | + The settings for your integration with Weights and Biases. This payload specifies the project that + metrics will be sent to. Optionally, you can set an explicit display name for your run, add tags + to your run, and set a default entity (team, username, etc) to be associated with your run. + required: + - project + properties: + project: + description: | + The name of the project that the new run will be created under. + type: string + example: "my-wandb-project" + name: + description: | + A display name to set for the run. If not set, we will use the Job ID as the name. + nullable: true + type: string + entity: + description: | + The entity to use for the run. This allows you to set the team or username of the WandB user that you would + like associated with the run. If not set, the default entity for the registered WandB API key is used. + nullable: true + type: string + tags: + description: | + A list of tags to be attached to the newly created run. These tags are passed through directly to WandB. Some + default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". + type: array + items: + type: string + example: "custom-tag" + + FineTuningJobEvent: + type: object + description: Fine-tuning job event object + properties: + id: + type: string + created_at: + type: integer + level: + type: string + enum: ["info", "warn", "error"] + message: + type: string + object: + type: string + enum: [fine_tuning.job.event] + required: + - id + - object + - created_at + - level + - message + x-oaiMeta: + name: The fine-tuning job event object + example: | + { + "object": "fine_tuning.job.event", + "id": "ftevent-abc123" + "created_at": 1677610602, + "level": "info", + "message": "Created fine-tuning job" + } + + FineTuningJobCheckpoint: + type: object + title: FineTuningJobCheckpoint + description: | + The `fine_tuning.job.checkpoint` object represents a model checkpoint for a fine-tuning job that is ready to use. + properties: + id: + type: string + description: The checkpoint identifier, which can be referenced in the API endpoints. + created_at: + type: integer + description: The Unix timestamp (in seconds) for when the checkpoint was created. + fine_tuned_model_checkpoint: + type: string + description: The name of the fine-tuned checkpoint model that is created. + step_number: + type: integer + description: The step number that the checkpoint was created at. + metrics: + type: object + description: Metrics at the step number during the fine-tuning job. + properties: + step: + type: number + train_loss: + type: number + train_mean_token_accuracy: + type: number + valid_loss: + type: number + valid_mean_token_accuracy: + type: number + full_valid_loss: + type: number + full_valid_mean_token_accuracy: + type: number + fine_tuning_job_id: + type: string + description: The name of the fine-tuning job that this checkpoint was created from. + object: + type: string + description: The object type, which is always "fine_tuning.job.checkpoint". + enum: [fine_tuning.job.checkpoint] + required: + - created_at + - fine_tuning_job_id + - fine_tuned_model_checkpoint + - id + - metrics + - object + - step_number + x-oaiMeta: + name: The fine-tuning job checkpoint object + example: | + { + "object": "fine_tuning.job.checkpoint", + "id": "ftckpt_qtZ5Gyk4BLq1SfLFWp3RtO3P", + "created_at": 1712211699, + "fine_tuned_model_checkpoint": "ft:gpt-3.5-turbo-0125:my-org:custom_suffix:9ABel2dg:ckpt-step-88", + "fine_tuning_job_id": "ftjob-fpbNQ3H1GrMehXRf8cO97xTN", + "metrics": { + "step": 88, + "train_loss": 0.478, + "train_mean_token_accuracy": 0.924, + "valid_loss": 10.112, + "valid_mean_token_accuracy": 0.145, + "full_valid_loss": 0.567, + "full_valid_mean_token_accuracy": 0.944 + }, + "step_number": 88 + } + + CompletionUsage: + type: object + description: Usage statistics for the completion request. + properties: + completion_tokens: + type: integer + description: Number of tokens in the generated completion. + prompt_tokens: + type: integer + description: Number of tokens in the prompt. + total_tokens: + type: integer + description: Total number of tokens used in the request (prompt + completion). + required: + - prompt_tokens + - completion_tokens + - total_tokens + + RunCompletionUsage: + type: object + description: Usage statistics related to the run. This value will be `null` if the run is not in a terminal state (i.e. `in_progress`, `queued`, etc.). + properties: + completion_tokens: + type: integer + description: Number of completion tokens used over the course of the run. + prompt_tokens: + type: integer + description: Number of prompt tokens used over the course of the run. + total_tokens: + type: integer + description: Total number of tokens used (prompt + completion). + required: + - prompt_tokens + - completion_tokens + - total_tokens + nullable: true + + RunStepCompletionUsage: + type: object + description: Usage statistics related to the run step. This value will be `null` while the run step's status is `in_progress`. + properties: + completion_tokens: + type: integer + description: Number of completion tokens used over the course of the run step. + prompt_tokens: + type: integer + description: Number of prompt tokens used over the course of the run step. + total_tokens: + type: integer + description: Total number of tokens used (prompt + completion). + required: + - prompt_tokens + - completion_tokens + - total_tokens + nullable: true + + AssistantsApiResponseFormatOption: + description: | + Specifies the format that the model must output. Compatible with [GPT-4 Turbo](/docs/models/gpt-4-and-gpt-4-turbo) and all GPT-3.5 Turbo models since `gpt-3.5-turbo-1106`. + + Setting to `{ "type": "json_object" }` enables JSON mode, which guarantees the message the model generates is valid JSON. + + **Important:** when using JSON mode, you **must** also instruct the model to produce JSON yourself via a system or user message. Without this, the model may generate an unending stream of whitespace until the generation reaches the token limit, resulting in a long-running and seemingly "stuck" request. Also note that the message content may be partially cut off if `finish_reason="length"`, which indicates the generation exceeded `max_tokens` or the conversation exceeded the max context length. + oneOf: + - type: string + description: > + `auto` is the default value + enum: [none, auto] + - $ref: "#/components/schemas/AssistantsApiResponseFormat" + x-oaiExpandable: true + + AssistantsApiResponseFormat: + type: object + description: | + An object describing the expected output of the model. If `json_object` only `function` type `tools` are allowed to be passed to the Run. If `text` the model can return text or any value needed. + properties: + type: + type: string + enum: ["text", "json_object"] + example: "json_object" + default: "text" + description: Must be one of `text` or `json_object`. + + AssistantObject: + type: object + title: Assistant + description: Represents an `assistant` that can call the model and use tools. + properties: + id: + description: The identifier, which can be referenced in API endpoints. + type: string + object: + description: The object type, which is always `assistant`. + type: string + enum: [assistant] + created_at: + description: The Unix timestamp (in seconds) for when the assistant was created. + type: integer + name: + description: &assistant_name_param_description | + The name of the assistant. The maximum length is 256 characters. + type: string + maxLength: 256 + nullable: true + description: + description: &assistant_description_param_description | + The description of the assistant. The maximum length is 512 characters. + type: string + maxLength: 512 + nullable: true + model: + description: *model_description + type: string + instructions: + description: &assistant_instructions_param_description | + The system instructions that the assistant uses. The maximum length is 256,000 characters. + type: string + maxLength: 256000 + nullable: true + tools: + description: &assistant_tools_param_description | + A list of tool enabled on the assistant. There can be a maximum of 128 tools per assistant. Tools can be of types `code_interpreter`, `file_search`, or `function`. + default: [] + type: array + maxItems: 128 + items: + oneOf: + - $ref: "#/components/schemas/AssistantToolsCode" + - $ref: "#/components/schemas/AssistantToolsFileSearch" + - $ref: "#/components/schemas/AssistantToolsFunction" + x-oaiExpandable: true + tool_resources: + type: object + description: | + A set of resources that are used by the assistant's tools. The resources are specific to the type of tool. For example, the `code_interpreter` tool requires a list of file IDs, while the `file_search` tool requires a list of vector store IDs. + properties: + code_interpreter: + type: object + properties: + file_ids: + type: array + description: | + A list of [file](/docs/api-reference/files) IDs made available to the `code_interpreter`` tool. There can be a maximum of 20 files associated with the tool. + default: [] + maxItems: 20 + items: + type: string + file_search: + type: object + properties: + vector_store_ids: + type: array + description: | + The ID of the [vector store](/docs/api-reference/vector-stores/object) attached to this assistant. There can be a maximum of 1 vector store attached to the assistant. + maxItems: 1 + items: + type: string + nullable: true + metadata: + description: &metadata_description | + Set of 16 key-value pairs that can be attached to an object. This can be useful for storing additional information about the object in a structured format. Keys can be a maximum of 64 characters long and values can be a maxium of 512 characters long. + type: object + x-oaiTypeLabel: map + nullable: true + temperature: + description: &run_temperature_description | + What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. + type: number + minimum: 0 + maximum: 2 + default: 1 + example: 1 + nullable: true + top_p: + type: number + minimum: 0 + maximum: 1 + default: 1 + example: 1 + nullable: true + description: &run_top_p_description | + An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. + + We generally recommend altering this or temperature but not both. + response_format: + $ref: "#/components/schemas/AssistantsApiResponseFormatOption" + nullable: true + required: + - id + - object + - created_at + - name + - description + - model + - instructions + - tools + - metadata + x-oaiMeta: + name: The assistant object + beta: true + example: *create_assistants_example + + CreateAssistantRequest: + type: object + additionalProperties: false + properties: + model: + description: *model_description + example: "gpt-4-turbo" + anyOf: + - type: string + - type: string + enum: + [ + "gpt-4-turbo", + "gpt-4-turbo-2024-04-09", + "gpt-4-0125-preview", + "gpt-4-turbo-preview", + "gpt-4-1106-preview", + "gpt-4-vision-preview", + "gpt-4", + "gpt-4-0314", + "gpt-4-0613", + "gpt-4-32k", + "gpt-4-32k-0314", + "gpt-4-32k-0613", + "gpt-3.5-turbo", + "gpt-3.5-turbo-16k", + "gpt-3.5-turbo-0613", + "gpt-3.5-turbo-1106", + "gpt-3.5-turbo-0125", + "gpt-3.5-turbo-16k-0613", + ] + x-oaiTypeLabel: string + name: + description: *assistant_name_param_description + type: string + nullable: true + maxLength: 256 + description: + description: *assistant_description_param_description + type: string + nullable: true + maxLength: 512 + instructions: + description: *assistant_instructions_param_description + type: string + nullable: true + maxLength: 256000 + tools: + description: *assistant_tools_param_description + default: [] + type: array + maxItems: 128 + items: + oneOf: + - $ref: "#/components/schemas/AssistantToolsCode" + - $ref: "#/components/schemas/AssistantToolsFileSearch" + - $ref: "#/components/schemas/AssistantToolsFunction" + x-oaiExpandable: true + tool_resources: + type: object + description: | + A set of resources that are used by the assistant's tools. The resources are specific to the type of tool. For example, the `code_interpreter` tool requires a list of file IDs, while the `file_search` tool requires a list of vector store IDs. + properties: + code_interpreter: + type: object + properties: + file_ids: + type: array + description: | + A list of [file](/docs/api-reference/files) IDs made available to the `code_interpreter` tool. There can be a maximum of 20 files associated with the tool. + default: [] + maxItems: 20 + items: + type: string + file_search: + type: object + properties: + vector_store_ids: + type: array + description: | + The [vector store](/docs/api-reference/vector-stores/object) attached to this assistant. There can be a maximum of 1 vector store attached to the assistant. + maxItems: 1 + items: + type: string + vector_stores: + type: array + description: | + A helper to create a [vector store](/docs/api-reference/vector-stores/object) with file_ids and attach it to this assistant. There can be a maximum of 1 vector store attached to the assistant. + maxItems: 1 + items: + type: object + properties: + file_ids: + type: array + description: | + A list of [file](/docs/api-reference/files) IDs to add to the vector store. There can be a maximum of 10000 files in a vector store. + maxItems: 10000 + items: + type: string + metadata: + type: object + description: | + Set of 16 key-value pairs that can be attached to a vector store. This can be useful for storing additional information about the vector store in a structured format. Keys can be a maximum of 64 characters long and values can be a maxium of 512 characters long. + x-oaiTypeLabel: map + oneOf: + - required: [vector_store_ids] + - required: [vector_stores] + nullable: true + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + temperature: + description: &run_temperature_description | + What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. + type: number + minimum: 0 + maximum: 2 + default: 1 + example: 1 + nullable: true + top_p: + type: number + minimum: 0 + maximum: 1 + default: 1 + example: 1 + nullable: true + description: &run_top_p_description | + An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. + + We generally recommend altering this or temperature but not both. + response_format: + $ref: "#/components/schemas/AssistantsApiResponseFormatOption" + nullable: true + required: + - model + + ModifyAssistantRequest: + type: object + additionalProperties: false + properties: + model: + description: *model_description + anyOf: + - type: string + name: + description: *assistant_name_param_description + type: string + nullable: true + maxLength: 256 + description: + description: *assistant_description_param_description + type: string + nullable: true + maxLength: 512 + instructions: + description: *assistant_instructions_param_description + type: string + nullable: true + maxLength: 256000 + tools: + description: *assistant_tools_param_description + default: [] + type: array + maxItems: 128 + items: + oneOf: + - $ref: "#/components/schemas/AssistantToolsCode" + - $ref: "#/components/schemas/AssistantToolsFileSearch" + - $ref: "#/components/schemas/AssistantToolsFunction" + x-oaiExpandable: true + tool_resources: + type: object + description: | + A set of resources that are used by the assistant's tools. The resources are specific to the type of tool. For example, the `code_interpreter` tool requires a list of file IDs, while the `file_search` tool requires a list of vector store IDs. + properties: + code_interpreter: + type: object + properties: + file_ids: + type: array + description: | + Overrides the list of [file](/docs/api-reference/files) IDs made available to the `code_interpreter` tool. There can be a maximum of 20 files associated with the tool. + default: [] + maxItems: 20 + items: + type: string + file_search: + type: object + properties: + vector_store_ids: + type: array + description: | + Overrides the [vector store](/docs/api-reference/vector-stores/object) attached to this assistant. There can be a maximum of 1 vector store attached to the assistant. + maxItems: 1 + items: + type: string + nullable: true + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + temperature: + description: *run_temperature_description + type: number + minimum: 0 + maximum: 2 + default: 1 + example: 1 + nullable: true + top_p: + type: number + minimum: 0 + maximum: 1 + default: 1 + example: 1 + nullable: true + description: &run_top_p_description | + An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. + + We generally recommend altering this or temperature but not both. + response_format: + $ref: "#/components/schemas/AssistantsApiResponseFormatOption" + nullable: true + + DeleteAssistantResponse: + type: object + properties: + id: + type: string + deleted: + type: boolean + object: + type: string + enum: [assistant.deleted] + required: + - id + - object + - deleted + + ListAssistantsResponse: + type: object + properties: + object: + type: string + example: "list" + data: + type: array + items: + $ref: "#/components/schemas/AssistantObject" + first_id: + type: string + example: "asst_abc123" + last_id: + type: string + example: "asst_abc456" + has_more: + type: boolean + example: false + required: + - object + - data + - first_id + - last_id + - has_more + x-oaiMeta: + name: List assistants response object + group: chat + example: *list_assistants_example + + AssistantToolsCode: + type: object + title: Code interpreter tool + properties: + type: + type: string + description: "The type of tool being defined: `code_interpreter`" + enum: ["code_interpreter"] + required: + - type + + AssistantToolsFileSearch: + type: object + title: FileSearch tool + properties: + type: + type: string + description: "The type of tool being defined: `file_search`" + enum: ["file_search"] + required: + - type + + AssistantToolsFunction: + type: object + title: Function tool + properties: + type: + type: string + description: "The type of tool being defined: `function`" + enum: ["function"] + function: + $ref: "#/components/schemas/FunctionObject" + required: + - type + - function + + TruncationObject: + type: object + title: Thread Truncation Controls + description: Controls for how a thread will be truncated prior to the run. Use this to control the intial context window of the run. + properties: + type: + type: string + description: The truncation strategy to use for the thread. The default is `auto`. If set to `last_messages`, the thread will be truncated to the n most recent messages in the thread. When set to `auto`, messages in the middle of the thread will be dropped to fit the context length of the model, `max_prompt_tokens`. + enum: ["auto", "last_messages"] + last_messages: + type: integer + description: The number of most recent messages from the thread when constructing the context for the run. + minimum: 1 + nullable: true + required: + - type + + AssistantsApiToolChoiceOption: + description: | + Controls which (if any) tool is called by the model. + `none` means the model will not call any tools and instead generates a message. + `auto` is the default value and means the model can pick between generating a message or calling one or more tools. + `required` means the model must call one or more tools before responding to the user. + Specifying a particular tool like `{"type": "file_search"}` or `{"type": "function", "function": {"name": "my_function"}}` forces the model to call that tool. + + oneOf: + - type: string + description: > + `none` means the model will not call any tools and instead generates a message. + `auto` means the model can pick between generating a message or calling one or more tools. + `required` means the model must call one or more tools before responding to the user. + enum: [none, auto, required] + - $ref: "#/components/schemas/AssistantsNamedToolChoice" + x-oaiExpandable: true + + AssistantsNamedToolChoice: + type: object + description: Specifies a tool the model should use. Use to force the model to call a specific tool. + properties: + type: + type: string + enum: ["function", "code_interpreter", "file_search"] + description: The type of the tool. If type is `function`, the function name must be set + function: + type: object + properties: + name: + type: string + description: The name of the function to call. + required: + - name + required: + - type + + RunObject: + type: object + title: A run on a thread + description: Represents an execution run on a [thread](/docs/api-reference/threads). + properties: + id: + description: The identifier, which can be referenced in API endpoints. + type: string + object: + description: The object type, which is always `thread.run`. + type: string + enum: ["thread.run"] + created_at: + description: The Unix timestamp (in seconds) for when the run was created. + type: integer + thread_id: + description: The ID of the [thread](/docs/api-reference/threads) that was executed on as a part of this run. + type: string + assistant_id: + description: The ID of the [assistant](/docs/api-reference/assistants) used for execution of this run. + type: string + status: + description: The status of the run, which can be either `queued`, `in_progress`, `requires_action`, `cancelling`, `cancelled`, `failed`, `completed`, or `expired`. + type: string + enum: + [ + "queued", + "in_progress", + "requires_action", + "cancelling", + "cancelled", + "failed", + "completed", + "expired", + ] + required_action: + type: object + description: Details on the action required to continue the run. Will be `null` if no action is required. + nullable: true + properties: + type: + description: For now, this is always `submit_tool_outputs`. + type: string + enum: ["submit_tool_outputs"] + submit_tool_outputs: + type: object + description: Details on the tool outputs needed for this run to continue. + properties: + tool_calls: + type: array + description: A list of the relevant tool calls. + items: + $ref: "#/components/schemas/RunToolCallObject" + required: + - tool_calls + required: + - type + - submit_tool_outputs + last_error: + type: object + description: The last error associated with this run. Will be `null` if there are no errors. + nullable: true + properties: + code: + type: string + description: One of `server_error`, `rate_limit_exceeded`, or `invalid_prompt`. + enum: ["server_error", "rate_limit_exceeded", "invalid_prompt"] + message: + type: string + description: A human-readable description of the error. + required: + - code + - message + expires_at: + description: The Unix timestamp (in seconds) for when the run will expire. + type: integer + nullable: true + started_at: + description: The Unix timestamp (in seconds) for when the run was started. + type: integer + nullable: true + cancelled_at: + description: The Unix timestamp (in seconds) for when the run was cancelled. + type: integer + nullable: true + failed_at: + description: The Unix timestamp (in seconds) for when the run failed. + type: integer + nullable: true + completed_at: + description: The Unix timestamp (in seconds) for when the run was completed. + type: integer + nullable: true + incomplete_details: + description: Details on why the run is incomplete. Will be `null` if the run is not incomplete. + type: object + nullable: true + properties: + reason: + description: The reason why the run is incomplete. This will point to which specific token limit was reached over the course of the run. + type: string + enum: ["max_completion_tokens", "max_prompt_tokens"] + model: + description: The model that the [assistant](/docs/api-reference/assistants) used for this run. + type: string + instructions: + description: The instructions that the [assistant](/docs/api-reference/assistants) used for this run. + type: string + tools: + description: The list of tools that the [assistant](/docs/api-reference/assistants) used for this run. + default: [] + type: array + maxItems: 20 + items: + oneOf: + - $ref: "#/components/schemas/AssistantToolsCode" + - $ref: "#/components/schemas/AssistantToolsFileSearch" + - $ref: "#/components/schemas/AssistantToolsFunction" + x-oaiExpandable: true + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + usage: + $ref: "#/components/schemas/RunCompletionUsage" + temperature: + description: The sampling temperature used for this run. If not set, defaults to 1. + type: number + nullable: true + top_p: + description: The nucleus sampling value used for this run. If not set, defaults to 1. + type: number + nullable: true + max_prompt_tokens: + type: integer + nullable: true + description: | + The maximum number of prompt tokens specified to have been used over the course of the run. + minimum: 256 + max_completion_tokens: + type: integer + nullable: true + description: | + The maximum number of completion tokens specified to have been used over the course of the run. + minimum: 256 + truncation_strategy: + $ref: "#/components/schemas/TruncationObject" + nullable: true + tool_choice: + $ref: "#/components/schemas/AssistantsApiToolChoiceOption" + nullable: true + response_format: + $ref: "#/components/schemas/AssistantsApiResponseFormatOption" + nullable: true + required: + - id + - object + - created_at + - thread_id + - assistant_id + - status + - required_action + - last_error + - expires_at + - started_at + - cancelled_at + - failed_at + - completed_at + - model + - instructions + - tools + - metadata + - usage + - incomplete_details + - max_prompt_tokens + - max_completion_tokens + - truncation_strategy + - tool_choice + - response_format + x-oaiMeta: + name: The run object + beta: true + example: | + { + "id": "run_abc123", + "object": "thread.run", + "created_at": 1698107661, + "assistant_id": "asst_abc123", + "thread_id": "thread_abc123", + "status": "completed", + "started_at": 1699073476, + "expires_at": null, + "cancelled_at": null, + "failed_at": null, + "completed_at": 1699073498, + "last_error": null, + "model": "gpt-4-turbo", + "instructions": null, + "tools": [{"type": "file_search"}, {"type": "code_interpreter"}], + "metadata": {}, + "incomplete_details": null, + "usage": { + "prompt_tokens": 123, + "completion_tokens": 456, + "total_tokens": 579 + }, + "temperature": 1.0, + "top_p": 1.0, + "max_prompt_tokens": 1000, + "max_completion_tokens": 1000, + "truncation_strategy": { + "type": "auto", + "last_messages": null + }, + "response_format": "auto", + "tool_choice": "auto" + } + CreateRunRequest: + type: object + additionalProperties: false + properties: + assistant_id: + description: The ID of the [assistant](/docs/api-reference/assistants) to use to execute this run. + type: string + model: + description: The ID of the [Model](/docs/api-reference/models) to be used to execute this run. If a value is provided here, it will override the model associated with the assistant. If not, the model associated with the assistant will be used. + example: "gpt-4-turbo" + anyOf: + - type: string + - type: string + enum: + [ + "gpt-4-turbo", + "gpt-4-turbo-2024-04-09", + "gpt-4-0125-preview", + "gpt-4-turbo-preview", + "gpt-4-1106-preview", + "gpt-4-vision-preview", + "gpt-4", + "gpt-4-0314", + "gpt-4-0613", + "gpt-4-32k", + "gpt-4-32k-0314", + "gpt-4-32k-0613", + "gpt-3.5-turbo", + "gpt-3.5-turbo-16k", + "gpt-3.5-turbo-0613", + "gpt-3.5-turbo-1106", + "gpt-3.5-turbo-0125", + "gpt-3.5-turbo-16k-0613", + ] + x-oaiTypeLabel: string + nullable: true + instructions: + description: Overrides the [instructions](/docs/api-reference/assistants/createAssistant) of the assistant. This is useful for modifying the behavior on a per-run basis. + type: string + nullable: true + additional_instructions: + description: Appends additional instructions at the end of the instructions for the run. This is useful for modifying the behavior on a per-run basis without overriding other instructions. + type: string + nullable: true + additional_messages: + description: Adds additional messages to the thread before creating the run. + type: array + items: + $ref: "#/components/schemas/CreateMessageRequest" + nullable: true + tools: + description: Override the tools the assistant can use for this run. This is useful for modifying the behavior on a per-run basis. + nullable: true + type: array + maxItems: 20 + items: + oneOf: + - $ref: "#/components/schemas/AssistantToolsCode" + - $ref: "#/components/schemas/AssistantToolsFileSearch" + - $ref: "#/components/schemas/AssistantToolsFunction" + x-oaiExpandable: true + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + temperature: + type: number + minimum: 0 + maximum: 2 + default: 1 + example: 1 + nullable: true + description: *run_temperature_description + top_p: + type: number + minimum: 0 + maximum: 1 + default: 1 + example: 1 + nullable: true + description: &run_top_p_description | + An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. + + We generally recommend altering this or temperature but not both. + stream: + type: boolean + nullable: true + description: | + If `true`, returns a stream of events that happen during the Run as server-sent events, terminating when the Run enters a terminal state with a `data: [DONE]` message. + max_prompt_tokens: + type: integer + nullable: true + description: | + The maximum number of prompt tokens that may be used over the course of the run. The run will make a best effort to use only the number of prompt tokens specified, across multiple turns of the run. If the run exceeds the number of prompt tokens specified, the run will end with status `incomplete`. See `incomplete_details` for more info. + minimum: 256 + max_completion_tokens: + type: integer + nullable: true + description: | + The maximum number of completion tokens that may be used over the course of the run. The run will make a best effort to use only the number of completion tokens specified, across multiple turns of the run. If the run exceeds the number of completion tokens specified, the run will end with status `incomplete`. See `incomplete_details` for more info. + minimum: 256 + truncation_strategy: + $ref: "#/components/schemas/TruncationObject" + nullable: true + tool_choice: + $ref: "#/components/schemas/AssistantsApiToolChoiceOption" + nullable: true + response_format: + $ref: "#/components/schemas/AssistantsApiResponseFormatOption" + nullable: true + required: + - thread_id + - assistant_id + ListRunsResponse: + type: object + properties: + object: + type: string + example: "list" + data: + type: array + items: + $ref: "#/components/schemas/RunObject" + first_id: + type: string + example: "run_abc123" + last_id: + type: string + example: "run_abc456" + has_more: + type: boolean + example: false + required: + - object + - data + - first_id + - last_id + - has_more + ModifyRunRequest: + type: object + additionalProperties: false + properties: + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + SubmitToolOutputsRunRequest: + type: object + additionalProperties: false + properties: + tool_outputs: + description: A list of tools for which the outputs are being submitted. + type: array + items: + type: object + properties: + tool_call_id: + type: string + description: The ID of the tool call in the `required_action` object within the run object the output is being submitted for. + output: + type: string + description: The output of the tool call to be submitted to continue the run. + stream: + type: boolean + nullable: true + description: | + If `true`, returns a stream of events that happen during the Run as server-sent events, terminating when the Run enters a terminal state with a `data: [DONE]` message. + required: + - tool_outputs + + RunToolCallObject: + type: object + description: Tool call objects + properties: + id: + type: string + description: The ID of the tool call. This ID must be referenced when you submit the tool outputs in using the [Submit tool outputs to run](/docs/api-reference/runs/submitToolOutputs) endpoint. + type: + type: string + description: The type of tool call the output is required for. For now, this is always `function`. + enum: ["function"] + function: + type: object + description: The function definition. + properties: + name: + type: string + description: The name of the function. + arguments: + type: string + description: The arguments that the model expects you to pass to the function. + required: + - name + - arguments + required: + - id + - type + - function + + CreateThreadAndRunRequest: + type: object + additionalProperties: false + properties: + assistant_id: + description: The ID of the [assistant](/docs/api-reference/assistants) to use to execute this run. + type: string + thread: + $ref: "#/components/schemas/CreateThreadRequest" + description: If no thread is provided, an empty thread will be created. + model: + description: The ID of the [Model](/docs/api-reference/models) to be used to execute this run. If a value is provided here, it will override the model associated with the assistant. If not, the model associated with the assistant will be used. + example: "gpt-4-turbo" + anyOf: + - type: string + - type: string + enum: + [ + "gpt-4-turbo", + "gpt-4-turbo-2024-04-09", + "gpt-4-0125-preview", + "gpt-4-turbo-preview", + "gpt-4-1106-preview", + "gpt-4-vision-preview", + "gpt-4", + "gpt-4-0314", + "gpt-4-0613", + "gpt-4-32k", + "gpt-4-32k-0314", + "gpt-4-32k-0613", + "gpt-3.5-turbo", + "gpt-3.5-turbo-16k", + "gpt-3.5-turbo-0613", + "gpt-3.5-turbo-1106", + "gpt-3.5-turbo-0125", + "gpt-3.5-turbo-16k-0613", + ] + x-oaiTypeLabel: string + nullable: true + instructions: + description: Override the default system message of the assistant. This is useful for modifying the behavior on a per-run basis. + type: string + nullable: true + tools: + description: Override the tools the assistant can use for this run. This is useful for modifying the behavior on a per-run basis. + nullable: true + type: array + maxItems: 20 + items: + oneOf: + - $ref: "#/components/schemas/AssistantToolsCode" + - $ref: "#/components/schemas/AssistantToolsFileSearch" + - $ref: "#/components/schemas/AssistantToolsFunction" + tool_resources: + type: object + description: | + A set of resources that are used by the assistant's tools. The resources are specific to the type of tool. For example, the `code_interpreter` tool requires a list of file IDs, while the `file_search` tool requires a list of vector store IDs. + properties: + code_interpreter: + type: object + properties: + file_ids: + type: array + description: | + A list of [file](/docs/api-reference/files) IDs made available to the `code_interpreter` tool. There can be a maximum of 20 files associated with the tool. + default: [] + maxItems: 20 + items: + type: string + file_search: + type: object + properties: + vector_store_ids: + type: array + description: | + The ID of the [vector store](/docs/api-reference/vector-stores/object) attached to this assistant. There can be a maximum of 1 vector store attached to the assistant. + maxItems: 1 + items: + type: string + nullable: true + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + temperature: + type: number + minimum: 0 + maximum: 2 + default: 1 + example: 1 + nullable: true + description: *run_temperature_description + top_p: + type: number + minimum: 0 + maximum: 1 + default: 1 + example: 1 + nullable: true + description: *run_top_p_description + stream: + type: boolean + nullable: true + description: | + If `true`, returns a stream of events that happen during the Run as server-sent events, terminating when the Run enters a terminal state with a `data: [DONE]` message. + max_prompt_tokens: + type: integer + nullable: true + description: | + The maximum number of prompt tokens that may be used over the course of the run. The run will make a best effort to use only the number of prompt tokens specified, across multiple turns of the run. If the run exceeds the number of prompt tokens specified, the run will end with status `incomplete`. See `incomplete_details` for more info. + minimum: 256 + max_completion_tokens: + type: integer + nullable: true + description: | + The maximum number of completion tokens that may be used over the course of the run. The run will make a best effort to use only the number of completion tokens specified, across multiple turns of the run. If the run exceeds the number of completion tokens specified, the run will end with status `incomplete`. See `incomplete_details` for more info. + minimum: 256 + truncation_strategy: + $ref: "#/components/schemas/TruncationObject" + nullable: true + tool_choice: + $ref: "#/components/schemas/AssistantsApiToolChoiceOption" + nullable: true + response_format: + $ref: "#/components/schemas/AssistantsApiResponseFormatOption" + nullable: true + required: + - thread_id + - assistant_id + + ThreadObject: + type: object + title: Thread + description: Represents a thread that contains [messages](/docs/api-reference/messages). + properties: + id: + description: The identifier, which can be referenced in API endpoints. + type: string + object: + description: The object type, which is always `thread`. + type: string + enum: ["thread"] + created_at: + description: The Unix timestamp (in seconds) for when the thread was created. + type: integer + tool_resources: + type: object + description: | + A set of resources that are made available to the assistant's tools in this thread. The resources are specific to the type of tool. For example, the `code_interpreter` tool requires a list of file IDs, while the `file_search` tool requires a list of vector store IDs. + properties: + code_interpreter: + type: object + properties: + file_ids: + type: array + description: | + A list of [file](/docs/api-reference/files) IDs made available to the `code_interpreter` tool. There can be a maximum of 20 files associated with the tool. + default: [] + maxItems: 20 + items: + type: string + file_search: + type: object + properties: + vector_store_ids: + type: array + description: | + The [vector store](/docs/api-reference/vector-stores/object) attached to this thread. There can be a maximum of 1 vector store attached to the thread. + maxItems: 1 + items: + type: string + nullable: true + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + required: + - id + - object + - created_at + - tool_resources + - metadata + x-oaiMeta: + name: The thread object + beta: true + example: | + { + "id": "thread_abc123", + "object": "thread", + "created_at": 1698107661, + "metadata": {} + } + + CreateThreadRequest: + type: object + additionalProperties: false + properties: + messages: + description: A list of [messages](/docs/api-reference/messages) to start the thread with. + type: array + items: + $ref: "#/components/schemas/CreateMessageRequest" + tool_resources: + type: object + description: | + A set of resources that are made available to the assistant's tools in this thread. The resources are specific to the type of tool. For example, the `code_interpreter` tool requires a list of file IDs, while the `file_search` tool requires a list of vector store IDs. + properties: + code_interpreter: + type: object + properties: + file_ids: + type: array + description: | + A list of [file](/docs/api-reference/files) IDs made available to the `code_interpreter` tool. There can be a maximum of 20 files associated with the tool. + default: [] + maxItems: 20 + items: + type: string + file_search: + type: object + properties: + vector_store_ids: + type: array + description: | + The [vector store](/docs/api-reference/vector-stores/object) attached to this thread. There can be a maximum of 1 vector store attached to the thread. + maxItems: 1 + items: + type: string + vector_stores: + type: array + description: | + A helper to create a [vector store](/docs/api-reference/vector-stores/object) with file_ids and attach it to this thread. There can be a maximum of 1 vector store attached to the thread. + maxItems: 1 + items: + type: object + properties: + file_ids: + type: array + description: | + A list of [file](/docs/api-reference/files) IDs to add to the vector store. There can be a maximum of 10000 files in a vector store. + maxItems: 10000 + items: + type: string + metadata: + type: object + description: | + Set of 16 key-value pairs that can be attached to a vector store. This can be useful for storing additional information about the vector store in a structured format. Keys can be a maximum of 64 characters long and values can be a maxium of 512 characters long. + x-oaiTypeLabel: map + oneOf: + - required: [vector_store_ids] + - required: [vector_stores] + nullable: true + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + + ModifyThreadRequest: + type: object + additionalProperties: false + properties: + tool_resources: + type: object + description: | + A set of resources that are made available to the assistant's tools in this thread. The resources are specific to the type of tool. For example, the `code_interpreter` tool requires a list of file IDs, while the `file_search` tool requires a list of vector store IDs. + properties: + code_interpreter: + type: object + properties: + file_ids: + type: array + description: | + A list of [file](/docs/api-reference/files) IDs made available to the `code_interpreter` tool. There can be a maximum of 20 files associated with the tool. + default: [] + maxItems: 20 + items: + type: string + file_search: + type: object + properties: + vector_store_ids: + type: array + description: | + The [vector store](/docs/api-reference/vector-stores/object) attached to this thread. There can be a maximum of 1 vector store attached to the thread. + maxItems: 1 + items: + type: string + nullable: true + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + + DeleteThreadResponse: + type: object + properties: + id: + type: string + deleted: + type: boolean + object: + type: string + enum: [thread.deleted] + required: + - id + - object + - deleted + + ListThreadsResponse: + properties: + object: + type: string + example: "list" + data: + type: array + items: + $ref: "#/components/schemas/ThreadObject" + first_id: + type: string + example: "asst_abc123" + last_id: + type: string + example: "asst_abc456" + has_more: + type: boolean + example: false + required: + - object + - data + - first_id + - last_id + - has_more + + MessageObject: + type: object + title: The message object + description: Represents a message within a [thread](/docs/api-reference/threads). + properties: + id: + description: The identifier, which can be referenced in API endpoints. + type: string + object: + description: The object type, which is always `thread.message`. + type: string + enum: ["thread.message"] + created_at: + description: The Unix timestamp (in seconds) for when the message was created. + type: integer + thread_id: + description: The [thread](/docs/api-reference/threads) ID that this message belongs to. + type: string + status: + description: The status of the message, which can be either `in_progress`, `incomplete`, or `completed`. + type: string + enum: ["in_progress", "incomplete", "completed"] + incomplete_details: + description: On an incomplete message, details about why the message is incomplete. + type: object + properties: + reason: + type: string + description: The reason the message is incomplete. + enum: + [ + "content_filter", + "max_tokens", + "run_cancelled", + "run_expired", + "run_failed", + ] + nullable: true + required: + - reason + completed_at: + description: The Unix timestamp (in seconds) for when the message was completed. + type: integer + nullable: true + incomplete_at: + description: The Unix timestamp (in seconds) for when the message was marked as incomplete. + type: integer + nullable: true + role: + description: The entity that produced the message. One of `user` or `assistant`. + type: string + enum: ["user", "assistant"] + content: + description: The content of the message in array of text and/or images. + type: array + items: + oneOf: + - $ref: "#/components/schemas/MessageContentImageFileObject" + - $ref: "#/components/schemas/MessageContentTextObject" + x-oaiExpandable: true + assistant_id: + description: If applicable, the ID of the [assistant](/docs/api-reference/assistants) that authored this message. + type: string + nullable: true + run_id: + description: The ID of the [run](/docs/api-reference/runs) associated with the creation of this message. Value is `null` when messages are created manually using the create message or create thread endpoints. + type: string + nullable: true + attachments: + type: array + items: + type: object + properties: + file_id: + type: string + description: The ID of the file to attach to the message. + tools: + description: The tools to add this file to. + type: array + items: + oneOf: + - $ref: "#/components/schemas/AssistantToolsCode" + - $ref: "#/components/schemas/AssistantToolsFileSearch" + x-oaiExpandable: true + description: A list of files attached to the message, and the tools they were added to. + nullable: true + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + required: + - id + - object + - created_at + - thread_id + - status + - incomplete_details + - completed_at + - incomplete_at + - role + - content + - assistant_id + - run_id + - attachments + - metadata + x-oaiMeta: + name: The message object + beta: true + example: | + { + "id": "msg_abc123", + "object": "thread.message", + "created_at": 1698983503, + "thread_id": "thread_abc123", + "role": "assistant", + "content": [ + { + "type": "text", + "text": { + "value": "Hi! How can I help you today?", + "annotations": [] + } + } + ], + "assistant_id": "asst_abc123", + "run_id": "run_abc123", + "attachments": [], + "metadata": {} + } + + MessageDeltaObject: + type: object + title: Message delta object + description: | + Represents a message delta i.e. any changed fields on a message during streaming. + properties: + id: + description: The identifier of the message, which can be referenced in API endpoints. + type: string + object: + description: The object type, which is always `thread.message.delta`. + type: string + enum: ["thread.message.delta"] + delta: + description: The delta containing the fields that have changed on the Message. + type: object + properties: + role: + description: The entity that produced the message. One of `user` or `assistant`. + type: string + enum: ["user", "assistant"] + content: + description: The content of the message in array of text and/or images. + type: array + items: + oneOf: + - $ref: "#/components/schemas/MessageDeltaContentImageFileObject" + - $ref: "#/components/schemas/MessageDeltaContentTextObject" + x-oaiExpandable: true + required: + - id + - object + - delta + x-oaiMeta: + name: The message delta object + beta: true + example: | + { + "id": "msg_123", + "object": "thread.message.delta", + "delta": { + "content": [ + { + "index": 0, + "type": "text", + "text": { "value": "Hello", "annotations": [] } + } + ] + } + } + + CreateMessageRequest: + type: object + additionalProperties: false + required: + - role + - content + properties: + role: + type: string + enum: ["user", "assistant"] + description: | + The role of the entity that is creating the message. Allowed values include: + - `user`: Indicates the message is sent by an actual user and should be used in most cases to represent user-generated messages. + - `assistant`: Indicates the message is generated by the assistant. Use this value to insert messages from the assistant into the conversation. + content: + type: string + minLength: 1 + maxLength: 256000 + description: The content of the message. + attachments: + type: array + items: + type: object + properties: + file_id: + type: string + description: The ID of the file to attach to the message. + tools: + description: The tools to add this file to. + type: array + items: + oneOf: + - $ref: "#/components/schemas/AssistantToolsCode" + - $ref: "#/components/schemas/AssistantToolsFileSearch" + x-oaiExpandable: true + description: A list of files attached to the message, and the tools they should be added to. + required: + - file_id + - tools + nullable: true + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + + ModifyMessageRequest: + type: object + additionalProperties: false + properties: + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + + DeleteMessageResponse: + type: object + properties: + id: + type: string + deleted: + type: boolean + object: + type: string + enum: [thread.message.deleted] + required: + - id + - object + - deleted + + ListMessagesResponse: + properties: + object: + type: string + example: "list" + data: + type: array + items: + $ref: "#/components/schemas/MessageObject" + first_id: + type: string + example: "msg_abc123" + last_id: + type: string + example: "msg_abc123" + has_more: + type: boolean + example: false + required: + - object + - data + - first_id + - last_id + - has_more + + MessageContentImageFileObject: + title: Image file + type: object + description: References an image [File](/docs/api-reference/files) in the content of a message. + properties: + type: + description: Always `image_file`. + type: string + enum: ["image_file"] + image_file: + type: object + properties: + file_id: + description: The [File](/docs/api-reference/files) ID of the image in the message content. + type: string + required: + - file_id + required: + - type + - image_file + + MessageDeltaContentImageFileObject: + title: Image file + type: object + description: References an image [File](/docs/api-reference/files) in the content of a message. + properties: + index: + type: integer + description: The index of the content part in the message. + type: + description: Always `image_file`. + type: string + enum: ["image_file"] + image_file: + type: object + properties: + file_id: + description: The [File](/docs/api-reference/files) ID of the image in the message content. + type: string + required: + - index + - type + + MessageContentTextObject: + title: Text + type: object + description: The text content that is part of a message. + properties: + type: + description: Always `text`. + type: string + enum: ["text"] + text: + type: object + properties: + value: + description: The data that makes up the text. + type: string + annotations: + type: array + items: + oneOf: + - $ref: "#/components/schemas/MessageContentTextAnnotationsFileCitationObject" + - $ref: "#/components/schemas/MessageContentTextAnnotationsFilePathObject" + x-oaiExpandable: true + required: + - value + - annotations + required: + - type + - text + + MessageContentTextAnnotationsFileCitationObject: + title: File citation + type: object + description: A citation within the message that points to a specific quote from a specific File associated with the assistant or the message. Generated when the assistant uses the "file_search" tool to search files. + properties: + type: + description: Always `file_citation`. + type: string + enum: ["file_citation"] + text: + description: The text in the message content that needs to be replaced. + type: string + file_citation: + type: object + properties: + file_id: + description: The ID of the specific File the citation is from. + type: string + quote: + description: The specific quote in the file. + type: string + required: + - file_id + - quote + start_index: + type: integer + minimum: 0 + end_index: + type: integer + minimum: 0 + required: + - type + - text + - file_citation + - start_index + - end_index + + MessageContentTextAnnotationsFilePathObject: + title: File path + type: object + description: A URL for the file that's generated when the assistant used the `code_interpreter` tool to generate a file. + properties: + type: + description: Always `file_path`. + type: string + enum: ["file_path"] + text: + description: The text in the message content that needs to be replaced. + type: string + file_path: + type: object + properties: + file_id: + description: The ID of the file that was generated. + type: string + required: + - file_id + start_index: + type: integer + minimum: 0 + end_index: + type: integer + minimum: 0 + required: + - type + - text + - file_path + - start_index + - end_index + + MessageDeltaContentTextObject: + title: Text + type: object + description: The text content that is part of a message. + properties: + index: + type: integer + description: The index of the content part in the message. + type: + description: Always `text`. + type: string + enum: ["text"] + text: + type: object + properties: + value: + description: The data that makes up the text. + type: string + annotations: + type: array + items: + oneOf: + - $ref: "#/components/schemas/MessageDeltaContentTextAnnotationsFileCitationObject" + - $ref: "#/components/schemas/MessageDeltaContentTextAnnotationsFilePathObject" + x-oaiExpandable: true + required: + - index + - type + + MessageDeltaContentTextAnnotationsFileCitationObject: + title: File citation + type: object + description: A citation within the message that points to a specific quote from a specific File associated with the assistant or the message. Generated when the assistant uses the "file_search" tool to search files. + properties: + index: + type: integer + description: The index of the annotation in the text content part. + type: + description: Always `file_citation`. + type: string + enum: ["file_citation"] + text: + description: The text in the message content that needs to be replaced. + type: string + file_citation: + type: object + properties: + file_id: + description: The ID of the specific File the citation is from. + type: string + quote: + description: The specific quote in the file. + type: string + start_index: + type: integer + minimum: 0 + end_index: + type: integer + minimum: 0 + required: + - index + - type + + MessageDeltaContentTextAnnotationsFilePathObject: + title: File path + type: object + description: A URL for the file that's generated when the assistant used the `code_interpreter` tool to generate a file. + properties: + index: + type: integer + description: The index of the annotation in the text content part. + type: + description: Always `file_path`. + type: string + enum: ["file_path"] + text: + description: The text in the message content that needs to be replaced. + type: string + file_path: + type: object + properties: + file_id: + description: The ID of the file that was generated. + type: string + start_index: + type: integer + minimum: 0 + end_index: + type: integer + minimum: 0 + required: + - index + - type + + RunStepObject: + type: object + title: Run steps + description: | + Represents a step in execution of a run. + properties: + id: + description: The identifier of the run step, which can be referenced in API endpoints. + type: string + object: + description: The object type, which is always `thread.run.step`. + type: string + enum: ["thread.run.step"] + created_at: + description: The Unix timestamp (in seconds) for when the run step was created. + type: integer + assistant_id: + description: The ID of the [assistant](/docs/api-reference/assistants) associated with the run step. + type: string + thread_id: + description: The ID of the [thread](/docs/api-reference/threads) that was run. + type: string + run_id: + description: The ID of the [run](/docs/api-reference/runs) that this run step is a part of. + type: string + type: + description: The type of run step, which can be either `message_creation` or `tool_calls`. + type: string + enum: ["message_creation", "tool_calls"] + status: + description: The status of the run step, which can be either `in_progress`, `cancelled`, `failed`, `completed`, or `expired`. + type: string + enum: ["in_progress", "cancelled", "failed", "completed", "expired"] + step_details: + type: object + description: The details of the run step. + oneOf: + - $ref: "#/components/schemas/RunStepDetailsMessageCreationObject" + - $ref: "#/components/schemas/RunStepDetailsToolCallsObject" + x-oaiExpandable: true + last_error: + type: object + description: The last error associated with this run step. Will be `null` if there are no errors. + nullable: true + properties: + code: + type: string + description: One of `server_error` or `rate_limit_exceeded`. + enum: ["server_error", "rate_limit_exceeded"] + message: + type: string + description: A human-readable description of the error. + required: + - code + - message + expired_at: + description: The Unix timestamp (in seconds) for when the run step expired. A step is considered expired if the parent run is expired. + type: integer + nullable: true + cancelled_at: + description: The Unix timestamp (in seconds) for when the run step was cancelled. + type: integer + nullable: true + failed_at: + description: The Unix timestamp (in seconds) for when the run step failed. + type: integer + nullable: true + completed_at: + description: The Unix timestamp (in seconds) for when the run step completed. + type: integer + nullable: true + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + usage: + $ref: "#/components/schemas/RunStepCompletionUsage" + required: + - id + - object + - created_at + - assistant_id + - thread_id + - run_id + - type + - status + - step_details + - last_error + - expired_at + - cancelled_at + - failed_at + - completed_at + - metadata + - usage + x-oaiMeta: + name: The run step object + beta: true + example: *run_step_object_example + + RunStepDeltaObject: + type: object + title: Run step delta object + description: | + Represents a run step delta i.e. any changed fields on a run step during streaming. + properties: + id: + description: The identifier of the run step, which can be referenced in API endpoints. + type: string + object: + description: The object type, which is always `thread.run.step.delta`. + type: string + enum: ["thread.run.step.delta"] + delta: + description: The delta containing the fields that have changed on the run step. + type: object + properties: + step_details: + type: object + description: The details of the run step. + oneOf: + - $ref: "#/components/schemas/RunStepDeltaStepDetailsMessageCreationObject" + - $ref: "#/components/schemas/RunStepDeltaStepDetailsToolCallsObject" + x-oaiExpandable: true + required: + - id + - object + - delta + x-oaiMeta: + name: The run step delta object + beta: true + example: | + { + "id": "step_123", + "object": "thread.run.step.delta", + "delta": { + "step_details": { + "type": "tool_calls", + "tool_calls": [ + { + "index": 0, + "id": "call_123", + "type": "code_interpreter", + "code_interpreter": { "input": "", "outputs": [] } + } + ] + } + } + } + + ListRunStepsResponse: + properties: + object: + type: string + example: "list" + data: + type: array + items: + $ref: "#/components/schemas/RunStepObject" + first_id: + type: string + example: "step_abc123" + last_id: + type: string + example: "step_abc456" + has_more: + type: boolean + example: false + required: + - object + - data + - first_id + - last_id + - has_more + + RunStepDetailsMessageCreationObject: + title: Message creation + type: object + description: Details of the message creation by the run step. + properties: + type: + description: Always `message_creation`. + type: string + enum: ["message_creation"] + message_creation: + type: object + properties: + message_id: + type: string + description: The ID of the message that was created by this run step. + required: + - message_id + required: + - type + - message_creation + + RunStepDeltaStepDetailsMessageCreationObject: + title: Message creation + type: object + description: Details of the message creation by the run step. + properties: + type: + description: Always `message_creation`. + type: string + enum: ["message_creation"] + message_creation: + type: object + properties: + message_id: + type: string + description: The ID of the message that was created by this run step. + required: + - type + + RunStepDetailsToolCallsObject: + title: Tool calls + type: object + description: Details of the tool call. + properties: + type: + description: Always `tool_calls`. + type: string + enum: ["tool_calls"] + tool_calls: + type: array + description: | + An array of tool calls the run step was involved in. These can be associated with one of three types of tools: `code_interpreter`, `file_search`, or `function`. + items: + oneOf: + - $ref: "#/components/schemas/RunStepDetailsToolCallsCodeObject" + - $ref: "#/components/schemas/RunStepDetailsToolCallsFileSearchObject" + - $ref: "#/components/schemas/RunStepDetailsToolCallsFunctionObject" + x-oaiExpandable: true + required: + - type + - tool_calls + + RunStepDeltaStepDetailsToolCallsObject: + title: Tool calls + type: object + description: Details of the tool call. + properties: + type: + description: Always `tool_calls`. + type: string + enum: ["tool_calls"] + tool_calls: + type: array + description: | + An array of tool calls the run step was involved in. These can be associated with one of three types of tools: `code_interpreter`, `file_search`, or `function`. + items: + oneOf: + - $ref: "#/components/schemas/RunStepDeltaStepDetailsToolCallsCodeObject" + - $ref: "#/components/schemas/RunStepDeltaStepDetailsToolCallsFileSearchObject" + - $ref: "#/components/schemas/RunStepDeltaStepDetailsToolCallsFunctionObject" + x-oaiExpandable: true + required: + - type + + RunStepDetailsToolCallsCodeObject: + title: Code Interpreter tool call + type: object + description: Details of the Code Interpreter tool call the run step was involved in. + properties: + id: + type: string + description: The ID of the tool call. + type: + type: string + description: The type of tool call. This is always going to be `code_interpreter` for this type of tool call. + enum: ["code_interpreter"] + code_interpreter: + type: object + description: The Code Interpreter tool call definition. + required: + - input + - outputs + properties: + input: + type: string + description: The input to the Code Interpreter tool call. + outputs: + type: array + description: The outputs from the Code Interpreter tool call. Code Interpreter can output one or more items, including text (`logs`) or images (`image`). Each of these are represented by a different object type. + items: + type: object + oneOf: + - $ref: "#/components/schemas/RunStepDetailsToolCallsCodeOutputLogsObject" + - $ref: "#/components/schemas/RunStepDetailsToolCallsCodeOutputImageObject" + x-oaiExpandable: true + required: + - id + - type + - code_interpreter + + RunStepDeltaStepDetailsToolCallsCodeObject: + title: Code interpreter tool call + type: object + description: Details of the Code Interpreter tool call the run step was involved in. + properties: + index: + type: integer + description: The index of the tool call in the tool calls array. + id: + type: string + description: The ID of the tool call. + type: + type: string + description: The type of tool call. This is always going to be `code_interpreter` for this type of tool call. + enum: ["code_interpreter"] + code_interpreter: + type: object + description: The Code Interpreter tool call definition. + properties: + input: + type: string + description: The input to the Code Interpreter tool call. + outputs: + type: array + description: The outputs from the Code Interpreter tool call. Code Interpreter can output one or more items, including text (`logs`) or images (`image`). Each of these are represented by a different object type. + items: + type: object + oneOf: + - $ref: "#/components/schemas/RunStepDeltaStepDetailsToolCallsCodeOutputLogsObject" + - $ref: "#/components/schemas/RunStepDeltaStepDetailsToolCallsCodeOutputImageObject" + x-oaiExpandable: true + required: + - index + - type + + RunStepDetailsToolCallsCodeOutputLogsObject: + title: Code Interpreter log output + type: object + description: Text output from the Code Interpreter tool call as part of a run step. + properties: + type: + description: Always `logs`. + type: string + enum: ["logs"] + logs: + type: string + description: The text output from the Code Interpreter tool call. + required: + - type + - logs + + RunStepDeltaStepDetailsToolCallsCodeOutputLogsObject: + title: Code interpreter log output + type: object + description: Text output from the Code Interpreter tool call as part of a run step. + properties: + index: + type: integer + description: The index of the output in the outputs array. + type: + description: Always `logs`. + type: string + enum: ["logs"] + logs: + type: string + description: The text output from the Code Interpreter tool call. + required: + - index + - type + + RunStepDetailsToolCallsCodeOutputImageObject: + title: Code Interpreter image output + type: object + properties: + type: + description: Always `image`. + type: string + enum: ["image"] + image: + type: object + properties: + file_id: + description: The [file](/docs/api-reference/files) ID of the image. + type: string + required: + - file_id + required: + - type + - image + + RunStepDeltaStepDetailsToolCallsCodeOutputImageObject: + title: Code interpreter image output + type: object + properties: + index: + type: integer + description: The index of the output in the outputs array. + type: + description: Always `image`. + type: string + enum: ["image"] + image: + type: object + properties: + file_id: + description: The [file](/docs/api-reference/files) ID of the image. + type: string + required: + - index + - type + + RunStepDetailsToolCallsFileSearchObject: + title: File search tool call + type: object + properties: + id: + type: string + description: The ID of the tool call object. + type: + type: string + description: The type of tool call. This is always going to be `file_search` for this type of tool call. + enum: ["file_search"] + file_search: + type: object + description: For now, this is always going to be an empty object. + x-oaiTypeLabel: map + required: + - id + - type + - file_search + + RunStepDeltaStepDetailsToolCallsFileSearchObject: + title: File search tool call + type: object + properties: + index: + type: integer + description: The index of the tool call in the tool calls array. + id: + type: string + description: The ID of the tool call object. + type: + type: string + description: The type of tool call. This is always going to be `file_search` for this type of tool call. + enum: ["file_search"] + file_search: + type: object + description: For now, this is always going to be an empty object. + x-oaiTypeLabel: map + required: + - index + - type + - file_search + + RunStepDetailsToolCallsFunctionObject: + type: object + title: Function tool call + properties: + id: + type: string + description: The ID of the tool call object. + type: + type: string + description: The type of tool call. This is always going to be `function` for this type of tool call. + enum: ["function"] + function: + type: object + description: The definition of the function that was called. + properties: + name: + type: string + description: The name of the function. + arguments: + type: string + description: The arguments passed to the function. + output: + type: string + description: The output of the function. This will be `null` if the outputs have not been [submitted](/docs/api-reference/runs/submitToolOutputs) yet. + nullable: true + required: + - name + - arguments + - output + required: + - id + - type + - function + + RunStepDeltaStepDetailsToolCallsFunctionObject: + type: object + title: Function tool call + properties: + index: + type: integer + description: The index of the tool call in the tool calls array. + id: + type: string + description: The ID of the tool call object. + type: + type: string + description: The type of tool call. This is always going to be `function` for this type of tool call. + enum: ["function"] + function: + type: object + description: The definition of the function that was called. + properties: + name: + type: string + description: The name of the function. + arguments: + type: string + description: The arguments passed to the function. + output: + type: string + description: The output of the function. This will be `null` if the outputs have not been [submitted](/docs/api-reference/runs/submitToolOutputs) yet. + nullable: true + required: + - index + - type + + VectorStoreExpirationAfter: + type: object + title: Vector store expiration policy + description: The expiration policy for a vector store. + properties: + anchor: + description: "Anchor timestamp after which the expiration policy applies. Supported anchors: `last_active_at`." + type: string + enum: ["last_active_at"] + days: + description: The number of days after the anchor time that the vector store will expire. + type: integer + minimum: 1 + maximum: 365 + required: + - anchor + - days + + VectorStoreObject: + type: object + title: Vector store + description: A vector store is a collection of processed files can be used by the `file_search` tool. + properties: + id: + description: The identifier, which can be referenced in API endpoints. + type: string + object: + description: The object type, which is always `vector_store`. + type: string + enum: ["vector_store"] + created_at: + description: The Unix timestamp (in seconds) for when the vector store was created. + type: integer + name: + description: The name of the vector store. + type: string + usage_bytes: + description: The total number of bytes used by the files in the vector store. + type: integer + file_counts: + type: object + properties: + in_progress: + description: The number of files that are currently being processed. + type: integer + completed: + description: The number of files that have been successfully processed. + type: integer + failed: + description: The number of files that have failed to process. + type: integer + cancelled: + description: The number of files that were cancelled. + type: integer + total: + description: The total number of files. + type: integer + required: + - in_progress + - completed + - failed + - cancelled + - total + status: + description: The status of the vector store, which can be either `expired`, `in_progress`, or `completed`. A status of `completed` indicates that the vector store is ready for use. + type: string + enum: ["expired", "in_progress", "completed"] + expires_after: + $ref: "#/components/schemas/VectorStoreExpirationAfter" + expires_at: + description: The Unix timestamp (in seconds) for when the vector store will expire. + type: integer + nullable: true + last_active_at: + description: The Unix timestamp (in seconds) for when the vector store was last active. + type: integer + nullable: true + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + required: + - id + - object + - usage_bytes + - created_at + - status + - last_active_at + - name + - file_counts + - metadata + x-oaiMeta: + name: The vector store object + beta: true + example: | + { + "id": "vs_123", + "object": "vector_store", + "created_at": 1698107661, + "usage_bytes": 123456, + "last_active_at": 1698107661, + "name": "my_vector_store", + "status": "completed", + "file_counts": { + "in_progress": 0, + "completed": 100, + "cancelled": 0, + "failed": 0, + "total": 100 + }, + "metadata": {}, + "last_used_at": 1698107661 + } + + CreateVectorStoreRequest: + type: object + additionalProperties: false + properties: + file_ids: + description: A list of [File](/docs/api-reference/files) IDs that the vector store should use. Useful for tools like `file_search` that can access files. + type: array + maxItems: 500 + items: + type: string + name: + description: The name of the vector store. + type: string + expires_after: + $ref: "#/components/schemas/VectorStoreExpirationAfter" + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + + UpdateVectorStoreRequest: + type: object + additionalProperties: false + properties: + name: + description: The name of the vector store. + type: string + nullable: true + expires_after: + $ref: "#/components/schemas/VectorStoreExpirationAfter" + nullable: true + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + + ListVectorStoresResponse: + properties: + object: + type: string + example: "list" + data: + type: array + items: + $ref: "#/components/schemas/VectorStoreObject" + first_id: + type: string + example: "vs_abc123" + last_id: + type: string + example: "vs_abc456" + has_more: + type: boolean + example: false + required: + - object + - data + - first_id + - last_id + - has_more + + DeleteVectorStoreResponse: + type: object + properties: + id: + type: string + deleted: + type: boolean + object: + type: string + enum: [vector_store.deleted] + required: + - id + - object + - deleted + + VectorStoreFileObject: + type: object + title: Vector store files + description: A list of files attached to a vector store. + properties: + id: + description: The identifier, which can be referenced in API endpoints. + type: string + object: + description: The object type, which is always `vector_store.file`. + type: string + enum: ["vector_store.file"] + usage_bytes: + description: The total vector store usage in bytes. Note that this may be different from the original file size. + type: integer + created_at: + description: The Unix timestamp (in seconds) for when the vector store file was created. + type: integer + vector_store_id: + description: The ID of the [vector store](/docs/api-reference/vector-stores/object) that the [File](/docs/api-reference/files) is attached to. + type: string + status: + description: The status of the vector store file, which can be either `in_progress`, `completed`, `cancelled`, or `failed`. The status `completed` indicates that the vector store file is ready for use. + type: string + enum: ["in_progress", "completed", "cancelled", "failed"] + last_error: + type: object + description: The last error associated with this vector store file. Will be `null` if there are no errors. + nullable: true + properties: + code: + type: string + description: One of `server_error` or `rate_limit_exceeded`. + enum: + [ + "internal_error", + "file_not_found", + "parsing_error", + "unhandled_mime_type", + ] + message: + type: string + description: A human-readable description of the error. + required: + - code + - message + required: + - id + - object + - usage_bytes + - created_at + - vector_store_id + - status + - last_error + x-oaiMeta: + name: The vector store file object + beta: true + example: | + { + "id": "file-abc123", + "object": "vector_store.file", + "usage_bytes": 1234, + "created_at": 1698107661, + "vector_store_id": "vs_abc123", + "status": "completed", + "last_error": null + } + + CreateVectorStoreFileRequest: + type: object + additionalProperties: false + properties: + file_id: + description: A [File](/docs/api-reference/files) ID that the vector store should use. Useful for tools like `file_search` that can access files. + type: string + required: + - file_id + + ListVectorStoreFilesResponse: + properties: + object: + type: string + example: "list" + data: + type: array + items: + $ref: "#/components/schemas/VectorStoreFileObject" + first_id: + type: string + example: "file-abc123" + last_id: + type: string + example: "file-abc456" + has_more: + type: boolean + example: false + required: + - object + - data + - first_id + - last_id + - has_more + + DeleteVectorStoreFileResponse: + type: object + properties: + id: + type: string + deleted: + type: boolean + object: + type: string + enum: [vector_store.file.deleted] + required: + - id + - object + - deleted + + VectorStoreFileBatchObject: + type: object + title: Vector store file batch + description: A batch of files attached to a vector store. + properties: + id: + description: The identifier, which can be referenced in API endpoints. + type: string + object: + description: The object type, which is always `vector_store.file_batch`. + type: string + enum: ["vector_store.files_batch"] + created_at: + description: The Unix timestamp (in seconds) for when the vector store files batch was created. + type: integer + vector_store_id: + description: The ID of the [vector store](/docs/api-reference/vector-stores/object) that the [File](/docs/api-reference/files) is attached to. + type: string + status: + description: The status of the vector store files batch, which can be either `in_progress`, `completed`, `cancelled` or `failed`. + type: string + enum: ["in_progress", "completed", "cancelled", "failed"] + file_counts: + type: object + properties: + in_progress: + description: The number of files that are currently being processed. + type: integer + completed: + description: The number of files that have been processed. + type: integer + failed: + description: The number of files that have failed to process. + type: integer + cancelled: + description: The number of files that where cancelled. + type: integer + total: + description: The total number of files. + type: integer + required: + - in_progress + - completed + - cancelled + - failed + - total + required: + - id + - object + - created_at + - vector_store_id + - status + - file_counts + x-oaiMeta: + name: The vector store files batch object + beta: true + example: | + { + "id": "vsfb_123", + "object": "vector_store.files_batch", + "created_at": 1698107661, + "vector_store_id": "vs_abc123", + "status": "completed", + "file_counts": { + "in_progress": 0, + "completed": 100, + "failed": 0, + "cancelled": 0, + "total": 100 + } + } + + CreateVectorStoreFileBatchRequest: + type: object + additionalProperties: false + properties: + file_ids: + description: A list of [File](/docs/api-reference/files) IDs that the vector store should use. Useful for tools like `file_search` that can access files. + type: array + minItems: 1 + maxItems: 500 + items: + type: string + required: + - file_ids + + AssistantStreamEvent: + description: | + Represents an event emitted when streaming a Run. + + Each event in a server-sent events stream has an `event` and `data` property: + + ``` + event: thread.created + data: {"id": "thread_123", "object": "thread", ...} + ``` + + We emit events whenever a new object is created, transitions to a new state, or is being + streamed in parts (deltas). For example, we emit `thread.run.created` when a new run + is created, `thread.run.completed` when a run completes, and so on. When an Assistant chooses + to create a message during a run, we emit a `thread.message.created event`, a + `thread.message.in_progress` event, many `thread.message.delta` events, and finally a + `thread.message.completed` event. + + We may add additional events over time, so we recommend handling unknown events gracefully + in your code. See the [Assistants API quickstart](/docs/assistants/overview) to learn how to + integrate the Assistants API with streaming. + oneOf: + - $ref: "#/components/schemas/ThreadStreamEvent" + - $ref: "#/components/schemas/RunStreamEvent" + - $ref: "#/components/schemas/RunStepStreamEvent" + - $ref: "#/components/schemas/MessageStreamEvent" + - $ref: "#/components/schemas/ErrorEvent" + - $ref: "#/components/schemas/DoneEvent" + x-oaiMeta: + name: Assistant stream events + beta: true + + ThreadStreamEvent: + oneOf: + - type: object + properties: + event: + type: string + enum: ["thread.created"] + data: + $ref: "#/components/schemas/ThreadObject" + required: + - event + - data + description: Occurs when a new [thread](/docs/api-reference/threads/object) is created. + x-oaiMeta: + dataDescription: "`data` is a [thread](/docs/api-reference/threads/object)" + + RunStreamEvent: + oneOf: + - type: object + properties: + event: + type: string + enum: ["thread.run.created"] + data: + $ref: "#/components/schemas/RunObject" + required: + - event + - data + description: Occurs when a new [run](/docs/api-reference/runs/object) is created. + x-oaiMeta: + dataDescription: "`data` is a [run](/docs/api-reference/runs/object)" + - type: object + properties: + event: + type: string + enum: ["thread.run.queued"] + data: + $ref: "#/components/schemas/RunObject" + required: + - event + - data + description: Occurs when a [run](/docs/api-reference/runs/object) moves to a `queued` status. + x-oaiMeta: + dataDescription: "`data` is a [run](/docs/api-reference/runs/object)" + - type: object + properties: + event: + type: string + enum: ["thread.run.in_progress"] + data: + $ref: "#/components/schemas/RunObject" + required: + - event + - data + description: Occurs when a [run](/docs/api-reference/runs/object) moves to an `in_progress` status. + x-oaiMeta: + dataDescription: "`data` is a [run](/docs/api-reference/runs/object)" + - type: object + properties: + event: + type: string + enum: ["thread.run.requires_action"] + data: + $ref: "#/components/schemas/RunObject" + required: + - event + - data + description: Occurs when a [run](/docs/api-reference/runs/object) moves to a `requires_action` status. + x-oaiMeta: + dataDescription: "`data` is a [run](/docs/api-reference/runs/object)" + - type: object + properties: + event: + type: string + enum: ["thread.run.completed"] + data: + $ref: "#/components/schemas/RunObject" + required: + - event + - data + description: Occurs when a [run](/docs/api-reference/runs/object) is completed. + x-oaiMeta: + dataDescription: "`data` is a [run](/docs/api-reference/runs/object)" + - type: object + properties: + event: + type: string + enum: ["thread.run.failed"] + data: + $ref: "#/components/schemas/RunObject" + required: + - event + - data + description: Occurs when a [run](/docs/api-reference/runs/object) fails. + x-oaiMeta: + dataDescription: "`data` is a [run](/docs/api-reference/runs/object)" + - type: object + properties: + event: + type: string + enum: ["thread.run.cancelling"] + data: + $ref: "#/components/schemas/RunObject" + required: + - event + - data + description: Occurs when a [run](/docs/api-reference/runs/object) moves to a `cancelling` status. + x-oaiMeta: + dataDescription: "`data` is a [run](/docs/api-reference/runs/object)" + - type: object + properties: + event: + type: string + enum: ["thread.run.cancelled"] + data: + $ref: "#/components/schemas/RunObject" + required: + - event + - data + description: Occurs when a [run](/docs/api-reference/runs/object) is cancelled. + x-oaiMeta: + dataDescription: "`data` is a [run](/docs/api-reference/runs/object)" + - type: object + properties: + event: + type: string + enum: ["thread.run.expired"] + data: + $ref: "#/components/schemas/RunObject" + required: + - event + - data + description: Occurs when a [run](/docs/api-reference/runs/object) expires. + x-oaiMeta: + dataDescription: "`data` is a [run](/docs/api-reference/runs/object)" + + RunStepStreamEvent: + oneOf: + - type: object + properties: + event: + type: string + enum: ["thread.run.step.created"] + data: + $ref: "#/components/schemas/RunStepObject" + required: + - event + - data + description: Occurs when a [run step](/docs/api-reference/runs/step-object) is created. + x-oaiMeta: + dataDescription: "`data` is a [run step](/docs/api-reference/runs/step-object)" + - type: object + properties: + event: + type: string + enum: ["thread.run.step.in_progress"] + data: + $ref: "#/components/schemas/RunStepObject" + required: + - event + - data + description: Occurs when a [run step](/docs/api-reference/runs/step-object) moves to an `in_progress` state. + x-oaiMeta: + dataDescription: "`data` is a [run step](/docs/api-reference/runs/step-object)" + - type: object + properties: + event: + type: string + enum: ["thread.run.step.delta"] + data: + $ref: "#/components/schemas/RunStepDeltaObject" + required: + - event + - data + description: Occurs when parts of a [run step](/docs/api-reference/runs/step-object) are being streamed. + x-oaiMeta: + dataDescription: "`data` is a [run step delta](/docs/api-reference/assistants-streaming/run-step-delta-object)" + - type: object + properties: + event: + type: string + enum: ["thread.run.step.completed"] + data: + $ref: "#/components/schemas/RunStepObject" + required: + - event + - data + description: Occurs when a [run step](/docs/api-reference/runs/step-object) is completed. + x-oaiMeta: + dataDescription: "`data` is a [run step](/docs/api-reference/runs/step-object)" + - type: object + properties: + event: + type: string + enum: ["thread.run.step.failed"] + data: + $ref: "#/components/schemas/RunStepObject" + required: + - event + - data + description: Occurs when a [run step](/docs/api-reference/runs/step-object) fails. + x-oaiMeta: + dataDescription: "`data` is a [run step](/docs/api-reference/runs/step-object)" + - type: object + properties: + event: + type: string + enum: ["thread.run.step.cancelled"] + data: + $ref: "#/components/schemas/RunStepObject" + required: + - event + - data + description: Occurs when a [run step](/docs/api-reference/runs/step-object) is cancelled. + x-oaiMeta: + dataDescription: "`data` is a [run step](/docs/api-reference/runs/step-object)" + - type: object + properties: + event: + type: string + enum: ["thread.run.step.expired"] + data: + $ref: "#/components/schemas/RunStepObject" + required: + - event + - data + description: Occurs when a [run step](/docs/api-reference/runs/step-object) expires. + x-oaiMeta: + dataDescription: "`data` is a [run step](/docs/api-reference/runs/step-object)" + + MessageStreamEvent: + oneOf: + - type: object + properties: + event: + type: string + enum: ["thread.message.created"] + data: + $ref: "#/components/schemas/MessageObject" + required: + - event + - data + description: Occurs when a [message](/docs/api-reference/messages/object) is created. + x-oaiMeta: + dataDescription: "`data` is a [message](/docs/api-reference/messages/object)" + - type: object + properties: + event: + type: string + enum: ["thread.message.in_progress"] + data: + $ref: "#/components/schemas/MessageObject" + required: + - event + - data + description: Occurs when a [message](/docs/api-reference/messages/object) moves to an `in_progress` state. + x-oaiMeta: + dataDescription: "`data` is a [message](/docs/api-reference/messages/object)" + - type: object + properties: + event: + type: string + enum: ["thread.message.delta"] + data: + $ref: "#/components/schemas/MessageDeltaObject" + required: + - event + - data + description: Occurs when parts of a [Message](/docs/api-reference/messages/object) are being streamed. + x-oaiMeta: + dataDescription: "`data` is a [message delta](/docs/api-reference/assistants-streaming/message-delta-object)" + - type: object + properties: + event: + type: string + enum: ["thread.message.completed"] + data: + $ref: "#/components/schemas/MessageObject" + required: + - event + - data + description: Occurs when a [message](/docs/api-reference/messages/object) is completed. + x-oaiMeta: + dataDescription: "`data` is a [message](/docs/api-reference/messages/object)" + - type: object + properties: + event: + type: string + enum: ["thread.message.incomplete"] + data: + $ref: "#/components/schemas/MessageObject" + required: + - event + - data + description: Occurs when a [message](/docs/api-reference/messages/object) ends before it is completed. + x-oaiMeta: + dataDescription: "`data` is a [message](/docs/api-reference/messages/object)" + + ErrorEvent: + type: object + properties: + event: + type: string + enum: ["error"] + data: + $ref: "#/components/schemas/Error" + required: + - event + - data + description: Occurs when an [error](/docs/guides/error-codes/api-errors) occurs. This can happen due to an internal server error or a timeout. + x-oaiMeta: + dataDescription: "`data` is an [error](/docs/guides/error-codes/api-errors)" + + DoneEvent: + type: object + properties: + event: + type: string + enum: ["done"] + data: + type: string + enum: ["[DONE]"] + required: + - event + - data + description: Occurs when a stream ends. + x-oaiMeta: + dataDescription: "`data` is `[DONE]`" + + Batch: + type: object + properties: + id: + type: string + object: + type: string + enum: [batch] + description: The object type, which is always `batch`. + endpoint: + type: string + description: The OpenAI API endpoint used by the batch. + + errors: + type: object + properties: + object: + type: string + description: The object type, which is always `list`. + data: + type: array + items: + type: object + properties: + code: + type: string + description: An error code identifying the error type. + message: + type: string + description: A human-readable message providing more details about the error. + param: + type: string + description: The name of the parameter that caused the error, if applicable. + nullable: true + line: + type: integer + description: The line number of the input file where the error occurred, if applicable. + nullable: true + input_file_id: + type: string + description: The ID of the input file for the batch. + completion_window: + type: string + description: The time frame within which the batch should be processed. + status: + type: string + description: The current status of the batch. + enum: + - validating + - failed + - in_progress + - finalizing + - completed + - expired + - cancelling + - cancelled + output_file_id: + type: string + description: The ID of the file containing the outputs of successfully executed requests. + error_file_id: + type: string + description: The ID of the file containing the outputs of requests with errors. + created_at: + type: integer + description: The Unix timestamp (in seconds) for when the batch was created. + in_progress_at: + type: integer + description: The Unix timestamp (in seconds) for when the batch started processing. + expires_at: + type: integer + description: The Unix timestamp (in seconds) for when the batch will expire. + finalizing_at: + type: integer + description: The Unix timestamp (in seconds) for when the batch started finalizing. + completed_at: + type: integer + description: The Unix timestamp (in seconds) for when the batch was completed. + failed_at: + type: integer + description: The Unix timestamp (in seconds) for when the batch failed. + expired_at: + type: integer + description: The Unix timestamp (in seconds) for when the batch expired. + cancelling_at: + type: integer + description: The Unix timestamp (in seconds) for when the batch started cancelling. + cancelled_at: + type: integer + description: The Unix timestamp (in seconds) for when the batch was cancelled. + request_counts: + type: object + properties: + total: + type: integer + description: Total number of requests in the batch. + completed: + type: integer + description: Number of requests that have been completed successfully. + failed: + type: integer + description: Number of requests that have failed. + required: + - total + - completed + - failed + description: The request counts for different statuses within the batch. + metadata: + description: *metadata_description + type: object + x-oaiTypeLabel: map + nullable: true + required: + - id + - object + - endpoint + - input_file_id + - completion_window + - status + - created_at + x-oaiMeta: + name: The batch object + example: *batch_object + + BatchRequestInput: + type: object + description: The per-line object of the batch input file + properties: + custom_id: + type: string + description: A developer-provided per-request id that will be used to match outputs to inputs. Must be unique for each request in a batch. + method: + type: string + enum: ["POST"] + description: The HTTP method to be used for the request. Currently only `POST` is supported. + url: + type: string + description: The OpenAI API relative URL to be used for the request. Currently `/v1/chat/completions` and `/v1/embeddings` are supported. + x-oaiMeta: + name: The request input object + example: | + {"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "What is 2+2?"}]}} + + BatchRequestOutput: + type: object + description: The per-line object of the batch output and error files + properties: + id: + type: string + custom_id: + type: string + description: A developer-provided per-request id that will be used to match outputs to inputs. + response: + type: object + nullable: true + properties: + status_code: + type: integer + description: The HTTP status code of the response + request_id: + type: string + description: An unique identifier for the OpenAI API request. Please include this request ID when contacting support. + body: + type: object + x-oaiTypeLabel: map + description: The JSON body of the response + error: + type: object + nullable: true + description: For requests that failed with a non-HTTP error, this will contain more information on the cause of the failure. + properties: + code: + type: string + description: A machine-readable error code. + message: + type: string + description: A human-readable error message. + x-oaiMeta: + name: The request output object + example: | + {"id": "batch_req_wnaDys", "custom_id": "request-2", "response": {"status_code": 200, "request_id": "req_c187b3", "body": {"id": "chatcmpl-9758Iw", "object": "chat.completion", "created": 1711475054, "model": "gpt-3.5-turbo", "choices": [{"index": 0, "message": {"role": "assistant", "content": "2 + 2 equals 4."}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 24, "completion_tokens": 15, "total_tokens": 39}, "system_fingerprint": null}}, "error": null} + + ListBatchesResponse: + type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/Batch" + first_id: + type: string + example: "batch_abc123" + last_id: + type: string + example: "batch_abc456" + has_more: + type: boolean + object: + type: string + enum: [list] + required: + - object + - data + - has_more + +security: + - ApiKeyAuth: [] + +x-oaiMeta: + navigationGroups: + - id: endpoints + title: Endpoints + - id: assistants + title: Assistants + - id: legacy + title: Legacy + groups: + # > General Notes + # The `groups` section is used to generate the API reference pages and navigation, in the same + # order listed below. Additionally, each `group` can have a list of `sections`, each of which + # will become a navigation subroute and subsection under the group. Each section has: + # - `type`: Currently, either an `endpoint` or `object`, depending on how the section needs to + # be rendered + # - `key`: The reference key that can be used to lookup the section definition + # - `path`: The path (url) of the section, which is used to generate the navigation link. + # + # > The `object` sections maps to a schema component and the following fields are read for rendering + # - `x-oaiMeta.name`: The name of the object, which will become the section title + # - `x-oaiMeta.example`: The example object, which will be used to generate the example sample (always JSON) + # - `description`: The description of the object, which will be used to generate the section description + # + # > The `endpoint` section maps to an operation path and the following fields are read for rendering: + # - `x-oaiMeta.name`: The name of the endpoint, which will become the section title + # - `x-oaiMeta.examples`: The endpoint examples, which can be an object (meaning a single variation, most + # endpoints, or an array of objects, meaning multiple variations, e.g. the + # chat completion and completion endpoints, with streamed and non-streamed examples. + # - `x-oaiMeta.returns`: text describing what the endpoint returns. + # - `summary`: The summary of the endpoint, which will be used to generate the section description + - id: audio + title: Audio + description: | + Learn how to turn audio into text or text into audio. + + Related guide: [Speech to text](/docs/guides/speech-to-text) + navigationGroup: endpoints + sections: + - type: endpoint + key: createSpeech + path: createSpeech + - type: endpoint + key: createTranscription + path: createTranscription + - type: endpoint + key: createTranslation + path: createTranslation + - type: object + key: CreateTranscriptionResponseJson + path: json-object + - type: object + key: CreateTranscriptionResponseVerboseJson + path: verbose-json-object + - id: chat + title: Chat + description: | + Given a list of messages comprising a conversation, the model will return a response. + + Related guide: [Chat Completions](/docs/guides/text-generation) + navigationGroup: endpoints + sections: + - type: endpoint + key: createChatCompletion + path: create + - type: object + key: CreateChatCompletionResponse + path: object + - type: object + key: CreateChatCompletionStreamResponse + path: streaming + - id: embeddings + title: Embeddings + description: | + Get a vector representation of a given input that can be easily consumed by machine learning models and algorithms. + + Related guide: [Embeddings](/docs/guides/embeddings) + navigationGroup: endpoints + sections: + - type: endpoint + key: createEmbedding + path: create + - type: object + key: Embedding + path: object + - id: fine-tuning + title: Fine-tuning + description: | + Manage fine-tuning jobs to tailor a model to your specific training data. + + Related guide: [Fine-tune models](/docs/guides/fine-tuning) + navigationGroup: endpoints + sections: + - type: endpoint + key: createFineTuningJob + path: create + - type: endpoint + key: listPaginatedFineTuningJobs + path: list + - type: endpoint + key: listFineTuningEvents + path: list-events + - type: endpoint + key: listFineTuningJobCheckpoints + path: list-checkpoints + - type: endpoint + key: retrieveFineTuningJob + path: retrieve + - type: endpoint + key: cancelFineTuningJob + path: cancel + - type: object + key: FineTuningJob + path: object + - type: object + key: FineTuningJobEvent + path: event-object + - type: object + key: FineTuningJobCheckpoint + path: checkpoint-object + - id: batch + title: Batch + description: | + Create large batches of API requests for asynchronous processing. The Batch API returns completions within 24 hours for a 50% discount. + + Related guide: [Batch](/docs/guides/batch) + navigationGroup: endpoints + sections: + - type: endpoint + key: createBatch + path: create + - type: endpoint + key: retrieveBatch + path: retrieve + - type: endpoint + key: cancelBatch + path: cancel + - type: endpoint + key: listBatches + path: list + - type: object + key: Batch + path: object + - type: object + key: BatchRequestInput + path: requestInput + - type: object + key: BatchRequestOutput + path: requestOutput + - id: files + title: Files + description: | + Files are used to upload documents that can be used with features like [Assistants](/docs/api-reference/assistants) and [Fine-tuning](/docs/api-reference/fine-tuning). + navigationGroup: endpoints + sections: + - type: endpoint + key: createFile + path: create + - type: endpoint + key: listFiles + path: list + - type: endpoint + key: retrieveFile + path: retrieve + - type: endpoint + key: deleteFile + path: delete + - type: endpoint + key: downloadFile + path: retrieve-contents + - type: object + key: OpenAIFile + path: object + - id: images + title: Images + description: | + Given a prompt and/or an input image, the model will generate a new image. + + Related guide: [Image generation](/docs/guides/images) + navigationGroup: endpoints + sections: + - type: endpoint + key: createImage + path: create + - type: endpoint + key: createImageEdit + path: createEdit + - type: endpoint + key: createImageVariation + path: createVariation + - type: object + key: Image + path: object + - id: models + title: Models + description: | + List and describe the various models available in the API. You can refer to the [Models](/docs/models) documentation to understand what models are available and the differences between them. + navigationGroup: endpoints + sections: + - type: endpoint + key: listModels + path: list + - type: endpoint + key: retrieveModel + path: retrieve + - type: endpoint + key: deleteModel + path: delete + - type: object + key: Model + path: object + - id: moderations + title: Moderations + description: | + Given some input text, outputs if the model classifies it as potentially harmful across several categories. + + Related guide: [Moderations](/docs/guides/moderation) + navigationGroup: endpoints + sections: + - type: endpoint + key: createModeration + path: create + - type: object + key: CreateModerationResponse + path: object + - id: assistants + title: Assistants + beta: true + description: | + Build assistants that can call models and use tools to perform tasks. + + [Get started with the Assistants API](/docs/assistants) + navigationGroup: assistants + sections: + - type: endpoint + key: createAssistant + path: createAssistant + - type: endpoint + key: listAssistants + path: listAssistants + - type: endpoint + key: getAssistant + path: getAssistant + - type: endpoint + key: modifyAssistant + path: modifyAssistant + - type: endpoint + key: deleteAssistant + path: deleteAssistant + - type: object + key: AssistantObject + path: object + - id: threads + title: Threads + beta: true + description: | + Create threads that assistants can interact with. + + Related guide: [Assistants](/docs/assistants/overview) + navigationGroup: assistants + sections: + - type: endpoint + key: createThread + path: createThread + - type: endpoint + key: getThread + path: getThread + - type: endpoint + key: modifyThread + path: modifyThread + - type: endpoint + key: deleteThread + path: deleteThread + - type: object + key: ThreadObject + path: object + - id: messages + title: Messages + beta: true + description: | + Create messages within threads + + Related guide: [Assistants](/docs/assistants/overview) + navigationGroup: assistants + sections: + - type: endpoint + key: createMessage + path: createMessage + - type: endpoint + key: listMessages + path: listMessages + - type: endpoint + key: getMessage + path: getMessage + - type: endpoint + key: modifyMessage + path: modifyMessage + - type: endpoint + key: deleteMessage + path: deleteMessage + - type: object + key: MessageObject + path: object + - id: runs + title: Runs + beta: true + description: | + Represents an execution run on a thread. + + Related guide: [Assistants](/docs/assistants/overview) + navigationGroup: assistants + sections: + - type: endpoint + key: createRun + path: createRun + - type: endpoint + key: createThreadAndRun + path: createThreadAndRun + - type: endpoint + key: listRuns + path: listRuns + - type: endpoint + key: getRun + path: getRun + - type: endpoint + key: modifyRun + path: modifyRun + - type: endpoint + key: submitToolOuputsToRun + path: submitToolOutputs + - type: endpoint + key: cancelRun + path: cancelRun + - type: object + key: RunObject + path: object + - id: run-steps + title: Run Steps + beta: true + description: | + Represents the steps (model and tool calls) taken during the run. + + Related guide: [Assistants](/docs/assistants/overview) + navigationGroup: assistants + sections: + - type: endpoint + key: listRunSteps + path: listRunSteps + - type: endpoint + key: getRunStep + path: getRunStep + - type: object + key: RunStepObject + path: step-object + - id: vector-stores + title: Vector Stores + beta: true + description: | + Vector stores are used to store files for use by the `file_search` tool. + + Related guide: [File Search](/docs/assistants/tools/file-search) + navigationGroup: assistants + sections: + - type: endpoint + key: createVectorStore + path: create + - type: endpoint + key: listVectorStores + path: list + - type: endpoint + key: getVectorStore + path: retrieve + - type: endpoint + key: modifyVectorStore + path: modify + - type: endpoint + key: deleteVectorStore + path: delete + - type: object + key: VectorStoreObject + path: object + - id: vector-stores-files + title: Vector Store Files + beta: true + description: | + Vector store files represent files inside a vector store. + + Related guide: [File Search](/docs/assistants/tools/file-search) + navigationGroup: assistants + sections: + - type: endpoint + key: createVectorStoreFile + path: createFile + - type: endpoint + key: listVectorStoreFiles + path: listFiles + - type: endpoint + key: getVectorStoreFile + path: getFile + - type: endpoint + key: deleteVectorStoreFile + path: deleteFile + - type: object + key: VectorStoreFileObject + path: file-object + - id: vector-stores-file-batches + title: Vector Store File Batches + beta: true + description: | + Vector store file batches represent operations to add multiple files to a vector store. + + Related guide: [File Search](/docs/assistants/tools/file-search) + navigationGroup: assistants + sections: + - type: endpoint + key: createVectorStoreFileBatch + path: createBatch + - type: endpoint + key: getVectorStoreFileBatch + path: getBatch + - type: endpoint + key: cancelVectorStoreFileBatch + path: cancelBatch + - type: endpoint + key: listFilesInVectorStoreBatch + path: listBatchFiles + - type: object + key: VectorStoreFileBatchObject + path: batch-object + - id: assistants-streaming + title: Streaming + beta: true + description: | + Stream the result of executing a Run or resuming a Run after submitting tool outputs. + + You can stream events from the [Create Thread and Run](/docs/api-reference/runs/createThreadAndRun), + [Create Run](/docs/api-reference/runs/createRun), and [Submit Tool Outputs](/docs/api-reference/runs/submitToolOutputs) + endpoints by passing `"stream": true`. The response will be a [Server-Sent events](https://html.spec.whatwg.org/multipage/server-sent-events.html#server-sent-events) stream. + + Our Node and Python SDKs provide helpful utilities to make streaming easy. Reference the + [Assistants API quickstart](/docs/assistants/overview) to learn more. + navigationGroup: assistants + sections: + - type: object + key: MessageDeltaObject + path: message-delta-object + - type: object + key: RunStepDeltaObject + path: run-step-delta-object + - type: object + key: AssistantStreamEvent + path: events + - id: completions + title: Completions + legacy: true + navigationGroup: legacy + description: | + Given a prompt, the model will return one or more predicted completions along with the probabilities of alternative tokens at each position. Most developer should use our [Chat Completions API](/docs/guides/text-generation/text-generation-models) to leverage our best and newest models. + sections: + - type: endpoint + key: createCompletion + path: create + - type: object + key: CreateCompletionResponse + path: object From 85c9c068ea278d1c00d6d9779617bdd1bb9b7f82 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 5 May 2024 11:40:00 -0700 Subject: [PATCH 16/62] moving generated files into subdirectories --- .../{simple => fastapi-codegen}/main.py | 0 .../{simple => fastapi-codegen}/models.py | 0 .../openai-server/.flake8 | 0 .../openai-server/.gitignore | 0 .../openai-server/.openapi-generator-ignore | 0 .../openai-server/.openapi-generator/FILES | 0 .../openai-server/.openapi-generator/VERSION | 0 .../openai-server/Dockerfile | 0 .../openai-server/README.md | 0 .../openai-server/docker-compose.yaml | 0 .../openai-server/openapi.yaml | 0 .../openai-server/pyproject.toml | 0 .../openai-server/requirements.txt | 0 .../openai-server/setup.cfg | 0 .../src/openapi_server/apis/__init__.py | 0 .../src/openapi_server/apis/chat_api.py | 3 +-- .../src/openapi_server/apis/chat_api_base.py | 0 .../openapi_server/apis/completions_api.py | 4 +--- .../apis/completions_api_base.py | 0 .../src/openapi_server/apis/models_api.py | 3 +-- .../openapi_server/apis/models_api_base.py | 0 .../src/openapi_server/impl/__init__.py | 0 .../openai-server/src/openapi_server/main.py | 1 - .../src/openapi_server/models/__init__.py | 0 .../chat_completion_function_call_option.py | 0 .../models/chat_completion_functions.py | 0 .../chat_completion_message_tool_call.py | 3 +-- ...chat_completion_message_tool_call_chunk.py | 3 +-- ...letion_message_tool_call_chunk_function.py | 0 ...t_completion_message_tool_call_function.py | 0 .../chat_completion_named_tool_choice.py | 3 +-- ...t_completion_named_tool_choice_function.py | 0 ...at_completion_request_assistant_message.py | 3 +-- ...request_assistant_message_function_call.py | 0 ...hat_completion_request_function_message.py | 0 .../models/chat_completion_request_message.py | 19 +++++++++---------- ...completion_request_message_content_part.py | 13 ++++++------- ...tion_request_message_content_part_image.py | 3 +-- ...st_message_content_part_image_image_url.py | 0 ...etion_request_message_content_part_text.py | 0 .../chat_completion_request_system_message.py | 0 .../chat_completion_request_tool_message.py | 0 .../chat_completion_request_user_message.py | 3 +-- ...completion_request_user_message_content.py | 7 +++---- .../chat_completion_response_message.py | 3 +-- .../models/chat_completion_role.py | 0 .../chat_completion_stream_response_delta.py | 3 +-- ...ion_stream_response_delta_function_call.py | 0 .../models/chat_completion_token_logprob.py | 3 +-- ...letion_token_logprob_top_logprobs_inner.py | 0 .../models/chat_completion_tool.py | 3 +-- .../chat_completion_tool_choice_option.py | 7 +++---- .../openapi_server/models/completion_usage.py | 0 ...reate_chat_completion_function_response.py | 3 +-- ...pletion_function_response_choices_inner.py | 3 +-- .../models/create_chat_completion_request.py | 5 ++--- ...e_chat_completion_request_function_call.py | 7 +++---- .../create_chat_completion_request_model.py | 0 ...chat_completion_request_response_format.py | 0 .../create_chat_completion_request_stop.py | 0 .../models/create_chat_completion_response.py | 3 +-- ..._chat_completion_response_choices_inner.py | 3 +-- ...pletion_response_choices_inner_logprobs.py | 3 +-- .../create_chat_completion_stream_response.py | 3 +-- ...ompletion_stream_response_choices_inner.py | 3 +-- .../models/create_completion_request.py | 5 ++--- .../models/create_completion_request_model.py | 0 .../create_completion_request_prompt.py | 0 .../models/create_completion_request_stop.py | 0 .../models/create_completion_response.py | 3 +-- ...reate_completion_response_choices_inner.py | 3 +-- ...pletion_response_choices_inner_logprobs.py | 0 .../models/delete_model_response.py | 0 .../src/openapi_server/models/done_event.py | 0 .../src/openapi_server/models/error.py | 0 .../src/openapi_server/models/error_event.py | 3 +-- .../openapi_server/models/error_response.py | 3 +-- .../src/openapi_server/models/extra_models.py | 0 .../openapi_server/models/function_object.py | 0 .../models/list_models_response.py | 3 +-- .../src/openapi_server/models/model.py | 0 .../src/openapi_server/security_api.py | 1 - .../openai-server/tests/conftest.py | 1 - .../openai-server/tests/test_chat_api.py | 1 - .../tests/test_completions_api.py | 1 - .../openai-server/tests/test_models_api.py | 1 - 86 files changed, 53 insertions(+), 92 deletions(-) rename Triton_Inference_Server_Python_API/examples/fastapi/{simple => fastapi-codegen}/main.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{simple => fastapi-codegen}/models.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/.flake8 (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/.gitignore (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/.openapi-generator-ignore (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/.openapi-generator/FILES (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/.openapi-generator/VERSION (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/Dockerfile (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/README.md (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/docker-compose.yaml (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/openapi.yaml (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/pyproject.toml (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/requirements.txt (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/setup.cfg (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/apis/__init__.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/apis/chat_api.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/apis/chat_api_base.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/apis/completions_api.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/apis/completions_api_base.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/apis/models_api.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/apis/models_api_base.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/impl/__init__.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/main.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/__init__.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_function_call_option.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_functions.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_message_tool_call.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk_function.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_message_tool_call_function.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_named_tool_choice.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_named_tool_choice_function.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_request_assistant_message.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_request_assistant_message_function_call.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_request_function_message.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_request_message.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_request_message_content_part.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image_image_url.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_text.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_request_system_message.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_request_tool_message.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_request_user_message.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_request_user_message_content.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_response_message.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_role.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_stream_response_delta.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_stream_response_delta_function_call.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_token_logprob.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_token_logprob_top_logprobs_inner.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_tool.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/chat_completion_tool_choice_option.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/completion_usage.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_chat_completion_function_response.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_chat_completion_function_response_choices_inner.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_chat_completion_request.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_chat_completion_request_function_call.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_chat_completion_request_model.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_chat_completion_request_response_format.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_chat_completion_request_stop.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_chat_completion_response.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner_logprobs.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_chat_completion_stream_response.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_chat_completion_stream_response_choices_inner.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_completion_request.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_completion_request_model.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_completion_request_prompt.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_completion_request_stop.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_completion_response.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_completion_response_choices_inner.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/create_completion_response_choices_inner_logprobs.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/delete_model_response.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/done_event.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/error.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/error_event.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/error_response.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/extra_models.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/function_object.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/list_models_response.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/models/model.py (100%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/src/openapi_server/security_api.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/tests/conftest.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/tests/test_chat_api.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/tests/test_completions_api.py (99%) rename Triton_Inference_Server_Python_API/examples/fastapi/{ => openapi-generator}/openai-server/tests/test_models_api.py (99%) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/simple/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/simple/main.py rename to Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/simple/models.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/models.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/simple/models.py rename to Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/models.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.flake8 b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/.flake8 similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.flake8 rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/.flake8 diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.gitignore b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/.gitignore similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.gitignore rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/.gitignore diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator-ignore b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/.openapi-generator-ignore similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator-ignore rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/.openapi-generator-ignore diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator/FILES b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/.openapi-generator/FILES similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator/FILES rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/.openapi-generator/FILES diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator/VERSION b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/.openapi-generator/VERSION similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/.openapi-generator/VERSION rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/.openapi-generator/VERSION diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/Dockerfile b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/Dockerfile similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/Dockerfile rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/Dockerfile diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/README.md b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/README.md similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/README.md rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/README.md diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/docker-compose.yaml b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/docker-compose.yaml similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/docker-compose.yaml rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/docker-compose.yaml diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/openapi.yaml b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/openapi.yaml similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/openapi.yaml rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/openapi.yaml diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/pyproject.toml b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/pyproject.toml similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/pyproject.toml rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/pyproject.toml diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/requirements.txt b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/requirements.txt similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/requirements.txt rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/requirements.txt diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/setup.cfg b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/setup.cfg similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/setup.cfg rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/setup.cfg diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/__init__.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/__init__.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/__init__.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/__init__.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/chat_api.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/chat_api.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/chat_api.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/chat_api.py index 013dc1b3..2864c310 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/chat_api.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/chat_api.py @@ -4,6 +4,7 @@ import pkgutil from typing import Dict, List # noqa: F401 +import openapi_server.impl from fastapi import ( # noqa: F401 APIRouter, Body, @@ -17,8 +18,6 @@ Security, status, ) - -import openapi_server.impl from openapi_server.apis.chat_api_base import BaseChatApi from openapi_server.models.create_chat_completion_request import ( CreateChatCompletionRequest, diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/chat_api_base.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/chat_api_base.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/chat_api_base.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/chat_api_base.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/completions_api.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/completions_api.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/completions_api.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/completions_api.py index 39c18780..67042de3 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/completions_api.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/completions_api.py @@ -7,9 +7,8 @@ import uuid from typing import Dict, List # noqa: F401 -import tritonserver - import openapi_server.impl +import tritonserver from openapi_server.apis.completions_api_base import BaseCompletionsApi triton_server = tritonserver.Server( @@ -29,7 +28,6 @@ Response, status, ) - from openapi_server.models.create_completion_request import CreateCompletionRequest from openapi_server.models.create_completion_response import CreateCompletionResponse from openapi_server.models.create_completion_response_choices_inner import ( diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/completions_api_base.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/completions_api_base.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/completions_api_base.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/completions_api_base.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/models_api.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/models_api.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/models_api.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/models_api.py index 045ce15b..6ba8a720 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/models_api.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/models_api.py @@ -4,6 +4,7 @@ import pkgutil from typing import Dict, List # noqa: F401 +import openapi_server.impl from fastapi import ( # noqa: F401 APIRouter, Body, @@ -17,8 +18,6 @@ Security, status, ) - -import openapi_server.impl from openapi_server.apis.models_api_base import BaseModelsApi from openapi_server.models.delete_model_response import DeleteModelResponse from openapi_server.models.extra_models import TokenModel # noqa: F401 diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/models_api_base.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/models_api_base.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/apis/models_api_base.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/apis/models_api_base.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/impl/__init__.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/impl/__init__.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/impl/__init__.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/impl/__init__.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/main.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/main.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/main.py index 92404d9d..6acaabfd 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/main.py @@ -13,7 +13,6 @@ from fastapi import FastAPI - from openapi_server.apis.chat_api import router as ChatApiRouter from openapi_server.apis.completions_api import router as CompletionsApiRouter from openapi_server.apis.models_api import router as ModelsApiRouter diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/__init__.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/__init__.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/__init__.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/__init__.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_function_call_option.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_function_call_option.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_function_call_option.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_function_call_option.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_functions.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_functions.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_functions.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_functions.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_message_tool_call.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_message_tool_call.py index 341a2a74..674e708d 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_message_tool_call.py @@ -19,11 +19,10 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List -from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator - from openapi_server.models.chat_completion_message_tool_call_function import ( ChatCompletionMessageToolCallFunction, ) +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk.py index a03dca04..66bc4f9e 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk.py @@ -19,11 +19,10 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional -from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator - from openapi_server.models.chat_completion_message_tool_call_chunk_function import ( ChatCompletionMessageToolCallChunkFunction, ) +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk_function.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk_function.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk_function.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_message_tool_call_chunk_function.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_function.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_message_tool_call_function.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_message_tool_call_function.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_message_tool_call_function.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_named_tool_choice.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_named_tool_choice.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_named_tool_choice.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_named_tool_choice.py index 7244baea..207520c9 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_named_tool_choice.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_named_tool_choice.py @@ -19,11 +19,10 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List -from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator - from openapi_server.models.chat_completion_named_tool_choice_function import ( ChatCompletionNamedToolChoiceFunction, ) +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_named_tool_choice_function.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_named_tool_choice_function.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_named_tool_choice_function.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_named_tool_choice_function.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_assistant_message.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_assistant_message.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_assistant_message.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_assistant_message.py index 2e3da02c..0359d711 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_assistant_message.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_assistant_message.py @@ -19,14 +19,13 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional -from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator - from openapi_server.models.chat_completion_message_tool_call import ( ChatCompletionMessageToolCall, ) from openapi_server.models.chat_completion_request_assistant_message_function_call import ( ChatCompletionRequestAssistantMessageFunctionCall, ) +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_assistant_message_function_call.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_assistant_message_function_call.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_assistant_message_function_call.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_assistant_message_function_call.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_function_message.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_function_message.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_function_message.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_function_message.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_message.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_message.py index dc47e6b7..eb3b67e9 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_message.py @@ -20,16 +20,6 @@ from inspect import getfullargspec from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union -from pydantic import ( - BaseModel, - ConfigDict, - Field, - StrictStr, - ValidationError, - field_validator, -) -from typing_extensions import Literal - from openapi_server.models.chat_completion_request_assistant_message import ( ChatCompletionRequestAssistantMessage, ) @@ -45,6 +35,15 @@ from openapi_server.models.chat_completion_request_user_message import ( ChatCompletionRequestUserMessage, ) +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictStr, + ValidationError, + field_validator, +) +from typing_extensions import Literal try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_message_content_part.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_message_content_part.py index c1f4025d..688fe2fb 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_message_content_part.py @@ -20,6 +20,12 @@ from inspect import getfullargspec from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from openapi_server.models.chat_completion_request_message_content_part_image import ( + ChatCompletionRequestMessageContentPartImage, +) +from openapi_server.models.chat_completion_request_message_content_part_text import ( + ChatCompletionRequestMessageContentPartText, +) from pydantic import ( BaseModel, ConfigDict, @@ -30,13 +36,6 @@ ) from typing_extensions import Literal -from openapi_server.models.chat_completion_request_message_content_part_image import ( - ChatCompletionRequestMessageContentPartImage, -) -from openapi_server.models.chat_completion_request_message_content_part_text import ( - ChatCompletionRequestMessageContentPartText, -) - try: from typing import Self except ImportError: diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image.py index a0de7cc4..d4ba08d2 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image.py @@ -19,11 +19,10 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List -from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator - from openapi_server.models.chat_completion_request_message_content_part_image_image_url import ( ChatCompletionRequestMessageContentPartImageImageUrl, ) +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image_image_url.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image_image_url.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image_image_url.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_image_image_url.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_text.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_text.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_text.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_message_content_part_text.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_system_message.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_system_message.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_system_message.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_system_message.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_tool_message.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_tool_message.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_tool_message.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_tool_message.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_user_message.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_user_message.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_user_message.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_user_message.py index 0ae1d3c2..718345b6 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_user_message.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_user_message.py @@ -19,11 +19,10 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional -from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator - from openapi_server.models.chat_completion_request_user_message_content import ( ChatCompletionRequestUserMessageContent, ) +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_user_message_content.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_user_message_content.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_user_message_content.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_user_message_content.py index 00611944..51fec222 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_request_user_message_content.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_request_user_message_content.py @@ -20,6 +20,9 @@ from inspect import getfullargspec from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from openapi_server.models.chat_completion_request_message_content_part import ( + ChatCompletionRequestMessageContentPart, +) from pydantic import ( BaseModel, ConfigDict, @@ -30,10 +33,6 @@ ) from typing_extensions import Annotated, Literal -from openapi_server.models.chat_completion_request_message_content_part import ( - ChatCompletionRequestMessageContentPart, -) - try: from typing import Self except ImportError: diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_response_message.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_response_message.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_response_message.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_response_message.py index 586bc2c8..51ed452c 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_response_message.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_response_message.py @@ -19,14 +19,13 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional -from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator - from openapi_server.models.chat_completion_message_tool_call import ( ChatCompletionMessageToolCall, ) from openapi_server.models.chat_completion_request_assistant_message_function_call import ( ChatCompletionRequestAssistantMessageFunctionCall, ) +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_role.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_role.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_role.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_role.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_stream_response_delta.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_stream_response_delta.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_stream_response_delta.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_stream_response_delta.py index 3b1ef2fb..dc9d676f 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_stream_response_delta.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_stream_response_delta.py @@ -19,14 +19,13 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional -from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator - from openapi_server.models.chat_completion_message_tool_call_chunk import ( ChatCompletionMessageToolCallChunk, ) from openapi_server.models.chat_completion_stream_response_delta_function_call import ( ChatCompletionStreamResponseDeltaFunctionCall, ) +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_stream_response_delta_function_call.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_stream_response_delta_function_call.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_stream_response_delta_function_call.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_stream_response_delta_function_call.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_token_logprob.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_token_logprob.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_token_logprob.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_token_logprob.py index 690b933e..127da7da 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_token_logprob.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_token_logprob.py @@ -19,11 +19,10 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional, Union -from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt, StrictStr - from openapi_server.models.chat_completion_token_logprob_top_logprobs_inner import ( ChatCompletionTokenLogprobTopLogprobsInner, ) +from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt, StrictStr try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_token_logprob_top_logprobs_inner.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_token_logprob_top_logprobs_inner.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_token_logprob_top_logprobs_inner.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_token_logprob_top_logprobs_inner.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_tool.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_tool.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_tool.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_tool.py index 430aa31e..e0d90207 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_tool.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_tool.py @@ -19,9 +19,8 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List -from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator - from openapi_server.models.function_object import FunctionObject +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_tool_choice_option.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_tool_choice_option.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_tool_choice_option.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_tool_choice_option.py index d6c41b50..bcc00db7 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/chat_completion_tool_choice_option.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/chat_completion_tool_choice_option.py @@ -20,6 +20,9 @@ from inspect import getfullargspec from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from openapi_server.models.chat_completion_named_tool_choice import ( + ChatCompletionNamedToolChoice, +) from pydantic import ( BaseModel, ConfigDict, @@ -30,10 +33,6 @@ ) from typing_extensions import Literal -from openapi_server.models.chat_completion_named_tool_choice import ( - ChatCompletionNamedToolChoice, -) - try: from typing import Self except ImportError: diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/completion_usage.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/completion_usage.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/completion_usage.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/completion_usage.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_function_response.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_function_response.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_function_response.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_function_response.py index b3c6049e..ed57852d 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_function_response.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_function_response.py @@ -19,12 +19,11 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional -from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator - from openapi_server.models.completion_usage import CompletionUsage from openapi_server.models.create_chat_completion_function_response_choices_inner import ( CreateChatCompletionFunctionResponseChoicesInner, ) +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_function_response_choices_inner.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_function_response_choices_inner.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_function_response_choices_inner.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_function_response_choices_inner.py index 0dd07942..7ab3b531 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_function_response_choices_inner.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_function_response_choices_inner.py @@ -19,11 +19,10 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List -from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator - from openapi_server.models.chat_completion_response_message import ( ChatCompletionResponseMessage, ) +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_request.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_request.py index 06bdab48..d6043643 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_request.py @@ -19,9 +19,6 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional, Union -from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, StrictStr -from typing_extensions import Annotated - from openapi_server.models.chat_completion_functions import ChatCompletionFunctions from openapi_server.models.chat_completion_request_message import ( ChatCompletionRequestMessage, @@ -42,6 +39,8 @@ from openapi_server.models.create_chat_completion_request_stop import ( CreateChatCompletionRequestStop, ) +from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, StrictStr +from typing_extensions import Annotated try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_function_call.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_request_function_call.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_function_call.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_request_function_call.py index 4d92a7c4..0f5d30ae 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_function_call.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_request_function_call.py @@ -20,6 +20,9 @@ from inspect import getfullargspec from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from openapi_server.models.chat_completion_function_call_option import ( + ChatCompletionFunctionCallOption, +) from pydantic import ( BaseModel, ConfigDict, @@ -30,10 +33,6 @@ ) from typing_extensions import Literal -from openapi_server.models.chat_completion_function_call_option import ( - ChatCompletionFunctionCallOption, -) - try: from typing import Self except ImportError: diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_model.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_request_model.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_model.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_request_model.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_response_format.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_request_response_format.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_response_format.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_request_response_format.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_stop.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_request_stop.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_request_stop.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_request_stop.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_response.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_response.py index bc43e294..2e98bd81 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_response.py @@ -19,12 +19,11 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional -from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator - from openapi_server.models.completion_usage import CompletionUsage from openapi_server.models.create_chat_completion_response_choices_inner import ( CreateChatCompletionResponseChoicesInner, ) +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner.py index fd1e7ea9..91689a33 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner.py @@ -19,14 +19,13 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional -from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator - from openapi_server.models.chat_completion_response_message import ( ChatCompletionResponseMessage, ) from openapi_server.models.create_chat_completion_response_choices_inner_logprobs import ( CreateChatCompletionResponseChoicesInnerLogprobs, ) +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner_logprobs.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner_logprobs.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner_logprobs.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner_logprobs.py index 86f46094..696c4205 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner_logprobs.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_response_choices_inner_logprobs.py @@ -19,11 +19,10 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional -from pydantic import BaseModel, ConfigDict, Field - from openapi_server.models.chat_completion_token_logprob import ( ChatCompletionTokenLogprob, ) +from pydantic import BaseModel, ConfigDict, Field try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_stream_response.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_stream_response.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_stream_response.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_stream_response.py index bd464c73..1331aeae 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_stream_response.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_stream_response.py @@ -19,11 +19,10 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional -from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator - from openapi_server.models.create_chat_completion_stream_response_choices_inner import ( CreateChatCompletionStreamResponseChoicesInner, ) +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_stream_response_choices_inner.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_stream_response_choices_inner.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_stream_response_choices_inner.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_stream_response_choices_inner.py index 910caa2f..88190af1 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_chat_completion_stream_response_choices_inner.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_chat_completion_stream_response_choices_inner.py @@ -19,14 +19,13 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional -from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator - from openapi_server.models.chat_completion_stream_response_delta import ( ChatCompletionStreamResponseDelta, ) from openapi_server.models.create_chat_completion_response_choices_inner_logprobs import ( CreateChatCompletionResponseChoicesInnerLogprobs, ) +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_request.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_request.py index bfcc91e0..17638e08 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_request.py @@ -19,9 +19,6 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional, Union -from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, StrictStr -from typing_extensions import Annotated - from openapi_server.models.create_completion_request_model import ( CreateCompletionRequestModel, ) @@ -31,6 +28,8 @@ from openapi_server.models.create_completion_request_stop import ( CreateCompletionRequestStop, ) +from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, StrictStr +from typing_extensions import Annotated try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_model.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_request_model.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_model.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_request_model.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_prompt.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_request_prompt.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_prompt.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_request_prompt.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_stop.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_request_stop.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_request_stop.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_request_stop.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_response.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_response.py index 84d00be6..8e54d48e 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_response.py @@ -19,12 +19,11 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional -from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator - from openapi_server.models.completion_usage import CompletionUsage from openapi_server.models.create_completion_response_choices_inner import ( CreateCompletionResponseChoicesInner, ) +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response_choices_inner.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_response_choices_inner.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response_choices_inner.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_response_choices_inner.py index 482dfa8e..a37ca952 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response_choices_inner.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_response_choices_inner.py @@ -19,11 +19,10 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List, Optional -from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator - from openapi_server.models.create_completion_response_choices_inner_logprobs import ( CreateCompletionResponseChoicesInnerLogprobs, ) +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response_choices_inner_logprobs.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_response_choices_inner_logprobs.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/create_completion_response_choices_inner_logprobs.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/create_completion_response_choices_inner_logprobs.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/delete_model_response.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/delete_model_response.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/delete_model_response.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/delete_model_response.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/done_event.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/done_event.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/done_event.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/done_event.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/error.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/error.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error_event.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/error_event.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error_event.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/error_event.py index d9ea8e1c..4ff734eb 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error_event.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/error_event.py @@ -19,9 +19,8 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List -from pydantic import BaseModel, ConfigDict, StrictStr, field_validator - from openapi_server.models.error import Error +from pydantic import BaseModel, ConfigDict, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error_response.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/error_response.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error_response.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/error_response.py index f265cd28..4b7c5463 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/error_response.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/error_response.py @@ -19,9 +19,8 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List -from pydantic import BaseModel, ConfigDict - from openapi_server.models.error import Error +from pydantic import BaseModel, ConfigDict try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/extra_models.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/extra_models.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/extra_models.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/extra_models.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/function_object.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/function_object.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/function_object.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/function_object.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/list_models_response.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/list_models_response.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/list_models_response.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/list_models_response.py index e02d2013..644d3b15 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/list_models_response.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/list_models_response.py @@ -19,9 +19,8 @@ import re # noqa: F401 from typing import Any, ClassVar, Dict, List -from pydantic import BaseModel, ConfigDict, StrictStr, field_validator - from openapi_server.models.model import Model +from pydantic import BaseModel, ConfigDict, StrictStr, field_validator try: from typing import Self diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/model.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/model.py similarity index 100% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/models/model.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/models/model.py diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/security_api.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/security_api.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/security_api.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/security_api.py index 368f87b8..661f4f40 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/src/openapi_server/security_api.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/src/openapi_server/security_api.py @@ -19,7 +19,6 @@ APIKeyHeader, APIKeyQuery, ) - from openapi_server.models.extra_models import TokenModel bearer_auth = HTTPBearer() diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/conftest.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/tests/conftest.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/conftest.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/tests/conftest.py index cbde552c..c021951d 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/conftest.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/tests/conftest.py @@ -1,7 +1,6 @@ import pytest from fastapi import FastAPI from fastapi.testclient import TestClient - from openapi_server.main import app as application diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_chat_api.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/tests/test_chat_api.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_chat_api.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/tests/test_chat_api.py index c4e48701..cb0eadfe 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_chat_api.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/tests/test_chat_api.py @@ -1,7 +1,6 @@ # coding: utf-8 from fastapi.testclient import TestClient - from openapi_server.models.create_chat_completion_request import ( # noqa: F401 CreateChatCompletionRequest, ) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_completions_api.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/tests/test_completions_api.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_completions_api.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/tests/test_completions_api.py index aea23486..ee52937b 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_completions_api.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/tests/test_completions_api.py @@ -1,7 +1,6 @@ # coding: utf-8 from fastapi.testclient import TestClient - from openapi_server.models.create_completion_request import ( # noqa: F401 CreateCompletionRequest, ) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_models_api.py b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/tests/test_models_api.py similarity index 99% rename from Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_models_api.py rename to Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/tests/test_models_api.py index 47bc763a..15419342 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/openai-server/tests/test_models_api.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/openapi-generator/openai-server/tests/test_models_api.py @@ -1,7 +1,6 @@ # coding: utf-8 from fastapi.testclient import TestClient - from openapi_server.models.delete_model_response import ( # noqa: F401 DeleteModelResponse, ) From 058a6cbc55f63d635c68ba7df9008b1d9f863274 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 5 May 2024 11:43:14 -0700 Subject: [PATCH 17/62] initial readme --- .../examples/fastapi/README.md | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 Triton_Inference_Server_Python_API/examples/fastapi/README.md diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/README.md b/Triton_Inference_Server_Python_API/examples/fastapi/README.md new file mode 100644 index 00000000..8bded82d --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/README.md @@ -0,0 +1,36 @@ + + +# Triton Inference Server Fast API / Open API / Open AI Example + +## Generating the Fast API server using fastapi-codegen + +## Generating the Fast API server using openapi-code-generator + + + From 0414fa124db2ba755e3af105374dbf79cecb5360 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 5 May 2024 11:47:06 -0700 Subject: [PATCH 18/62] updates --- .../examples/fastapi/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/README.md b/Triton_Inference_Server_Python_API/examples/fastapi/README.md index 8bded82d..891608fb 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/README.md +++ b/Triton_Inference_Server_Python_API/examples/fastapi/README.md @@ -28,8 +28,18 @@ # Triton Inference Server Fast API / Open API / Open AI Example +## Open AI API Specification + +We use +https://raw.githubusercontent.com/openai/openai-openapi/25d9dacc86a94df1db98725fe87494564317cafa/openapi.yaml +as the base specification. + +As this tutorial only covers LLM applications we use a trimmed specficiation (api-spec/openai_trimmed.yml). + ## Generating the Fast API server using fastapi-codegen +### Modifications + ## Generating the Fast API server using openapi-code-generator From f96e550a05e56e935e01fa29c31083ec1028a910 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 5 May 2024 12:22:23 -0700 Subject: [PATCH 19/62] update to make repl installation optional --- .../deps/requirements_python_repl.txt | 3 +++ Triton_Inference_Server_Python_API/docker/Dockerfile | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 Triton_Inference_Server_Python_API/deps/requirements_python_repl.txt diff --git a/Triton_Inference_Server_Python_API/deps/requirements_python_repl.txt b/Triton_Inference_Server_Python_API/deps/requirements_python_repl.txt new file mode 100644 index 00000000..95ba3438 --- /dev/null +++ b/Triton_Inference_Server_Python_API/deps/requirements_python_repl.txt @@ -0,0 +1,3 @@ +bpython +ipython +ptpython diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index fc1be5b7..7cfa0d73 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -38,7 +38,9 @@ COPY ./deps/requirements.txt /tmp/requirements.txt COPY ./deps/requirements_trt_llm.txt /tmp/requirements_trt_llm.txt -COPY ./deps/requirements_trt_llm.txt /tmp/requirements_vllm.txt +COPY ./deps/requirements_vllm.txt /tmp/requirements_vllm.txt + +COPY ./deps/requirements_python_repl.txt /tmp/requirements_python_repl.txt COPY ./deps/tritonserver-2.46.0.dev0-py3-none-any.whl /tmp/tritonserver-2.46.0.dev0-py3-none-any.whl @@ -61,11 +63,9 @@ ARG GENAI_PERF_TAG=r24.04 RUN pip install "git+https://github.com/triton-inference-server/client.git@${GENAI_PERF_TAG}#subdirectory=src/c++/perf_analyzer/genai-perf" -RUN pip install bpython - -RUN pip install ipython +ARG INCLUDE_PYTHON_REPL -RUN pip install ptpython +RUN if [[ "$INCLUDE_PYTHON_REPL" != "" ]] ; then pip install --timeout=2000 -r /tmp/python_repl.txt ; fi ARG FRAMEWORK=DIFFUSION From 32b9a7a67e739da8b87adfe9437993782d470e31 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 5 May 2024 13:15:00 -0700 Subject: [PATCH 20/62] updating to 24.04 base image for stable diffusion --- Popular_Models_Guide/StableDiffusion/build.sh | 4 ++-- Popular_Models_Guide/StableDiffusion/run.sh | 2 +- Triton_Inference_Server_Python_API/build.sh | 8 ++++---- Triton_Inference_Server_Python_API/docker/Dockerfile | 4 ++-- Triton_Inference_Server_Python_API/run.sh | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Popular_Models_Guide/StableDiffusion/build.sh b/Popular_Models_Guide/StableDiffusion/build.sh index b2507d77..d6b3a374 100755 --- a/Popular_Models_Guide/StableDiffusion/build.sh +++ b/Popular_Models_Guide/StableDiffusion/build.sh @@ -39,7 +39,7 @@ DOCKERFILE=${SOURCE_DIR}/docker/Dockerfile # Base Images BASE_IMAGE=nvcr.io/nvidia/tritonserver -BASE_IMAGE_TAG_DIFFUSION=24.01-py3 +BASE_IMAGE_TAG_DIFFUSION=24.04-py3 get_options() { while :; do @@ -141,7 +141,7 @@ get_options() { fi if [ -z "$TAG" ]; then - TAG="tritonserver:r24.01" + TAG="tritonserver:r24.04" if [[ $FRAMEWORK == "DIFFUSION" ]]; then TAG+="-diffusion" diff --git a/Popular_Models_Guide/StableDiffusion/run.sh b/Popular_Models_Guide/StableDiffusion/run.sh index be47a600..c32a560c 100755 --- a/Popular_Models_Guide/StableDiffusion/run.sh +++ b/Popular_Models_Guide/StableDiffusion/run.sh @@ -99,7 +99,7 @@ get_options() { fi if [ -z "$IMAGE" ]; then - IMAGE="tritonserver:r24.01" + IMAGE="tritonserver:r24.04" if [[ $FRAMEWORK == "DIFFUSION" ]]; then IMAGE+="-diffusion" diff --git a/Triton_Inference_Server_Python_API/build.sh b/Triton_Inference_Server_Python_API/build.sh index 05ea4a1e..ad507515 100755 --- a/Triton_Inference_Server_Python_API/build.sh +++ b/Triton_Inference_Server_Python_API/build.sh @@ -40,7 +40,7 @@ DOCKERFILE=${SOURCE_DIR}/docker/Dockerfile # Base Images BASE_IMAGE=nvcr.io/nvidia/tritonserver BASE_IMAGE_TAG_IDENTITY=24.04-py3 -BASE_IMAGE_TAG_DIFFUSION=24.01-py3 +BASE_IMAGE_TAG_DIFFUSION=24.04-py3 BASE_IMAGE_TAG_TRT_LLM=24.04-trtllm-python-py3 BASE_IMAGE_TAG_VLLM=24.04-vllm-python-py3 @@ -140,14 +140,14 @@ get_options() { fi if [ -z "$TAG" ]; then - TAG="triton-python-api:r24.03" + TAG="triton-python-api:r24.04" if [[ $FRAMEWORK == "TRT_LLM" ]]; then TAG+="-trt-llm" fi if [[ $FRAMEWORK == "DIFFUSION" ]]; then - TAG="triton-python-api:r24.01-diffusion" + TAG+="-diffusion" fi if [[ $FRAMEWORK == "VLLM" ]]; then @@ -193,7 +193,7 @@ get_options "$@" if [[ $FRAMEWORK == DIFFUSION ]]; then BASE_IMAGE="tritonserver" - BASE_IMAGE_TAG="r24.01-diffusion" + BASE_IMAGE_TAG="r24.04-diffusion" fi # BUILD RUN TIME IMAGE diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index 7cfa0d73..cc8e64bd 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -25,9 +25,9 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver -ARG BASE_IMAGE_TAG=24.03-py3 +ARG BASE_IMAGE_TAG=24.04-py3 ARG FRAMEWORK=DIFFUSION -ARG GENAI_PERF_TAG=r24.03 +ARG GENAI_PERF_TAG=r24.04 ARG TRITON_CLI_TAG="0.0.6" FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} as triton-python-api diff --git a/Triton_Inference_Server_Python_API/run.sh b/Triton_Inference_Server_Python_API/run.sh index f48b91aa..fa67e971 100755 --- a/Triton_Inference_Server_Python_API/run.sh +++ b/Triton_Inference_Server_Python_API/run.sh @@ -101,14 +101,14 @@ get_options() { fi if [ -z "$IMAGE" ]; then - IMAGE="triton-python-api:r24.03" + IMAGE="triton-python-api:r24.04" if [[ $FRAMEWORK == "TRT_LLM" ]]; then IMAGE+="-trt-llm" fi if [[ $FRAMEWORK == "DIFFUSION" ]]; then - IMAGE="triton-python-api:r24.01-diffusion" + IMAGE+="-diffusion" fi if [[ $FRAMEWORK == "VLLM" ]]; then From c78767d23b0993a6e7cdec6bfa63420f2f5daafe Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 5 May 2024 14:43:52 -0700 Subject: [PATCH 21/62] update with script for fastapi code gen --- .../docker/Dockerfile | 2 +- .../fastapi/scripts/fastapi-codegen.sh | 33 +++++++++++++++++++ Triton_Inference_Server_Python_API/run.sh | 2 +- 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100755 Triton_Inference_Server_Python_API/examples/fastapi/scripts/fastapi-codegen.sh diff --git a/Triton_Inference_Server_Python_API/docker/Dockerfile b/Triton_Inference_Server_Python_API/docker/Dockerfile index cc8e64bd..7566c804 100644 --- a/Triton_Inference_Server_Python_API/docker/Dockerfile +++ b/Triton_Inference_Server_Python_API/docker/Dockerfile @@ -65,7 +65,7 @@ RUN pip install "git+https://github.com/triton-inference-server/client.git@${GEN ARG INCLUDE_PYTHON_REPL -RUN if [[ "$INCLUDE_PYTHON_REPL" != "" ]] ; then pip install --timeout=2000 -r /tmp/python_repl.txt ; fi +RUN if [[ "$INCLUDE_PYTHON_REPL" != "" ]] ; then pip install --timeout=2000 -r /tmp/requirements_python_repl.txt ; fi ARG FRAMEWORK=DIFFUSION diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/scripts/fastapi-codegen.sh b/Triton_Inference_Server_Python_API/examples/fastapi/scripts/fastapi-codegen.sh new file mode 100755 index 00000000..22cb7f4d --- /dev/null +++ b/Triton_Inference_Server_Python_API/examples/fastapi/scripts/fastapi-codegen.sh @@ -0,0 +1,33 @@ +#!/bin/bash -e +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +SOURCE_DIR=$(dirname "$(readlink -f "$0")") + +BASE_IMAGE=nvcr.io/nvidia/tritonserver +BASE_IMAGE_TAG=24.04-py3 + +docker run --rm -it --workdir $PWD -v $PWD:$PWD $BASE_IMAGE:$BASE_IMAGE_TAG /bin/bash -c "pip3 install fastapi-code-generator;fastapi-codegen $@" diff --git a/Triton_Inference_Server_Python_API/run.sh b/Triton_Inference_Server_Python_API/run.sh index fa67e971..efa95797 100755 --- a/Triton_Inference_Server_Python_API/run.sh +++ b/Triton_Inference_Server_Python_API/run.sh @@ -142,7 +142,7 @@ fi $RUN_PREFIX mkdir -p backend/diffusion -$RUN_PREFIX docker run --gpus all -it --rm --network host --shm-size=10G --ulimit memlock=-1 --ulimit stack=67108864 -eHF_TOKEN -eGITHUB_TOKEN -eAWS_DEFAULT_REGION -eAWS_ACCESS_KEY_ID -eAWS_SECRET_ACCESS_KEY -eS3_BUCKET_URL -v ${SOURCE_DIR}:/workspace -v${SOURCE_DIR}/.cache/huggingface:/root/.cache/huggingface -w /workspace -v${SOURCE_DIR}/backend/diffusion:/opt/tritonserver/backends/diffusion -v/tmp:/tmp -v${SOURCE_DIR}/examples/litellm/triton.py:/usr/local/lib/python3.10/dist-packages/litellm/llms/triton.py $IMAGE +$RUN_PREFIX docker run --gpus all -it --rm --network host --shm-size=10G --ulimit memlock=-1 --ulimit stack=67108864 -eHF_TOKEN -eGITHUB_TOKEN -eAWS_DEFAULT_REGION -eAWS_ACCESS_KEY_ID -eAWS_SECRET_ACCESS_KEY -eS3_BUCKET_URL -v ${SOURCE_DIR}:/workspace -v${SOURCE_DIR}/.cache/huggingface:/root/.cache/huggingface -w /workspace -v${SOURCE_DIR}/backend/diffusion:/opt/tritonserver/backends/diffusion -v/tmp:/tmp $IMAGE { set +x; } 2>/dev/null From 7a061ca267a71da17d96f1c7c3217106206158c8 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 5 May 2024 15:10:26 -0700 Subject: [PATCH 22/62] initial check in for codegen. use this as base for showing steps to modify --- .../examples/fastapi/README.md | 5 + .../examples/fastapi/fastapi-codegen/main.py | 131 +----------------- .../{models.py => openai_protocol_types.py} | 28 ++-- 3 files changed, 25 insertions(+), 139 deletions(-) rename Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/{models.py => openai_protocol_types.py} (98%) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/README.md b/Triton_Inference_Server_Python_API/examples/fastapi/README.md index 891608fb..8e3734a3 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/README.md +++ b/Triton_Inference_Server_Python_API/examples/fastapi/README.md @@ -38,8 +38,13 @@ As this tutorial only covers LLM applications we use a trimmed specficiation (ap ## Generating the Fast API server using fastapi-codegen + +``` +./scripts/fastapi-codegen.sh "-i $PWD/api-spec/openai_trimmed.yml -o $PWD/foo --model-file openai_protocol_types" +``` ### Modifications + ## Generating the Fast API server using openapi-code-generator diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index 2a56e572..6e29bd86 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -1,44 +1,19 @@ # generated by fastapi-codegen: -# filename: openapi_modified.yaml -# timestamp: 2024-05-04T14:14:42+00:00 +# filename: openai_trimmed.yml +# timestamp: 2024-05-05T21:52:36+00:00 from __future__ import annotations -import copy -import time -import uuid -from typing import TypedDict - -import tritonserver from fastapi import FastAPI -from vllm.transformers_utils.tokenizer import get_tokenizer - -triton_server = tritonserver.Server( - model_repository="/workspace/llm-models", log_verbose=6, strict_model_config=False -).start(wait_until_ready=True) - -model = triton_server.model("llama-3-8b-instruct") -model.tokenizer = get_tokenizer(tokenizer_name="meta-llama/Meta-Llama-3-8B-Instruct") - -from models import ( - ChatCompletionResponseMessage, - Choice, - Choice1, +from .openai_protocol_types import ( CreateChatCompletionRequest, CreateChatCompletionResponse, CreateCompletionRequest, CreateCompletionResponse, DeleteModelResponse, - FinishReason, - FinishReason1, ListModelsResponse, - Logprobs, - Logprobs2, Model, - Object1, - Object2, - Role5, ) app = FastAPI( @@ -55,11 +30,6 @@ ) -class ConversationMessage(TypedDict): - role: str - content: str - - @app.post( "/chat/completions", response_model=CreateChatCompletionResponse, tags=["Chat"] ) @@ -69,62 +39,7 @@ def create_chat_completion( """ Creates a model response for the given chat conversation. """ - - conversation = [ - ConversationMessage( - role=str(message.dict()["role"]), content=str(message.dict()["content"]) - ) - for message in body.messages - ] - - prompt = model.tokenizer.apply_chat_template( - conversation=conversation, tokenize=False, add_generation_prompt=False - ) - - exclude_input_in_output = True - - parameters = copy.deepcopy(body.dict()) - if "prompt" in parameters: - del parameters["prompt"] - if "stream" in parameters: - del parameters["stream"] - if "echo" in parameters: - del parameters["echo"] - if "model" in parameters: - del parameters["model"] - if "messages" in parameters: - del parameters["messages"] - - response = list( - model.infer( - inputs={ - "text_input": [prompt], - "stream": [False], - "exclude_input_in_output": [exclude_input_in_output], - }, - parameters=parameters, - ) - )[0] - - return CreateChatCompletionResponse( - id="foo", - choices=[ - Choice1( - finish_reason=FinishReason1.stop, - index=0, - message=ChatCompletionResponseMessage( - content=response.outputs["text_output"].to_string_array()[0], - role=Role5.assistant, - function_call=None, - ), - logprobs=Logprobs2(content=[]), - ) - ], - created=0, - model="foo", - system_fingerprint=None, - object=Object2.chat_completion, - ) + pass @app.post("/completions", response_model=CreateCompletionResponse, tags=["Completions"]) @@ -132,43 +47,7 @@ def create_completion(body: CreateCompletionRequest) -> CreateCompletionResponse """ Creates a completion for the provided prompt and parameters. """ - exclude_input_in_output = True - - if body.echo: - exclude_input_in_output = False - - parameters = copy.deepcopy(body.dict()) - del parameters["prompt"] - del parameters["stream"] - del parameters["echo"] - del parameters["model"] - - response = list( - model.infer( - inputs={ - "text_input": [body.prompt], - "stream": [False], - "exclude_input_in_output": [exclude_input_in_output], - }, - parameters=parameters, - ) - )[0] - - choice = Choice( - finish_reason=FinishReason.stop, - index=0, - logprobs=Logprobs(), - text=response.outputs["text_output"].to_string_array()[0], - ) - - return CreateCompletionResponse( - id=f"cmpl-{uuid.uuid1()}", - created=int(time.time()), - model=model.name, - choices=[choice], - system_fingerprint=None, - object=Object1.text_completion, - ) + pass @app.get("/models", response_model=ListModelsResponse, tags=["Models"]) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/models.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py similarity index 98% rename from Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/models.py rename to Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py index 0051af83..40056837 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/models.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py @@ -1,13 +1,13 @@ # generated by fastapi-codegen: -# filename: openai-openapi/openapi_modified.yaml -# timestamp: 2024-05-04T14:14:42+00:00 +# filename: api-spec/openai_trimmed.yml +# timestamp: 2024-05-05T21:52:36+00:00 from __future__ import annotations from enum import Enum from typing import Any, Dict, List, Optional, Union -from pydantic import AnyUrl, BaseModel, Extra, Field, RootModel, confloat, conint +from pydantic import AnyUrl, BaseModel, Extra, Field, confloat, conint class Error(BaseModel): @@ -37,8 +37,8 @@ class Model1(Enum): babbage_002 = "babbage-002" -class PromptItem(RootModel): - root: List[Any] +class PromptItem(BaseModel): + __root__: List[Any] class CreateCompletionRequest(BaseModel): @@ -592,8 +592,8 @@ class CreateCompletionResponse(BaseModel): usage: Optional[CompletionUsage] = None -class ChatCompletionRequestMessageContentPart(RootModel): - root: Union[ +class ChatCompletionRequestMessageContentPart(BaseModel): + __root__: Union[ ChatCompletionRequestMessageContentPartText, ChatCompletionRequestMessageContentPartImage, ] @@ -620,15 +620,17 @@ class ChatCompletionTool(BaseModel): function: FunctionObject -class ChatCompletionToolChoiceOption(RootModel): - root: Union[ChatCompletionToolChoiceOption1, ChatCompletionNamedToolChoice] = Field( +class ChatCompletionToolChoiceOption(BaseModel): + __root__: Union[ + ChatCompletionToolChoiceOption1, ChatCompletionNamedToolChoice + ] = Field( ..., description='Controls which (if any) tool is called by the model.\n`none` means the model will not call any tool and instead generates a message.\n`auto` means the model can pick between generating a message or calling one or more tools.\n`required` means the model must call one or more tools.\nSpecifying a particular tool via `{"type": "function", "function": {"name": "my_function"}}` forces the model to call that tool.\n\n`none` is the default when no tools are present. `auto` is the default if tools are present.\n', ) -class ChatCompletionMessageToolCalls(RootModel): - root: List[ChatCompletionMessageToolCall] = Field( +class ChatCompletionMessageToolCalls(BaseModel): + __root__: List[ChatCompletionMessageToolCall] = Field( ..., description="The tool calls generated by the model, such as function calls.", ) @@ -730,8 +732,8 @@ class ChatCompletionRequestAssistantMessage(BaseModel): ) -class ChatCompletionRequestMessage(RootModel): - root: Union[ +class ChatCompletionRequestMessage(BaseModel): + __root__: Union[ ChatCompletionRequestSystemMessage, ChatCompletionRequestUserMessage, ChatCompletionRequestAssistantMessage, From 9b984797d0d5378656c76cc34a0c0707090a2ccb Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 5 May 2024 16:45:24 -0700 Subject: [PATCH 23/62] added in models api --- .../examples/fastapi/README.md | 47 ++++++++++++++++++- .../examples/fastapi/fastapi-codegen/main.py | 25 ++++++++-- .../fastapi-codegen/openai_protocol_types.py | 8 ++++ 3 files changed, 75 insertions(+), 5 deletions(-) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/README.md b/Triton_Inference_Server_Python_API/examples/fastapi/README.md index 8e3734a3..006e54cd 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/README.md +++ b/Triton_Inference_Server_Python_API/examples/fastapi/README.md @@ -28,6 +28,17 @@ # Triton Inference Server Fast API / Open API / Open AI Example +## Build Image +``` +../../build.sh --framework vllm --build-arg TRITON_CLI_TAG=rmccormick-trtllm-0.9 +``` + +## Import Model +``` +triton remove -m all --model-repository llm-models +triton import -m llama-3-8b-instruct --backend vllm --model-repository llm-models +``` + ## Open AI API Specification We use @@ -38,14 +49,46 @@ As this tutorial only covers LLM applications we use a trimmed specficiation (ap ## Generating the Fast API server using fastapi-codegen - ``` -./scripts/fastapi-codegen.sh "-i $PWD/api-spec/openai_trimmed.yml -o $PWD/foo --model-file openai_protocol_types" +./scripts/fastapi-codegen.sh "-i api-spec/openai_trimmed.yml -o fastapi-codegen --model-file openai_protocol_types" ``` + ### Modifications +1. Remove relative import + +Before: + +``` +from .openapi_protocol_types +``` + +After: +``` +from openapi_protocol_types +``` + ## Generating the Fast API server using openapi-code-generator +## curl examples + +### Models +``` +curl -s http://localhost:8000/models | jq . +``` + +``` +{ + "object": "list", + "data": [ + { + "id": "llama-3-8b-instruct", + "created": 1714952401, + "object": "model", + "owned_by": "ACME" + } + ] +``` diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index 6e29bd86..5c1d6a62 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -4,9 +4,10 @@ from __future__ import annotations -from fastapi import FastAPI +import time -from .openai_protocol_types import ( +from fastapi import FastAPI +from openai_protocol_types import ( CreateChatCompletionRequest, CreateChatCompletionResponse, CreateCompletionRequest, @@ -14,8 +15,26 @@ DeleteModelResponse, ListModelsResponse, Model, + ObjectType, ) +owned_by = "ACME" + +model_list = [ + Model( + id="llama-3-8b-instruct", + created=int(time.time()), + object=ObjectType.model, + owned_by=owned_by, + ) +] + +import tritonserver + +server = tritonserver.Server( + model_repository="/workspace/llm-models", log_verbose=6, strict_model_config=False +).start(wait_until_ready=True) + app = FastAPI( title="OpenAI API", description="The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details.", @@ -55,7 +74,7 @@ def list_models() -> ListModelsResponse: """ Lists the currently available models, and provides basic information about each one such as the owner and availability. """ - pass + return ListModelsResponse(object=ObjectType.list, data=model_list) @app.get("/models/{model}", response_model=Model, tags=["Models"]) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py index 40056837..d1319175 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py @@ -830,3 +830,11 @@ class CreateChatCompletionRequest(BaseModel): max_items=128, min_items=1, ) + + +# Additional Aliases for Convenience + + +class ObjectType: + model = Object5.model + list = Object.list From 5c9acd2c970ba4243a38c1fef2b69573f0793bcf Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 5 May 2024 16:56:40 -0700 Subject: [PATCH 24/62] updated with model info call --- .../examples/fastapi/README.md | 16 ++++++++++++++++ .../examples/fastapi/fastapi-codegen/main.py | 16 ++++++++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/README.md b/Triton_Inference_Server_Python_API/examples/fastapi/README.md index 006e54cd..265785a8 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/README.md +++ b/Triton_Inference_Server_Python_API/examples/fastapi/README.md @@ -76,6 +76,8 @@ from openapi_protocol_types ### Models +#### List + ``` curl -s http://localhost:8000/models | jq . ``` @@ -92,3 +94,17 @@ curl -s http://localhost:8000/models | jq . } ] ``` +#### Retrieve Model Info + +``` +curl -s http://localhost:8000/models/llama-3-8b-instruct | jq . +``` + +``` +{ + "id": "llama-3-8b-instruct", + "created": 1714953302, + "object": "model", + "owned_by": "ACME" +} +``` diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index 5c1d6a62..1262e561 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -6,7 +6,7 @@ import time -from fastapi import FastAPI +from fastapi import FastAPI, HTTPException from openai_protocol_types import ( CreateChatCompletionRequest, CreateChatCompletionResponse, @@ -20,14 +20,14 @@ owned_by = "ACME" -model_list = [ - Model( +model_map = { + "llama-3-8b-instruct": Model( id="llama-3-8b-instruct", created=int(time.time()), object=ObjectType.model, owned_by=owned_by, ) -] +} import tritonserver @@ -74,7 +74,7 @@ def list_models() -> ListModelsResponse: """ Lists the currently available models, and provides basic information about each one such as the owner and availability. """ - return ListModelsResponse(object=ObjectType.list, data=model_list) + return ListModelsResponse(object=ObjectType.list, data=list(model_map.values())) @app.get("/models/{model}", response_model=Model, tags=["Models"]) @@ -82,7 +82,11 @@ def retrieve_model(model: str) -> Model: """ Retrieves a model instance, providing basic information about the model such as the owner and permissioning. """ - pass + + if model in model_map: + return model_map[model] + + raise HTTPException(status_code=404, detail=f"Unknown model: {model}") @app.delete("/models/{model}", response_model=DeleteModelResponse, tags=["Models"]) From b051b5a4997b86ad61d52ce40191e1f61a968f01 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 5 May 2024 17:04:56 -0700 Subject: [PATCH 25/62] remove delete --- ...{opanai_trimmed.yml => openai_trimmed.yml} | 52 ------------------- 1 file changed, 52 deletions(-) rename Triton_Inference_Server_Python_API/examples/fastapi/api-spec/{opanai_trimmed.yml => openai_trimmed.yml} (98%) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/api-spec/opanai_trimmed.yml b/Triton_Inference_Server_Python_API/examples/fastapi/api-spec/openai_trimmed.yml similarity index 98% rename from Triton_Inference_Server_Python_API/examples/fastapi/api-spec/opanai_trimmed.yml rename to Triton_Inference_Server_Python_API/examples/fastapi/api-spec/openai_trimmed.yml index 2cfa92ae..370f8ae9 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/api-spec/opanai_trimmed.yml +++ b/Triton_Inference_Server_Python_API/examples/fastapi/api-spec/openai_trimmed.yml @@ -923,58 +923,6 @@ paths: "created": 1686935002, "owned_by": "openai" } - delete: - operationId: deleteModel - tags: - - Models - summary: Delete a fine-tuned model. You must have the Owner role in your organization to delete a model. - parameters: - - in: path - name: model - required: true - schema: - type: string - example: ft:gpt-3.5-turbo:acemeco:suffix:abc123 - description: The model to delete - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: "#/components/schemas/DeleteModelResponse" - x-oaiMeta: - name: Delete a fine-tuned model - group: models - returns: Deletion status. - examples: - request: - curl: | - curl https://api.openai.com/v1/models/ft:gpt-3.5-turbo:acemeco:suffix:abc123 \ - -X DELETE \ - -H "Authorization: Bearer $OPENAI_API_KEY" - python: | - from openai import OpenAI - client = OpenAI() - - client.models.delete("ft:gpt-3.5-turbo:acemeco:suffix:abc123") - node.js: |- - import OpenAI from "openai"; - - const openai = new OpenAI(); - - async function main() { - const model = await openai.models.del("ft:gpt-3.5-turbo:acemeco:suffix:abc123"); - - console.log(model); - } - main(); - response: | - { - "id": "ft:gpt-3.5-turbo:acemeco:suffix:abc123", - "object": "model", - "deleted": true - } components: securitySchemes: From 67b1ab602c802c674e08dfe4a185bec7231ee39b Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 5 May 2024 17:21:52 -0700 Subject: [PATCH 26/62] remove delete --- .../examples/fastapi/fastapi-codegen/main.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index 1262e561..b4e34fd3 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -87,11 +87,3 @@ def retrieve_model(model: str) -> Model: return model_map[model] raise HTTPException(status_code=404, detail=f"Unknown model: {model}") - - -@app.delete("/models/{model}", response_model=DeleteModelResponse, tags=["Models"]) -def delete_model(model: str) -> DeleteModelResponse: - """ - Delete a fine-tuned model. You must have the Owner role in your organization to delete a model. - """ - pass From e9521957616197e4a100c45a654c5569a9fd0dfc Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 5 May 2024 23:20:36 -0700 Subject: [PATCH 27/62] adding support for completions api --- .../examples/fastapi/README.md | 4 + .../examples/fastapi/fastapi-codegen/main.py | 104 +++++++++++++++++- .../fastapi-codegen/openai_protocol_types.py | 5 +- 3 files changed, 106 insertions(+), 7 deletions(-) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/README.md b/Triton_Inference_Server_Python_API/examples/fastapi/README.md index 265785a8..8aa5c5e5 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/README.md +++ b/Triton_Inference_Server_Python_API/examples/fastapi/README.md @@ -108,3 +108,7 @@ curl -s http://localhost:8000/models/llama-3-8b-instruct | jq . "owned_by": "ACME" } ``` + +## Comparison + +python3 -m vllm.entrypoints.openai.api_server --model "meta-llama/Meta-Llama-3-8B-Instruct" diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index b4e34fd3..295f67f7 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -5,14 +5,18 @@ from __future__ import annotations import time +import uuid -from fastapi import FastAPI, HTTPException +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import StreamingResponse from openai_protocol_types import ( + Choice, CreateChatCompletionRequest, CreateChatCompletionResponse, CreateCompletionRequest, CreateCompletionResponse, DeleteModelResponse, + FinishReason, ListModelsResponse, Model, ObjectType, @@ -32,9 +36,15 @@ import tritonserver server = tritonserver.Server( - model_repository="/workspace/llm-models", log_verbose=6, strict_model_config=False + model_repository="/workspace/llm-models", + log_verbose=6, + strict_model_config=False, + model_control_mode=tritonserver.ModelControlMode.EXPLICIT, ).start(wait_until_ready=True) +for model in model_map.values(): + server.load(model.id) + app = FastAPI( title="OpenAI API", description="The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details.", @@ -53,20 +63,104 @@ "/chat/completions", response_model=CreateChatCompletionResponse, tags=["Chat"] ) def create_chat_completion( - body: CreateChatCompletionRequest, + request: CreateChatCompletionRequest, ) -> CreateChatCompletionResponse: """ Creates a model response for the given chat conversation. """ + pass +def streaming_response(request_id, created, model, responses): + for response in responses: + choice = Choice( + finish_reason=FinishReason.stop if response.final else None, + index=0, + logprobs=None, + text=response.outputs["text_output"].to_string_array()[0], + ) + response = CreateCompletionResponse( + id=request_id, + choices=[choice], + system_fingerprint=None, + object=ObjectType.text_completion, + created=created, + model=model, + ) + + yield f"data: {response.json(exclude_unset=True)}\n\n" + yield "data: [DONE]\n\n" + + @app.post("/completions", response_model=CreateCompletionResponse, tags=["Completions"]) -def create_completion(body: CreateCompletionRequest) -> CreateCompletionResponse: +def create_completion( + request: CreateCompletionRequest, raw_request: Request +) -> CreateCompletionResponse | StreamingResponse: """ Creates a completion for the provided prompt and parameters. """ - pass + if request.suffix is not None: + raise HTTPException(status_code=400, detail="suffix is not currently supported") + + if request.model not in model_map: + raise HTTPException(status_code=404, detail=f"Unknown model: {request.model}") + else: + model = server.model(model_map[request.model].id) + + if request.prompt is None: + request.prompt = "<|endoftext|>" + + # Currently only support single string as input + if not isinstance(request.prompt, str): + raise HTTPException( + status_code=400, detail="only single string input is supported" + ) + + if request.logit_bias is not None or request.logprobs is not None: + raise HTTPException( + status_code=400, detail="logit bias and log probs not supported" + ) + + request_id = f"cmpl-{uuid.uuid1()}" + created = int(time.time()) + exclude_input_in_output = True + + if request.echo: + exclude_input_in_output = False + + sampling_parameters = request.copy( + exclude={"model", "prompt", "stream", "echo"} + ).dict() + + responses = model.infer( + inputs={ + "text_input": [request.prompt], + "stream": [request.stream], + "exclude_input_in_output": [exclude_input_in_output], + }, + parameters=sampling_parameters, + ) + if request.stream: + return StreamingResponse( + streaming_response(request_id, created, request.model, responses) + ) + else: + response = list(responses)[0] + choice = Choice( + finish_reason=FinishReason.stop if response.final else None, + index=0, + logprobs=None, + text=response.outputs["text_output"].to_string_array()[0], + ) + return CreateCompletionResponse( + id=request_id, + choices=[choice], + system_fingerprint=None, + object=ObjectType.text_completion, + created=created, + model=request.model, + ) @app.get("/models", response_model=ListModelsResponse, tags=["Models"]) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py index d1319175..5a1d5407 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py @@ -131,12 +131,12 @@ class Logprobs(BaseModel): class Choice(BaseModel): - finish_reason: FinishReason = Field( + finish_reason: FinishReason | None = Field( ..., description="The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence,\n`length` if the maximum number of tokens specified in the request was reached,\nor `content_filter` if content was omitted due to a flag from our content filters.\n", ) index: int - logprobs: Logprobs + logprobs: Logprobs | None text: str @@ -838,3 +838,4 @@ class CreateChatCompletionRequest(BaseModel): class ObjectType: model = Object5.model list = Object.list + text_completion = Object1.text_completion From 680499cb0a388d035258cca98f6e515787d5cd3f Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Sun, 5 May 2024 23:45:15 -0700 Subject: [PATCH 28/62] disabled logging --- .../examples/fastapi/fastapi-codegen/main.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index 295f67f7..2ce62596 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -37,7 +37,7 @@ server = tritonserver.Server( model_repository="/workspace/llm-models", - log_verbose=6, + # log_verbose=6, strict_model_config=False, model_control_mode=tritonserver.ModelControlMode.EXPLICIT, ).start(wait_until_ready=True) @@ -147,11 +147,16 @@ def create_completion( ) else: response = list(responses)[0] + try: + text = response.outputs["text_output"].to_string_array()[0] + except: + text = str(response.outputs["text_output"].to_bytes_array()[0]) + choice = Choice( finish_reason=FinishReason.stop if response.final else None, index=0, logprobs=None, - text=response.outputs["text_output"].to_string_array()[0], + text=text, ) return CreateCompletionResponse( id=request_id, From de62259dcfb0d99d803da742b811801845a85183 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 7 May 2024 06:52:25 -0700 Subject: [PATCH 29/62] incremental updates for chat completions --- .../examples/fastapi/fastapi-codegen/main.py | 130 ++++++++++++++---- .../fastapi-codegen/openai_protocol_types.py | 17 ++- 2 files changed, 117 insertions(+), 30 deletions(-) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py index 2ce62596..8673ce57 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/main.py @@ -6,24 +6,31 @@ import time import uuid +from dataclasses import dataclass +from typing import TypedDict from fastapi import FastAPI, HTTPException, Request from fastapi.responses import StreamingResponse from openai_protocol_types import ( + ChatCompletionStreamingResponseChoice, + ChatCompletionStreamResponseDelta, Choice, CreateChatCompletionRequest, CreateChatCompletionResponse, + CreateChatCompletionStreamResponse, CreateCompletionRequest, CreateCompletionResponse, - DeleteModelResponse, FinishReason, ListModelsResponse, Model, ObjectType, ) +from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast +from vllm.transformers_utils.tokenizer import get_tokenizer owned_by = "ACME" + model_map = { "llama-3-8b-instruct": Model( id="llama-3-8b-instruct", @@ -32,12 +39,18 @@ owned_by=owned_by, ) } +tokenizer_map = { + "llama-3-8b-instruct": get_tokenizer( + tokenizer_name="meta-llama/Meta-Llama-3-8B-Instruct" + ) +} + import tritonserver server = tritonserver.Server( model_repository="/workspace/llm-models", - # log_verbose=6, + log_verbose=6, strict_model_config=False, model_control_mode=tritonserver.ModelControlMode.EXPLICIT, ).start(wait_until_ready=True) @@ -59,20 +72,85 @@ ) +def streaming_chat_completion_response(request_id, created, model, role, responses): + first_response = True + + for response in responses: + choice = ChatCompletionResponse + + choice = Choice( + finish_reason=FinishReason.stop if response.final else None, + index=0, + logprobs=None, + text=response.outputs["text_output"].to_string_array()[0], + ) + response = CreateCompletionResponse( + id=request_id, + choices=[choice], + system_fingerprint=None, + object=ObjectType.text_completion, + created=created, + model=model, + ) + + yield f"data: {response.json(exclude_unset=True)}\n\n" + yield "data: [DONE]\n\n" + + @app.post( "/chat/completions", response_model=CreateChatCompletionResponse, tags=["Chat"] ) def create_chat_completion( request: CreateChatCompletionRequest, -) -> CreateChatCompletionResponse: +) -> CreateChatCompletionResponse | StreamingResponse: """ Creates a model response for the given chat conversation. """ + if request.model not in model_map: + raise HTTPException(status_code=404, detail=f"Unknown model: {request.model}") + + if request.n and request.n > 1: + raise HTTPException(status_code=400, detail=f"Only single choice is supported") + + model = server.model(request.model) + tokenizer = tokenizer_map[request.model] + + conversation = [ + {"role": str(message.role), "content": str(message.content)} + for message in request.messages + ] + + prompt = tokenizer.apply_chat_template( + conversation=conversation, tokenize=False, add_generation_prompt=False + ) + + request_id = f"cmpl-{uuid.uuid1()}" + created = int(time.time()) + exclude_input_in_output = True + + sampling_parameters = request.copy(exclude={"model", "stream", "messages"}).dict() + + responses = model.infer( + inputs={ + "text_input": [prompt], + "stream": [request.stream], + "exclude_input_in_output": [exclude_input_in_output], + }, + parameters=sampling_parameters, + ) + + if request.stream: + return StreamingResponse( + streaming_chat_completion_response( + request_id, created, request.model, conversation[-1]["role"], responses + ) + ) + pass -def streaming_response(request_id, created, model, responses): +def streaming_completion_response(request_id, created, model, responses): for response in responses: choice = Choice( finish_reason=FinishReason.stop if response.final else None, @@ -143,29 +221,29 @@ def create_completion( ) if request.stream: return StreamingResponse( - streaming_response(request_id, created, request.model, responses) - ) - else: - response = list(responses)[0] - try: - text = response.outputs["text_output"].to_string_array()[0] - except: - text = str(response.outputs["text_output"].to_bytes_array()[0]) - - choice = Choice( - finish_reason=FinishReason.stop if response.final else None, - index=0, - logprobs=None, - text=text, - ) - return CreateCompletionResponse( - id=request_id, - choices=[choice], - system_fingerprint=None, - object=ObjectType.text_completion, - created=created, - model=request.model, + streaming_completion_response(request_id, created, request.model, responses) ) + response = list(responses)[0] + + try: + text = response.outputs["text_output"].to_string_array()[0] + except: + text = str(response.outputs["text_output"].to_bytes_array()[0]) + + choice = Choice( + finish_reason=FinishReason.stop if response.final else None, + index=0, + logprobs=None, + text=text, + ) + return CreateCompletionResponse( + id=request_id, + choices=[choice], + system_fingerprint=None, + object=ObjectType.text_completion, + created=created, + model=request.model, + ) @app.get("/models", response_model=ListModelsResponse, tags=["Models"]) diff --git a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py index 5a1d5407..5a509616 100644 --- a/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py +++ b/Triton_Inference_Server_Python_API/examples/fastapi/fastapi-codegen/openai_protocol_types.py @@ -462,7 +462,7 @@ class Logprobs2(BaseModel): ) -class FinishReason3(Enum): +class ChatCompletionFinishReason(Enum): stop = "stop" length = "length" tool_calls = "tool_calls" @@ -470,12 +470,12 @@ class FinishReason3(Enum): function_call = "function_call" -class Choice3(BaseModel): +class ChatCompletionStreamingResponseChoice(BaseModel): delta: ChatCompletionStreamResponseDelta logprobs: Optional[Logprobs2] = Field( None, description="Log probability information for the choice." ) - finish_reason: FinishReason3 = Field( + finish_reason: ChatCompletionFinishReason = Field( ..., description="The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence,\n`length` if the maximum number of tokens specified in the request was reached,\n`content_filter` if content was omitted due to a flag from our content filters,\n`tool_calls` if the model called a tool, or `function_call` (deprecated) if the model called a function.\n", ) @@ -493,7 +493,7 @@ class CreateChatCompletionStreamResponse(BaseModel): ..., description="A unique identifier for the chat completion. Each chunk has the same ID.", ) - choices: List[Choice3] = Field( + choices: List[ChatCompletionStreamingResponseChoice] = Field( ..., description="A list of chat completion choices. Can be more than one if `n` is greater than 1.", ) @@ -741,6 +741,14 @@ class ChatCompletionRequestMessage(BaseModel): ChatCompletionRequestFunctionMessage, ] + @property + def role(self): + return self.role() + + @property + def content(self): + return self.content() + class CreateChatCompletionRequest(BaseModel): messages: List[ChatCompletionRequestMessage] = Field( @@ -839,3 +847,4 @@ class ObjectType: model = Object5.model list = Object.list text_completion = Object1.text_completion + chat_completion_chunk = Object4.chat_completion_chunk From 2509c34240bf73f26fc2fedb515e80f5acee3d3e Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 7 May 2024 15:20:24 -0700 Subject: [PATCH 30/62] updates for chat completion api --- .../tritonserver-2.46.0.dev0-py3-none-any.whl | Bin 258264 -> 258285 bytes .../examples/fastapi/README.md | 12 +++ .../examples/fastapi/fastapi-codegen/main.py | 86 +++++++++++++++--- .../fastapi-codegen/openai_protocol_types.py | 49 +++++----- 4 files changed, 109 insertions(+), 38 deletions(-) diff --git a/Triton_Inference_Server_Python_API/deps/tritonserver-2.46.0.dev0-py3-none-any.whl b/Triton_Inference_Server_Python_API/deps/tritonserver-2.46.0.dev0-py3-none-any.whl index 953ac2ce9ef81cd0683822d1648b9c7d350cd605..1f2fdd490698182bbdc44c4b6779f19cf4377b4e 100644 GIT binary patch delta 4344 zcmZu#byU<}(+9qUz|u$yC?L`eODagiO5@TXxpaquOGtOPON+E1pe!i}igY90DJHg$PJTyAYX0-wCV|A`i1_LO@x)e{v>nXs zIsBUbVJn04RS&&14dcMvWq)m;y|^vhYT~j{#DjP7Ow*2okjfOVFW~F33Ufx7L>OG6 z|A&rgT0o%1YY#-VU_8{oou=e{cUfRhN9r}0F2VUA6I|ITbADWJG ze|pMSU)!=#Btw%PMn_RDM>|3Z?mBX>I9`G+Q`ld`j0!ks6zOPc;ysFT8R(-jt#0)_ z;fB_IC*nihdS#%~mLmKr{}ug$ec}mR{W3z3P6H)u?g`*}e5L$Cq0Xk4URO?T!=EZ> zemplTGH(W^(yM=vcP;MZCFPPiG{9Vh)A(>AJ-e*3?rqxB-l~#P5=I^oa;%1 zlZffiJbnL%uALox0T-x9t4B_(#v8N2`_{9kQyW7Fg5EbTpG|nj-MC^|-+1lK)X+_4QoARUCzW^Z>!0L=TT;y&fG4t$S)vR|y3ZHR~dihJeSqZHn)7iC%Y<_71SlwP%8%sox2r4#G%d)D$6VLZ*j4 z2sb{=PgeUh=e8g4{Y)nr`ePMQ7kHDro`CNjnL?iY?OPF{2OaZrD@9!U0Emjnq@p}k zE<%mT=#qA1eOKm29?Pam%V#T}GE^wgV!Nu+BGJ%3kw;|vR8{z%F=~D9G)njl0YG11 zw22t?Rm>fsP z&>F&pz~wcEslix^F;Ze2siQ%I)z@&3%q&qxq$C;aT^Ug*#dx3LGNsYy+jCQIn5#<# zGj6`rKT{YDr7mIYABCh_M_8*QZAS#E7HflY!8Gk3=t{q~#I))+&Iy$^J~}2*dS%5= zHEHZ0_f8j1RRDE=&cvPjzRWw|=yH1OlGhSDbo>>fp)d*S*jUBm+6%*3L3Ng^+SEaH zm1-U}nh)x6ardo@3rB99+9#r-_2zLsaJA=Ox>Br51CAAK++5S58UcKc$(>7-XAB6?Epeb+04_Yue@0H4FB#?bZT#p>G=f7vG_^8T79`p~>9JQ|8|;f4=| zF5%9CcStnX-=lZNvK>Szu^)DzjlkZ#EJ@g4WVrW5_jiU>=20dSifbh?Qc+7jEt`OX zR>@2O?{g|!VzKC1eNo*7N*BhJmZg&v-61&WU3(I-GI39-P&=!Y^l|ZzAuJ!_Djo1O zP|_LIE^=?4cJcWrjGzRy!xVEc{)B*2OxR;@kp=Mt?3gCSYpm>a`F&(WMd8k60#O@= z*8L%_A5MM=XfPr>2Ge+g>jy2$cF*V#TpN4XP$K{Uir9bwXJ&L<_&j2 z<4sZ56J)*j=aJz?vmEgbl!eOXWw7T@SDiI{V&y!)vvWe|L>8&TZl;+d&oJUivk;RA z%FIiE#U&`bRY)0q8uy}OwyuWj7G3gp0s@%7S6yCb!)kn|ITN{P)BUGTp%{DBd%pI~ zf#E2g3h*=m_~_WZCNKeQl+cdGz~{dOvZKSdh#IM z^Zguih4{W4?#eov^bxFf!FfP%=-ckW*;&rz+1}n1h`6`sZD756^@qT7`TSj8y_5Kg z5|sny%%~bV?vXCenZ)$Zuy1XFY=3I#FyuU=kqK(D zXLCBFOa@Esu%h`KX%#CJo5+eDYC-h{C_R(0xm|?wsJgb^I?9Jgk!Dk3JnUIu1xp0v zy0RZ*0ts|m59L~EY9MlgO}TJ-Gnc*j`}5DXPI=Q&v$?Z^@^|HuJ8#x?+`B)PbV->McB*ktX9Ktj=&*(NnWVGve;u5M!816@SCg5y|-IF9s(#L-R2c@d}5IlG= ztr>^#5po`hRYa7VnxYG4v=1EX{YsT&yA>ZAP}{NgT9gT7B>MS-=dM99OXxE8#mt(| zeUO!tnqT_Qa_2AVCRGC29(vH;PN-6AUeT`?9Z-6)tuq-i2Wd7rV)>CdW*G?~%aNjd zytz|ZeVV#*cMjdZ72+%ohla5~VTHo3N%qQbU-^dxBYk+<}V zG$ifvik+@k2bu3A_$|+LgmMTv7*C_#T}m(dilCNd(sMBVichCf5i=hPlVq?(pOY|- z%+Jm7U}Mhl2=n}9WyH>C52DvjaoWdT$}6N9oD<%d({nqAJIn;ZEl0|IGtO=YF?T|Z z&w?PkC6btm1Zo!%z8`P*%b_}j)r;PqAFD1dea9X0lZx@=}a}P&)Zz{mrTCehx4F?Z&$jqzPLGiR5!9cjkHfrQ)sots(x0NI&?rr zF}4adoeg^_W+_iJc2jHnpaS$7ryhz@ADUj!we3rj+rBPgfE`$%-fNAi|7mx_!ZIVn zGO>RPPJLJ=08;n^CX?_W-B$p@`p!Av-rx4PetQvkiu1>ZR9XS@{u<~Nz?kqi0u84e z?EQ(J4g!ZFrA%l9XbdHi9FnwTE{uDUS;Zz&|s26tTpe#12idJ450nVc+X!Iw9*4~#mI~WnTc=HmDMjgFgyy&i&SU7AJ zC;KeZC?`F?vFltT?4jY@V`oDZC#+BR<7_D)^cT?L+}7a#@lvM_A@{gDWg_=9xP%hr zlv(8PSaD|{x;lzKPYW$7Du<7|MJ8;K-^T$j7ZC8RMj7fT1Wk#Wvh+7g5r^$?Feimr z_Xn$1tqb?^A#66=sXfb<>ix8-Osb;@2x5~nJk74`{YrR_Gie|5}ueH|(vFVYS zykb2cAeh35_bcd%XcpTSLu{@WIx(@j{MOSGifitkQ)}pOVFM4UTsDHR8{^NOb?vu~Vi7RZL5v&W_ zwhvF`0K4o2iqo>ud~>mNP<@xol?ZE!Dfg9~F?1-;PRCwfOcRg`=^4=fW?iB3(0!qNwWIO>~oC*GEHCsy;}?`_iMbRAT22R)S4m0hT-xL>Pm?E;