diff --git a/kedro-docker/kedro_docker/template/Dockerfile.spark b/kedro-docker/kedro_docker/template/Dockerfile.spark index f0911e8c8..ca08cba39 100644 --- a/kedro-docker/kedro_docker/template/Dockerfile.spark +++ b/kedro-docker/kedro_docker/template/Dockerfile.spark @@ -5,6 +5,9 @@ FROM $BASE_IMAGE as runtime-environment RUN apt-get update && mkdir -p /usr/share/man/man1 && \ apt-get install -y procps default-jre-headless && rm -rf /var/lib/apt/lists/* +# set Spark configuration to handle Arrow properly +ENV ARROW_PRE_0_15_IPC_FORMAT=1 + # update pip and install uv RUN python -m pip install -U "pip>=21.2" RUN pip install uv