ci(docker): set legacy IPC format to fix e2e tests

kedro-docker/kedro_docker/template/Dockerfile.spark

@@ -5,6 +5,16 @@ FROM $BASE_IMAGE as runtime-environment
 RUN apt-get update && mkdir -p /usr/share/man/man1 && \
     apt-get install -y procps default-jre-headless && rm -rf /var/lib/apt/lists/*
 
+# set JVM options to allow Unsafe operations
+ENV JAVA_OPTS="-Dio.netty.tryReflectionSetAccessible=true \
+    --add-opens=java.base/java.nio=ALL-UNNAMED \
+    --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \
+    --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED"
+
+# set Spark configuration to handle Arrow properly
+ENV SPARK_SUBMIT_OPTS="-Dio.netty.tryReflectionSetAccessible=true"
+ENV ARROW_PRE_0_15_IPC_FORMAT=1
+
 # update pip and install uv
 RUN python -m pip install -U "pip>=21.2"
 RUN pip install uv