diff --git a/spk/python311/Makefile b/spk/python311/Makefile
index 5b96eff5c568..a3759e128178 100644
--- a/spk/python311/Makefile
+++ b/spk/python311/Makefile
@@ -31,11 +31,6 @@ WHEELS = src/requirements-pure.txt
 # Force testing all wheel builds
 WHEELS_TEST_ALL = 0
 
-# Force compiler LTO optimizations
-ifeq ($(strip $(WHEELS_TEST_ALL)),0)
-ENV += PYTHON_OPTIMIZE=1
-endif
-
 SERVICE_SETUP = src/service-setup.sh
 
 PYTHON_LIB_DIR = lib/python$(SPK_VERS_MAJOR_MINOR)
@@ -108,6 +103,18 @@ ifeq ($(findstring $(ARCH),qoriq $(x64_ARCHS)),$(ARCH))
 GCC_DEBUG_INFO := 1
 endif
 
+# Force compiler LTO optimizations except:
+#   - when testing all wheels
+#   - when including debug_info symbols
+#   - when using PPC arch (fails on qoriq)
+ifneq ($(strip $(WHEELS_TEST_ALL)),1)
+ifneq ($(strip $(GCC_DEBUG_INFO)),1)
+ifneq ($(findstring $(ARCH),$(OLD_PPC_ARCHS)),$(ARCH))
+ENV += PYTHON_OPTIMIZE=1
+endif
+endif
+endif
+
 ## WHEELS_TEST_ALL: Force testing all wheel building
 ifeq ($(strip $(WHEELS_TEST_ALL)),1)