Test that Python virtual environments are treated correctly

Author: knoepfel

plugins/python/src/pymodule.cpp

@@ -124,13 +124,12 @@ static bool initialize()
   PyConfig_InitPythonConfig(&config);
   PyConfig_SetString(&config, &config.program_name, L"phlex");
 
-  // Ensure Python finds site-packages from a virtual environment if it exists
-  if (char const* python_home = std::getenv("VIRTUAL_ENV")) {
-    PyConfig_SetBytesString(&config, &config.home, python_home);
+  PyStatus status = Py_InitializeFromConfig(&config);
+  PyConfig_Clear(&config);
+  if (PyStatus_Exception(status)) {
+    throw std::runtime_error("Python initialization failed");
   }
 
-  Py_InitializeFromConfig(&config);
-
   // try again to see if the interpreter is now initialized
   if (!Py_IsInitialized())
     throw std::runtime_error("Python can not be initialized");

test/python/CMakeLists.txt

@@ -1,3 +1,5 @@
+find_package(Python 3.12 COMPONENTS Interpreter REQUIRED)
+
 # Verify installation of necessary python modules for specific tests
 
 function(check_python_module_version MODULE_NAME MIN_VERSION OUT_VAR)
@@ -92,11 +94,26 @@ set_tests_properties(
 )
 list(APPEND ACTIVE_PY_CPHLEX_TESTS py:failure)
 
+# Environment variables required:
 set(TEST_PYTHONPATH ${CMAKE_CURRENT_SOURCE_DIR}:$ENV{PYTHONPATH})
+set(
+  PYTHON_TEST_ENVIRONMENT
+  "SPDLOG_LEVEL=debug;PHLEX_PLUGIN_PATH=${PROJECT_BINARY_DIR};PYTHONPATH=${TEST_PYTHONPATH};PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL}"
+)
+
+set_tests_properties(${ACTIVE_PY_CPHLEX_TESTS} PROPERTIES ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT}")
+
+# phlex-based test to ensure that sys.path is sanely set for virtual environment
+set(PY_VIRTUAL_ENV_DIR ${CMAKE_CURRENT_BINARY_DIR}/py_virtual_env)
+execute_process(COMMAND python -m venv ${PY_VIRTUAL_ENV_DIR})
+configure_file(pysyspath.jsonnet.in pysyspath.jsonnet @ONLY)
+add_test(NAME py:syspath COMMAND phlex -c ${CMAKE_CURRENT_BINARY_DIR}/pysyspath.jsonnet)
 
+# Activate the Python virtual environment "by hand".  Requires setting the VIRTUAL_ENV
+# environment variable and prepending to the PATH environment variable.
 set_tests_properties(
-  ${ACTIVE_PY_CPHLEX_TESTS}
+  py:syspath
   PROPERTIES
-    ENVIRONMENT
-      "SPDLOG_LEVEL=debug;PHLEX_PLUGIN_PATH=${PROJECT_BINARY_DIR};PYTHONPATH=${TEST_PYTHONPATH};PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL}"
+    ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT};VIRTUAL_ENV=${PY_VIRTUAL_ENV_DIR}"
+    ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:${PY_VIRTUAL_ENV_DIR}/bin"
 )

test/python/check_sys_path.py

@@ -0,0 +1,32 @@
+"""An observer to check for output in tests.
+
+Test algorithms produce outputs. To ensure that a test is run correctly,
+this observer verifies its result against the expected value.
+"""
+
+import sys
+
+class Checker:
+
+    __name__ = 'checker'
+
+    def __init__(self, venv_path: str):
+        self._venv_path = venv_path
+
+    def __call__(self, i: int) -> None:
+        assert len(sys.path) > 0
+        venv_site_packages = f"{sys.prefix}/lib/python{sys.version_info.major}.{sys.version_info.minor}/site-packages"
+        assert any(p == venv_site_packages for p in sys.path)
+
+
+def PHLEX_REGISTER_ALGORITHMS(m, config):
+    """Register check_sys_path for checking sys.path.
+
+    Args:
+        m (internal): Phlex registrar representation.
+        config (internal): Phlex configuration representation.
+
+    Returns:
+        None
+    """
+    m.observe(Checker(config["venv"]), input_family = config["input"])

test/python/pysyspath.jsonnet.in

@@ -0,0 +1,20 @@
+{
+  driver: {
+    cpp: 'generate_layers',
+    layers: {
+      event: { parent: 'job', total: 10, starting_number: 1 }
+    }
+  },
+  sources: {
+    provider: {
+      cpp: 'cppsource4py',
+    }
+  },
+  modules: {
+    pysyspath: {
+      py: 'check_sys_path',
+      input: ['i'],
+      venv: '@PY_VIRTUAL_ENV_DIR@',
+    },
+  },
+}