Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

gh-90556: add SharedMemory.rename on multiprocessing.shared_memory #132130

Open
wants to merge 29 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
4ecc7bd
posixshmem module shm_rename freebsd support.
devnexen Jan 16, 2022
1336f8c
update shared_memory library
devnexen Jan 16, 2022
d11ff82
doc update
devnexen Jan 16, 2022
13734b6
📜🤖 Added by blurb_it.
blurb-it[bot] Jan 17, 2022
c82d4db
Merge remote-tracking branch 'origin/main' into shm_rename
aisk Mar 29, 2025
69d583f
Update generated files
aisk Mar 29, 2025
83d63f4
Update by review
aisk Mar 29, 2025
ba0e6d0
Improve document by introduce more RST markers
aisk Mar 29, 2025
514ed3d
Add test
aisk Apr 5, 2025
810e765
update quote in .in file
aisk Apr 5, 2025
fa45c2e
regen-config
aisk Apr 5, 2025
bea3158
Merge branch 'main' into shm_rename
aisk Apr 5, 2025
ca0714b
Fix news entry
aisk Apr 5, 2025
b3ec978
Fix docs
aisk Apr 5, 2025
778e646
fix news entry
aisk Apr 5, 2025
7e3242c
fix news entry
aisk Apr 5, 2025
6ec2152
Update Lib/multiprocessing/shared_memory.py
aisk Apr 6, 2025
8109775
Update Doc/library/multiprocessing.shared_memory.rst
aisk Apr 6, 2025
8e86d21
Update Doc/library/multiprocessing.shared_memory.rst
aisk Apr 6, 2025
ef3b5ce
Update Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst
aisk Apr 6, 2025
86debb3
Update Doc/library/multiprocessing.shared_memory.rst
aisk Apr 6, 2025
cb181b0
Remove whitespace in empty line
aisk Apr 6, 2025
f6f8312
Define the rename method conditionally
aisk Apr 6, 2025
1cc08d7
Fix test on Windows
aisk Apr 6, 2025
d77049c
Fix codes on Windows
aisk Apr 6, 2025
416c6d8
Move constants
aisk Apr 6, 2025
620da83
Update document
aisk Apr 6, 2025
4993685
Fix indent in document
aisk Apr 6, 2025
8adc394
Add more tests
aisk Apr 7, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions Doc/library/multiprocessing.shared_memory.rst
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,32 @@ copying of data.
This method has no effect on Windows, where the only way to delete a
shared memory block is to close all handles.

.. method:: rename(newname, flags=0)

Rename the underlying shared memory block to *newname*.
By default, if *newname* already exists, it will be unlinked beforehand.

*flags* (0 by default) takes :ref:`bitwise ORed <bitwise>` flags together.

.. availability:: FreeBSD >= 13.0

.. versionadded:: next

.. data:: SHM_RENAME_EXCHANGE
SHM_RENAME_NOREPLACE

Parameters to the :func:`rename` function.

:const:`SHM_RENAME_EXCHANGE`
Atomically exchange the SharedMemory to the *newname*.

:const:`SHM_RENAME_NOREPLACE`
Raise an error if *newname* exists, rather than unlinking it.

.. availability:: FreeBSD >= 13.0

.. versionadded:: next

.. attribute:: buf

A memoryview of contents of the shared memory block.
Expand Down
26 changes: 26 additions & 0 deletions Lib/multiprocessing/shared_memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from functools import partial
import mmap
import os
import platform
import errno
import struct
import secrets
Expand All @@ -36,6 +37,10 @@
else:
_SHM_NAME_PREFIX = 'wnsm_'

if _USE_POSIX and hasattr(_posixshmem, "shm_rename"):
from _posixshmem import SHM_RENAME_EXCHANGE
from _posixshmem import SHM_RENAME_NOREPLACE


def _make_filename():
"Create a random filename for the shared memory object."
Expand Down Expand Up @@ -253,6 +258,27 @@ def unlink(self):
if self._track:
resource_tracker.unregister(self._name, "shared_memory")

if _USE_POSIX and hasattr(_posixshmem, "shm_rename"):
def rename(self, newname, flags=0):
"""Renames a shared memory block.

The policy how the operation is handled depends on the flag passed.
The default behavior is if the newname already exists, it will
be unlinked beforehand.
With the SHM_RENAME_EXCHANGE flag, the old and new name will
be exchanged.
With the SHM_RENAME_NOREPLACE flag, an error will be returned
if the new name exists.
"""
oldname = self._name
if _USE_POSIX and self._prepend_leading_slash:
newname = "/" + newname
self._fd = _posixshmem.shm_rename(self._name, newname, flags)
if self._track:
resource_tracker.unregister(oldname, "shared_memory")
resource_tracker.register(newname, "shared_memory")
self._name = newname


_encoding = "utf8"

Expand Down
35 changes: 35 additions & 0 deletions Lib/test/_test_multiprocessing.py
Original file line number Diff line number Diff line change
Expand Up @@ -4376,6 +4376,41 @@ def test_shared_memory_recreate(self):
self.addCleanup(shm2.unlink)
self.assertEqual(shm2._name, names[1])

@unittest.skipUnless(hasattr(shared_memory.SharedMemory, "rename"),
"requires SharedMomery.rename")
def test_shared_memory_rename(self):
name1 = self._new_shm_name('testrename01_tsmb')
name2 = self._new_shm_name('testrename02_tsmb')
sms = shared_memory.SharedMemory(name1, create=True, size=512)
self.addCleanup(sms.unlink)

sms.rename(name2)
self.assertEqual(sms.name, name2)

@unittest.skipUnless(hasattr(shared_memory.SharedMemory, "rename"),
"requires SharedMomery.rename")
def test_shared_memory_rename_noreplace(self):
name1 = self._new_shm_name('testrename01_tsmb')
name2 = self._new_shm_name('testrename02_tsmb')
sms1 = shared_memory.SharedMemory(name1, create=True, size=512)
sms2 = shared_memory.SharedMemory(name2, create=True, size=512)
self.addCleanup(sms1.unlink)
self.addCleanup(sms2.unlink)

with self.assertRaises(FileExistsError):
sms1.rename(name2, flags=shared_memory.SHM_RENAME_NOREPLACE)

@unittest.skipUnless(hasattr(shared_memory.SharedMemory, "rename"),
"requires SharedMomery.rename")
def test_shared_memory_rename_exchange(self):
name1 = self._new_shm_name('testrename01_tsmb')
name2 = self._new_shm_name('testrename02_tsmb')
sms1 = shared_memory.SharedMemory(name1, create=True, size=512)
sms2 = shared_memory.SharedMemory(name2, create=True, size=512)
self.addCleanup(sms2.unlink)

sms1.rename(name2, flags=shared_memory.SHM_RENAME_EXCHANGE)

def test_invalid_shared_memory_creation(self):
# Test creating a shared memory segment with negative size
with self.assertRaises(ValueError):
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add the :mod:`multiprocessing.shared_memory.SharedMemory.rename` method for FreeBSD.
67 changes: 66 additions & 1 deletion Modules/_multiprocessing/clinic/posixshmem.c.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

70 changes: 70 additions & 0 deletions Modules/_multiprocessing/posixshmem.c
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,59 @@ _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags,
}
#endif /* HAVE_SHM_OPEN */

#ifdef HAVE_SHM_RENAME
/*[clinic input]
_posixshmem.shm_rename -> int
path_from: unicode
path_to: unicode
flags: int
/

Rename a shared memory object.

Remove a shared memory object and relink to another path.
By default, if the destination path already exist, it will be unlinked.
With the SHM_RENAME_EXCHANGE flag, source and destination paths
will be exchanged.
With the SHM_RENAME_NOREPLACE flag, an error will be triggered
if the destination alredady exists.

[clinic start generated code]*/

static int
_posixshmem_shm_rename_impl(PyObject *module, PyObject *path_from,
PyObject *path_to, int flags)
/*[clinic end generated code: output=d9a710c512166e18 input=5fb42d1ce077caec]*/
{
int rv;
int async_err = 0;
Py_ssize_t from_size;
Py_ssize_t to_size;
const char *from = PyUnicode_AsUTF8AndSize(path_from, &from_size);
const char *to = PyUnicode_AsUTF8AndSize(path_to, &to_size);
if (from == NULL || to == NULL) {
return -1;
}
if (strlen(from) != (size_t)from_size || strlen(to) != (size_t)to_size) {
PyErr_SetString(PyExc_ValueError, "embedded null character");
return -1;
}
do {
Py_BEGIN_ALLOW_THREADS
rv = shm_rename(from, to, flags);
Py_END_ALLOW_THREADS
} while (rv < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals()));

if (rv < 0) {
if (!async_err)
PyErr_SetFromErrnoWithFilenameObjects(PyExc_OSError, path_from, path_to);
return -1;
}

return rv;
}
#endif /* HAVE_SHM_RENAME */

#ifdef HAVE_SHM_UNLINK
/*[clinic input]
_posixshmem.shm_unlink
Expand Down Expand Up @@ -122,12 +175,29 @@ _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path)

static PyMethodDef module_methods[ ] = {
_POSIXSHMEM_SHM_OPEN_METHODDEF
#if defined(HAVE_SHM_RENAME)
_POSIXSHMEM_SHM_RENAME_METHODDEF
#endif
_POSIXSHMEM_SHM_UNLINK_METHODDEF
{NULL} /* Sentinel */
};

static int
posixshmem_exec(PyObject *m)
{
#ifdef HAVE_SHM_RENAME
#ifdef SHM_RENAME_EXCHANGE
if (PyModule_AddIntMacro(m, SHM_RENAME_EXCHANGE)) return -1;
#endif
#ifdef SHM_RENAME_NOREPLACE
if (PyModule_AddIntMacro(m, SHM_RENAME_NOREPLACE)) return -1;
#endif
#endif /* HAVE_SHM_RENAME */
return 0;
}

static PyModuleDef_Slot module_slots[] = {
{Py_mod_exec, posixshmem_exec},
{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
{Py_mod_gil, Py_MOD_GIL_NOT_USED},
{0, NULL}
Expand Down
2 changes: 1 addition & 1 deletion configure

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -7297,7 +7297,7 @@ WITH_SAVE_ENV([
# endif
#endif
"
AC_CHECK_FUNCS([shm_open shm_unlink], [have_posix_shmem=yes], [have_posix_shmem=no])
AC_CHECK_FUNCS([shm_open shm_rename shm_unlink], [have_posix_shmem=yes], [have_posix_shmem=no])
_RESTORE_VAR([ac_includes_default])
])

Expand Down
3 changes: 3 additions & 0 deletions pyconfig.h.in
Original file line number Diff line number Diff line change
Expand Up @@ -1191,6 +1191,9 @@
/* Define to 1 if you have the 'shm_open' function. */
#undef HAVE_SHM_OPEN

/* Define to 1 if you have the 'shm_rename' function. */
#undef HAVE_SHM_RENAME

/* Define to 1 if you have the 'shm_unlink' function. */
#undef HAVE_SHM_UNLINK

Expand Down
Loading