From 25210c6103d710f19c54a9b1caa8b4a802a4377b Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 19 Apr 2024 09:23:02 +0200 Subject: [PATCH 01/16] Add SharedMemory and NDArray SharedMemory implementation is lifted from JDLL. NDArray wraps SharedMemory, and adds shape and data-type information. Co-authored-by: Carlos Garcia Lopez de Haro <100329787@alumnos.uc3m.es> --- pom.xml | 2 +- src/main/java/org/apposed/appose/DType.java | 63 + src/main/java/org/apposed/appose/NDArray.java | 2755 ++--------------- .../java/org/apposed/appose/SharedMemory.java | 387 +-- .../java/org/apposed/appose/ShmNDArray.java | 59 - src/main/java/org/apposed/appose/Types.java | 140 +- .../java/org/apposed/appose/shm/CLibrary.java | 40 + .../java/org/apposed/appose/shm/LibRt.java | 41 + .../org/apposed/appose/shm/MacosHelpers.java | 61 + .../java/org/apposed/appose/shm/ShmImpl.java | 55 + .../java/org/apposed/appose/shm/ShmLinux.java | 294 ++ .../java/org/apposed/appose/shm/ShmMacOS.java | 193 ++ .../java/org/apposed/appose/shm/ShmUtils.java | 124 + .../org/apposed/appose/shm/ShmWindows.java | 246 ++ .../org/apposed/appose/shm/ShmCreate.dylib | Bin 0 -> 99378 bytes .../apposed/appose/NDArrayExampleGroovy.java | 45 + 16 files changed, 1629 insertions(+), 2876 deletions(-) create mode 100644 src/main/java/org/apposed/appose/DType.java delete mode 100644 src/main/java/org/apposed/appose/ShmNDArray.java create mode 100644 src/main/java/org/apposed/appose/shm/CLibrary.java create mode 100644 src/main/java/org/apposed/appose/shm/LibRt.java create mode 100644 src/main/java/org/apposed/appose/shm/MacosHelpers.java create mode 100644 src/main/java/org/apposed/appose/shm/ShmImpl.java create mode 100644 src/main/java/org/apposed/appose/shm/ShmLinux.java create mode 100644 src/main/java/org/apposed/appose/shm/ShmMacOS.java create mode 100644 src/main/java/org/apposed/appose/shm/ShmUtils.java create mode 100644 src/main/java/org/apposed/appose/shm/ShmWindows.java create mode 100755 src/main/resources/org/apposed/appose/shm/ShmCreate.dylib create mode 100644 src/test/java/org/apposed/appose/NDArrayExampleGroovy.java diff --git a/pom.xml b/pom.xml index 0467583..263aacb 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ org.apposed appose - 0.1.1-SNAPSHOT + 0.2.0-SNAPSHOT Appose Appose: multi-language interprocess cooperation with shared memory. diff --git a/src/main/java/org/apposed/appose/DType.java b/src/main/java/org/apposed/appose/DType.java new file mode 100644 index 0000000..1c229ef --- /dev/null +++ b/src/main/java/org/apposed/appose/DType.java @@ -0,0 +1,63 @@ +package org.apposed.appose; + +/** + * Enumerates possible data type of {@link NDArray} elements. + */ +public enum DType { + INT8("int8", Byte.BYTES), // + INT16("int16", Short.BYTES), // + INT32("int32", Integer.BYTES), // + INT64("int64", Long.BYTES), // + UINT8("uint8", Byte.BYTES), // + UINT16("uint16", Short.BYTES), // + UINT32("uint32", Integer.BYTES), // + UINT64("uint64", Long.BYTES), // + FLOAT32("float32", Float.BYTES), // + FLOAT64("float64", Double.BYTES), // + COMPLEX64("complex64", Float.BYTES * 2), // + COMPLEX128("complex128", Double.BYTES * 2), // + BOOL("bool", 1); + + private final String label; + + private final int bytesPerElement; + + DType(final String label, final int bytesPerElement) + { + this.label = label; + this.bytesPerElement = bytesPerElement; + } + + /** + * Get the number of bytes per element for this data type. + */ + public int bytesPerElement() + { + return bytesPerElement; + } + + /** + * Get the label of this {@code DType}. + *

+ * The label can used as a {@code dtype} in Python. It is also used for JSON + * serialization. + * + * @return the label. + */ + public String label() + { + return label; + } + + /** + * Returns the {@code DType} corresponding to the given {@code label}. + * + * @param label a label. + * @return {@code DType} corresponding to {@code label}. + * @throws IllegalArgumentException if no {@code DType} corresponds to the given label. + */ + public static DType fromLabel(final String label) throws IllegalArgumentException + { + return valueOf( label.toUpperCase() ); + } +} diff --git a/src/main/java/org/apposed/appose/NDArray.java b/src/main/java/org/apposed/appose/NDArray.java index aec6766..92f2d38 100644 --- a/src/main/java/org/apposed/appose/NDArray.java +++ b/src/main/java/org/apposed/appose/NDArray.java @@ -29,2516 +29,265 @@ package org.apposed.appose; +import java.nio.ByteBuffer; +import java.util.Arrays; + /** - * Java interface for a NumPy-style ndarray. + * Represents a multi-dimensional array similar to NumPy ndarray. *

- * An array object represents a multidimensional, homogeneous array - * of fixed-size items. An associated data-type object describes the - * format of each element in the array (its byte-order, how many bytes it - * occupies in memory, whether it is an integer, a floating point number, - * or something else, etc.) - *

+ * The array contains elements of a {@link DType data type}, arranged in a + * particular {@link Shape}, and flattened into {@link SharedMemory}, */ -public interface NDArray extends Iterable { +public class NDArray implements AutoCloseable { + + /** + * shared memory containing the flattened array data. + */ + private final SharedMemory sharedMemory; + + /** + * data type of the array elements. + */ + private final DType dType; + + /** + * shape of the array. + */ + private final Shape shape; + + /** + * Constructs an {@code NDArray} with the specified {@code SharedMemory}. + * + * @param sharedMemory the flattened array data. + * @param dType element data type + * @param shape array shape + */ + public NDArray(final SharedMemory sharedMemory, final DType dType, final Shape shape) { + this.sharedMemory = sharedMemory; + this.dType = dType; + this.shape = shape; + } + + /** + * Constructs an {@code NDArray} with the specified data type and shape, + * allocating a new {@code SharedMemory} block. + * + * @param dType element data type + * @param shape array shape + */ + public NDArray(final DType dType, final Shape shape) { + this( + new SharedMemory(null,true,safeInt(shape.numElements() * dType.bytesPerElement())), + dType, shape); + } + + /** + * @return The data type of the array elements. + */ + public DType dType() { + return dType; + } + + /** + * @return The shape of the array. + */ + public Shape shape() { + return shape; + } + + /** + * @return The shared memory block containing the array data. + */ + public SharedMemory shm() { + return sharedMemory; + } - // Buffer protocol! - // https://peps.python.org/pep-3118/ + /** + * Returns a ByteBuffer view of the array data. + * + * @return A ByteBuffer view of {@link #shm()}. + */ + public ByteBuffer buffer() { + final long length = shape.numElements() * dType.bytesPerElement(); + return sharedMemory.pointer().getByteBuffer(0, length); + } - long len(); + /** + * Release resources ({@code SharedMemory}) associated with this {@code NDArray}. + */ + @Override + public void close() throws Exception { + sharedMemory.close(); + } - /* -Help on class ndarray in module numpy: + @Override + public String toString() { + return "NDArray{" + + "sharedMemory=" + sharedMemory + + ", dType=" + dType + + ", shape=" + shape + + '}'; + } -class ndarray(builtins.object) - | ndarray(shape, dtype=float, buffer=None, offset=0, - | strides=None, order=None) - | - | - | Attributes - | ---------- - | T : ndarray - | Transpose of the array. - | data : buffer - | The array's elements, in memory. - | dtype : dtype object - | Describes the format of the elements in the array. - | flags : dict - | Dictionary containing information related to memory use, e.g., - | 'C_CONTIGUOUS', 'OWNDATA', 'WRITEABLE', etc. - | flat : numpy.flatiter object - | Flattened version of the array as an iterator. The iterator - | allows assignments, e.g., ``x.flat = 3`` (See `ndarray.flat` for - | assignment examples; TODO). - | imag : ndarray - | Imaginary part of the array. - | real : ndarray - | Real part of the array. - | size : int - | Number of elements in the array. - | itemsize : int - | The memory use of each array element in bytes. - | nbytes : int - | The total number of bytes required to store the array data, - | i.e., ``itemsize * size``. - | ndim : int - | The array's number of dimensions. - | shape : tuple of ints - | Shape of the array. - | strides : tuple of ints - | The step-size required to move from one element to the next in - | memory. For example, a contiguous ``(3, 4)`` array of type - | ``int16`` in C-order has strides ``(8, 2)``. This implies that - | to move from element to element in memory requires jumps of 2 bytes. - | To move from row-to-row, one needs to jump 8 bytes at a time - | (``2 * 4``). - | ctypes : ctypes object - | Class containing properties of the array needed for interaction - | with ctypes. - | base : ndarray - | If the array is a view into another array, that array is its `base` - | (unless that array is also a view). The `base` array is where the - | array data is actually stored. - | - | See Also - | -------- - | array : Construct an array. - | zeros : Create an array, each element of which is zero. - | empty : Create an array, but leave its allocated memory unchanged (i.e., - | it contains "garbage"). - | dtype : Create a data-type. - | numpy.typing.NDArray : An ndarray alias :term:`generic ` - | w.r.t. its `dtype.type `. - | - | Notes - | ----- - | There are two modes of creating an array using ``__new__``: - | - | 1. If `buffer` is None, then only `shape`, `dtype`, and `order` - | are used. - | 2. If `buffer` is an object exposing the buffer interface, then - | all keywords are interpreted. - | - | No ``__init__`` method is needed because the array is fully initialized - | after the ``__new__`` method. - | - | Examples - | -------- - | These examples illustrate the low-level `ndarray` constructor. Refer - | to the `See Also` section above for easier ways of constructing an - | ndarray. - | - | First mode, `buffer` is None: - | - | >>> np.ndarray(shape=(2,2), dtype=float, order='F') - | array([[0.0e+000, 0.0e+000], # random - | [ nan, 2.5e-323]]) - | - | Second mode: - | - | >>> np.ndarray((2,), buffer=np.array([1,2,3]), - | ... offset=np.int_().itemsize, - | ... dtype=int) # offset = 1*itemsize, i.e. skip first element - | array([2, 3]) - | - | Methods defined here: - | - | __abs__(self, /) - | abs(self) - | - | __add__(self, value, /) - | Return self+value. - | - | __and__(self, value, /) - | Return self&value. - | - | __array__(...) - | a.__array__([dtype], /) -> reference if type unchanged, copy otherwise. - | - | Returns either a new reference to self if dtype is not given or a new array - | of provided data type if dtype is different from the current dtype of the - | array. - | - | __array_finalize__(...) - | a.__array_finalize__(obj, /) - | - | Present so subclasses can call super. Does nothing. - | - | __array_function__(...) - | - | __array_prepare__(...) - | a.__array_prepare__(array[, context], /) - | - | Returns a view of `array` with the same type as self. - | - | __array_ufunc__(...) - | - | __array_wrap__(...) - | a.__array_wrap__(array[, context], /) - | - | Returns a view of `array` with the same type as self. - | - | __bool__(self, /) - | True if self else False - | - | __complex__(...) - | - | __contains__(self, key, /) - | Return key in self. - | - | __copy__(...) - | a.__copy__() - | - | Used if :func:`copy.copy` is called on an array. Returns a copy of the array. - | - | Equivalent to ``a.copy(order='K')``. - | - | __deepcopy__(...) - | a.__deepcopy__(memo, /) -> Deep copy of array. - | - | Used if :func:`copy.deepcopy` is called on an array. - | - | __delitem__(self, key, /) - | Delete self[key]. - | - | __divmod__(self, value, /) - | Return divmod(self, value). - | - | __dlpack__(...) - | a.__dlpack__(*, stream=None) - | - | DLPack Protocol: Part of the Array API. - | - | __dlpack_device__(...) - | a.__dlpack_device__() - | - | DLPack Protocol: Part of the Array API. - | - | __eq__(self, value, /) - | Return self==value. - | - | __float__(self, /) - | float(self) - | - | __floordiv__(self, value, /) - | Return self//value. - | - | __format__(...) - | Default object formatter. - | - | __ge__(self, value, /) - | Return self>=value. - | - | __getitem__(self, key, /) - | Return self[key]. - | - | __gt__(self, value, /) - | Return self>value. - | - | __iadd__(self, value, /) - | Return self+=value. - | - | __iand__(self, value, /) - | Return self&=value. - | - | __ifloordiv__(self, value, /) - | Return self//=value. - | - | __ilshift__(self, value, /) - | Return self<<=value. - | - | __imatmul__(self, value, /) - | Return self@=value. - | - | __imod__(self, value, /) - | Return self%=value. - | - | __imul__(self, value, /) - | Return self*=value. - | - | __index__(self, /) - | Return self converted to an integer, if self is suitable for use as an index into a list. - | - | __int__(self, /) - | int(self) - | - | __invert__(self, /) - | ~self - | - | __ior__(self, value, /) - | Return self|=value. - | - | __ipow__(self, value, /) - | Return self**=value. - | - | __irshift__(self, value, /) - | Return self>>=value. - | - | __isub__(self, value, /) - | Return self-=value. - | - | __iter__(self, /) - | Implement iter(self). - | - | __itruediv__(self, value, /) - | Return self/=value. - | - | __ixor__(self, value, /) - | Return self^=value. - | - | __le__(self, value, /) - | Return self<=value. - | - | __len__(self, /) - | Return len(self). - | - | __lshift__(self, value, /) - | Return self<>self. - | - | __rshift__(self, value, /) - | Return self>>value. - | - | __rsub__(self, value, /) - | Return value-self. - | - | __rtruediv__(self, value, /) - | Return value/self. - | - | __rxor__(self, value, /) - | Return value^self. - | - | __setitem__(self, key, value, /) - | Set self[key] to value. - | - | __setstate__(...) - | a.__setstate__(state, /) - | - | For unpickling. - | - | The `state` argument must be a sequence that contains the following - | elements: - | - | Parameters - | ---------- - | version : int - | optional pickle version. If omitted defaults to 0. - | shape : tuple - | dtype : data-type - | isFortran : bool - | rawdata : string or list - | a binary string with the data (or a list if 'a' is an object array) - | - | __sizeof__(...) - | Size of object in memory, in bytes. - | - | __str__(self, /) - | Return str(self). - | - | __sub__(self, value, /) - | Return self-value. - | - | __truediv__(self, value, /) - | Return self/value. - | - | __xor__(self, value, /) - | Return self^value. - | - | all(...) - | a.all(axis=None, out=None, keepdims=False, *, where=True) - | - | Returns True if all elements evaluate to True. - | - | Refer to `numpy.all` for full documentation. - | - | See Also - | -------- - | numpy.all : equivalent function - | - | any(...) - | a.any(axis=None, out=None, keepdims=False, *, where=True) - | - | Returns True if any of the elements of `a` evaluate to True. - | - | Refer to `numpy.any` for full documentation. - | - | See Also - | -------- - | numpy.any : equivalent function - | - | argmax(...) - | a.argmax(axis=None, out=None, *, keepdims=False) - | - | Return indices of the maximum values along the given axis. - | - | Refer to `numpy.argmax` for full documentation. - | - | See Also - | -------- - | numpy.argmax : equivalent function - | - | argmin(...) - | a.argmin(axis=None, out=None, *, keepdims=False) - | - | Return indices of the minimum values along the given axis. - | - | Refer to `numpy.argmin` for detailed documentation. - | - | See Also - | -------- - | numpy.argmin : equivalent function - | - | argpartition(...) - | a.argpartition(kth, axis=-1, kind='introselect', order=None) - | - | Returns the indices that would partition this array. - | - | Refer to `numpy.argpartition` for full documentation. - | - | .. versionadded:: 1.8.0 - | - | See Also - | -------- - | numpy.argpartition : equivalent function - | - | argsort(...) - | a.argsort(axis=-1, kind=None, order=None) - | - | Returns the indices that would sort this array. - | - | Refer to `numpy.argsort` for full documentation. - | - | See Also - | -------- - | numpy.argsort : equivalent function - | - | astype(...) - | a.astype(dtype, order='K', casting='unsafe', subok=True, copy=True) - | - | Copy of the array, cast to a specified type. - | - | Parameters - | ---------- - | dtype : str or dtype - | Typecode or data-type to which the array is cast. - | order : {'C', 'F', 'A', 'K'}, optional - | Controls the memory layout order of the result. - | 'C' means C order, 'F' means Fortran order, 'A' - | means 'F' order if all the arrays are Fortran contiguous, - | 'C' order otherwise, and 'K' means as close to the - | order the array elements appear in memory as possible. - | Default is 'K'. - | casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional - | Controls what kind of data casting may occur. Defaults to 'unsafe' - | for backwards compatibility. - | - | * 'no' means the data types should not be cast at all. - | * 'equiv' means only byte-order changes are allowed. - | * 'safe' means only casts which can preserve values are allowed. - | * 'same_kind' means only safe casts or casts within a kind, - | like float64 to float32, are allowed. - | * 'unsafe' means any data conversions may be done. - | subok : bool, optional - | If True, then sub-classes will be passed-through (default), otherwise - | the returned array will be forced to be a base-class array. - | copy : bool, optional - | By default, astype always returns a newly allocated array. If this - | is set to false, and the `dtype`, `order`, and `subok` - | requirements are satisfied, the input array is returned instead - | of a copy. - | - | Returns - | ------- - | arr_t : ndarray - | Unless `copy` is False and the other conditions for returning the input - | array are satisfied (see description for `copy` input parameter), `arr_t` - | is a new array of the same shape as the input array, with dtype, order - | given by `dtype`, `order`. - | - | Notes - | ----- - | .. versionchanged:: 1.17.0 - | Casting between a simple data type and a structured one is possible only - | for "unsafe" casting. Casting to multiple fields is allowed, but - | casting from multiple fields is not. - | - | .. versionchanged:: 1.9.0 - | Casting from numeric to string types in 'safe' casting mode requires - | that the string dtype length is long enough to store the max - | integer/float value converted. - | - | Raises - | ------ - | ComplexWarning - | When casting from complex to float or int. To avoid this, - | one should use ``a.real.astype(t)``. - | - | Examples - | -------- - | >>> x = np.array([1, 2, 2.5]) - | >>> x - | array([1. , 2. , 2.5]) - | - | >>> x.astype(int) - | array([1, 2, 2]) - | - | byteswap(...) - | a.byteswap(inplace=False) - | - | Swap the bytes of the array elements - | - | Toggle between low-endian and big-endian data representation by - | returning a byteswapped array, optionally swapped in-place. - | Arrays of byte-strings are not swapped. The real and imaginary - | parts of a complex number are swapped individually. - | - | Parameters - | ---------- - | inplace : bool, optional - | If ``True``, swap bytes in-place, default is ``False``. - | - | Returns - | ------- - | out : ndarray - | The byteswapped array. If `inplace` is ``True``, this is - | a view to self. - | - | Examples - | -------- - | >>> A = np.array([1, 256, 8755], dtype=np.int16) - | >>> list(map(hex, A)) - | ['0x1', '0x100', '0x2233'] - | >>> A.byteswap(inplace=True) - | array([ 256, 1, 13090], dtype=int16) - | >>> list(map(hex, A)) - | ['0x100', '0x1', '0x3322'] - | - | Arrays of byte-strings are not swapped - | - | >>> A = np.array([b'ceg', b'fac']) - | >>> A.byteswap() - | array([b'ceg', b'fac'], dtype='|S3') - | - | ``A.newbyteorder().byteswap()`` produces an array with the same values - | but different representation in memory - | - | >>> A = np.array([1, 2, 3]) - | >>> A.view(np.uint8) - | array([1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, - | 0, 0], dtype=uint8) - | >>> A.newbyteorder().byteswap(inplace=True) - | array([1, 2, 3]) - | >>> A.view(np.uint8) - | array([0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, - | 0, 3], dtype=uint8) - | - | choose(...) - | a.choose(choices, out=None, mode='raise') - | - | Use an index array to construct a new array from a set of choices. - | - | Refer to `numpy.choose` for full documentation. - | - | See Also - | -------- - | numpy.choose : equivalent function - | - | clip(...) - | a.clip(min=None, max=None, out=None, **kwargs) - | - | Return an array whose values are limited to ``[min, max]``. - | One of max or min must be given. - | - | Refer to `numpy.clip` for full documentation. - | - | See Also - | -------- - | numpy.clip : equivalent function - | - | compress(...) - | a.compress(condition, axis=None, out=None) - | - | Return selected slices of this array along given axis. - | - | Refer to `numpy.compress` for full documentation. - | - | See Also - | -------- - | numpy.compress : equivalent function - | - | conj(...) - | a.conj() - | - | Complex-conjugate all elements. - | - | Refer to `numpy.conjugate` for full documentation. - | - | See Also - | -------- - | numpy.conjugate : equivalent function - | - | conjugate(...) - | a.conjugate() - | - | Return the complex conjugate, element-wise. - | - | Refer to `numpy.conjugate` for full documentation. - | - | See Also - | -------- - | numpy.conjugate : equivalent function - | - | copy(...) - | a.copy(order='C') - | - | Return a copy of the array. - | - | Parameters - | ---------- - | order : {'C', 'F', 'A', 'K'}, optional - | Controls the memory layout of the copy. 'C' means C-order, - | 'F' means F-order, 'A' means 'F' if `a` is Fortran contiguous, - | 'C' otherwise. 'K' means match the layout of `a` as closely - | as possible. (Note that this function and :func:`numpy.copy` are very - | similar but have different default values for their order= - | arguments, and this function always passes sub-classes through.) - | - | See also - | -------- - | numpy.copy : Similar function with different default behavior - | numpy.copyto - | - | Notes - | ----- - | This function is the preferred method for creating an array copy. The - | function :func:`numpy.copy` is similar, but it defaults to using order 'K', - | and will not pass sub-classes through by default. - | - | Examples - | -------- - | >>> x = np.array([[1,2,3],[4,5,6]], order='F') - | - | >>> y = x.copy() - | - | >>> x.fill(0) - | - | >>> x - | array([[0, 0, 0], - | [0, 0, 0]]) - | - | >>> y - | array([[1, 2, 3], - | [4, 5, 6]]) - | - | >>> y.flags['C_CONTIGUOUS'] - | True - | - | cumprod(...) - | a.cumprod(axis=None, dtype=None, out=None) - | - | Return the cumulative product of the elements along the given axis. - | - | Refer to `numpy.cumprod` for full documentation. - | - | See Also - | -------- - | numpy.cumprod : equivalent function - | - | cumsum(...) - | a.cumsum(axis=None, dtype=None, out=None) - | - | Return the cumulative sum of the elements along the given axis. - | - | Refer to `numpy.cumsum` for full documentation. - | - | See Also - | -------- - | numpy.cumsum : equivalent function - | - | diagonal(...) - | a.diagonal(offset=0, axis1=0, axis2=1) - | - | Return specified diagonals. In NumPy 1.9 the returned array is a - | read-only view instead of a copy as in previous NumPy versions. In - | a future version the read-only restriction will be removed. - | - | Refer to :func:`numpy.diagonal` for full documentation. - | - | See Also - | -------- - | numpy.diagonal : equivalent function - | - | dot(...) - | - | dump(...) - | a.dump(file) - | - | Dump a pickle of the array to the specified file. - | The array can be read back with pickle.load or numpy.load. - | - | Parameters - | ---------- - | file : str or Path - | A string naming the dump file. - | - | .. versionchanged:: 1.17.0 - | `pathlib.Path` objects are now accepted. - | - | dumps(...) - | a.dumps() - | - | Returns the pickle of the array as a string. - | pickle.loads will convert the string back to an array. - | - | Parameters - | ---------- - | None - | - | fill(...) - | a.fill(value) - | - | Fill the array with a scalar value. - | - | Parameters - | ---------- - | value : scalar - | All elements of `a` will be assigned this value. - | - | Examples - | -------- - | >>> a = np.array([1, 2]) - | >>> a.fill(0) - | >>> a - | array([0, 0]) - | >>> a = np.empty(2) - | >>> a.fill(1) - | >>> a - | array([1., 1.]) - | - | Fill expects a scalar value and always behaves the same as assigning - | to a single array element. The following is a rare example where this - | distinction is important: - | - | >>> a = np.array([None, None], dtype=object) - | >>> a[0] = np.array(3) - | >>> a - | array([array(3), None], dtype=object) - | >>> a.fill(np.array(3)) - | >>> a - | array([array(3), array(3)], dtype=object) - | - | Where other forms of assignments will unpack the array being assigned: - | - | >>> a[...] = np.array(3) - | >>> a - | array([3, 3], dtype=object) - | - | flatten(...) - | a.flatten(order='C') - | - | Return a copy of the array collapsed into one dimension. - | - | Parameters - | ---------- - | order : {'C', 'F', 'A', 'K'}, optional - | 'C' means to flatten in row-major (C-style) order. - | 'F' means to flatten in column-major (Fortran- - | style) order. 'A' means to flatten in column-major - | order if `a` is Fortran *contiguous* in memory, - | row-major order otherwise. 'K' means to flatten - | `a` in the order the elements occur in memory. - | The default is 'C'. - | - | Returns - | ------- - | y : ndarray - | A copy of the input array, flattened to one dimension. - | - | See Also - | -------- - | ravel : Return a flattened array. - | flat : A 1-D flat iterator over the array. - | - | Examples - | -------- - | >>> a = np.array([[1,2], [3,4]]) - | >>> a.flatten() - | array([1, 2, 3, 4]) - | >>> a.flatten('F') - | array([1, 3, 2, 4]) - | - | getfield(...) - | a.getfield(dtype, offset=0) - | - | Returns a field of the given array as a certain type. - | - | A field is a view of the array data with a given data-type. The values in - | the view are determined by the given type and the offset into the current - | array in bytes. The offset needs to be such that the view dtype fits in the - | array dtype; for example an array of dtype complex128 has 16-byte elements. - | If taking a view with a 32-bit integer (4 bytes), the offset needs to be - | between 0 and 12 bytes. - | - | Parameters - | ---------- - | dtype : str or dtype - | The data type of the view. The dtype size of the view can not be larger - | than that of the array itself. - | offset : int - | Number of bytes to skip before beginning the element view. - | - | Examples - | -------- - | >>> x = np.diag([1.+1.j]*2) - | >>> x[1, 1] = 2 + 4.j - | >>> x - | array([[1.+1.j, 0.+0.j], - | [0.+0.j, 2.+4.j]]) - | >>> x.getfield(np.float64) - | array([[1., 0.], - | [0., 2.]]) - | - | By choosing an offset of 8 bytes we can select the complex part of the - | array for our view: - | - | >>> x.getfield(np.float64, offset=8) - | array([[1., 0.], - | [0., 4.]]) - | - | item(...) - | a.item(*args) - | - | Copy an element of an array to a standard Python scalar and return it. - | - | Parameters - | ---------- - | \*args : Arguments (variable number and type) - | - | * none: in this case, the method only works for arrays - | with one element (`a.size == 1`), which element is - | copied into a standard Python scalar object and returned. - | - | * int_type: this argument is interpreted as a flat index into - | the array, specifying which element to copy and return. - | - | * tuple of int_types: functions as does a single int_type argument, - | except that the argument is interpreted as an nd-index into the - | array. - | - | Returns - | ------- - | z : Standard Python scalar object - | A copy of the specified element of the array as a suitable - | Python scalar - | - | Notes - | ----- - | When the data type of `a` is longdouble or clongdouble, item() returns - | a scalar array object because there is no available Python scalar that - | would not lose information. Void arrays return a buffer object for item(), - | unless fields are defined, in which case a tuple is returned. - | - | `item` is very similar to a[args], except, instead of an array scalar, - | a standard Python scalar is returned. This can be useful for speeding up - | access to elements of the array and doing arithmetic on elements of the - | array using Python's optimized math. - | - | Examples - | -------- - | >>> np.random.seed(123) - | >>> x = np.random.randint(9, size=(3, 3)) - | >>> x - | array([[2, 2, 6], - | [1, 3, 6], - | [1, 0, 1]]) - | >>> x.item(3) - | 1 - | >>> x.item(7) - | 0 - | >>> x.item((0, 1)) - | 2 - | >>> x.item((2, 2)) - | 1 - | - | itemset(...) - | a.itemset(*args) - | - | Insert scalar into an array (scalar is cast to array's dtype, if possible) - | - | There must be at least 1 argument, and define the last argument - | as *item*. Then, ``a.itemset(*args)`` is equivalent to but faster - | than ``a[args] = item``. The item should be a scalar value and `args` - | must select a single item in the array `a`. - | - | Parameters - | ---------- - | \*args : Arguments - | If one argument: a scalar, only used in case `a` is of size 1. - | If two arguments: the last argument is the value to be set - | and must be a scalar, the first argument specifies a single array - | element location. It is either an int or a tuple. - | - | Notes - | ----- - | Compared to indexing syntax, `itemset` provides some speed increase - | for placing a scalar into a particular location in an `ndarray`, - | if you must do this. However, generally this is discouraged: - | among other problems, it complicates the appearance of the code. - | Also, when using `itemset` (and `item`) inside a loop, be sure - | to assign the methods to a local variable to avoid the attribute - | look-up at each loop iteration. - | - | Examples - | -------- - | >>> np.random.seed(123) - | >>> x = np.random.randint(9, size=(3, 3)) - | >>> x - | array([[2, 2, 6], - | [1, 3, 6], - | [1, 0, 1]]) - | >>> x.itemset(4, 0) - | >>> x.itemset((2, 2), 9) - | >>> x - | array([[2, 2, 6], - | [1, 0, 6], - | [1, 0, 9]]) - | - | max(...) - | a.max(axis=None, out=None, keepdims=False, initial=, where=True) - | - | Return the maximum along a given axis. - | - | Refer to `numpy.amax` for full documentation. - | - | See Also - | -------- - | numpy.amax : equivalent function - | - | mean(...) - | a.mean(axis=None, dtype=None, out=None, keepdims=False, *, where=True) - | - | Returns the average of the array elements along given axis. - | - | Refer to `numpy.mean` for full documentation. - | - | See Also - | -------- - | numpy.mean : equivalent function - | - | min(...) - | a.min(axis=None, out=None, keepdims=False, initial=, where=True) - | - | Return the minimum along a given axis. - | - | Refer to `numpy.amin` for full documentation. - | - | See Also - | -------- - | numpy.amin : equivalent function - | - | newbyteorder(...) - | arr.newbyteorder(new_order='S', /) - | - | Return the array with the same data viewed with a different byte order. - | - | Equivalent to:: - | - | arr.view(arr.dtype.newbytorder(new_order)) - | - | Changes are also made in all fields and sub-arrays of the array data - | type. - | - | - | - | Parameters - | ---------- - | new_order : string, optional - | Byte order to force; a value from the byte order specifications - | below. `new_order` codes can be any of: - | - | * 'S' - swap dtype from current to opposite endian - | * {'<', 'little'} - little endian - | * {'>', 'big'} - big endian - | * {'=', 'native'} - native order, equivalent to `sys.byteorder` - | * {'|', 'I'} - ignore (no change to byte order) - | - | The default value ('S') results in swapping the current - | byte order. - | - | - | Returns - | ------- - | new_arr : array - | New array object with the dtype reflecting given change to the - | byte order. - | - | nonzero(...) - | a.nonzero() - | - | Return the indices of the elements that are non-zero. - | - | Refer to `numpy.nonzero` for full documentation. - | - | See Also - | -------- - | numpy.nonzero : equivalent function - | - | partition(...) - | a.partition(kth, axis=-1, kind='introselect', order=None) - | - | Rearranges the elements in the array in such a way that the value of the - | element in kth position is in the position it would be in a sorted array. - | All elements smaller than the kth element are moved before this element and - | all equal or greater are moved behind it. The ordering of the elements in - | the two partitions is undefined. - | - | .. versionadded:: 1.8.0 - | - | Parameters - | ---------- - | kth : int or sequence of ints - | Element index to partition by. The kth element value will be in its - | final sorted position and all smaller elements will be moved before it - | and all equal or greater elements behind it. - | The order of all elements in the partitions is undefined. - | If provided with a sequence of kth it will partition all elements - | indexed by kth of them into their sorted position at once. - | - | .. deprecated:: 1.22.0 - | Passing booleans as index is deprecated. - | axis : int, optional - | Axis along which to sort. Default is -1, which means sort along the - | last axis. - | kind : {'introselect'}, optional - | Selection algorithm. Default is 'introselect'. - | order : str or list of str, optional - | When `a` is an array with fields defined, this argument specifies - | which fields to compare first, second, etc. A single field can - | be specified as a string, and not all fields need to be specified, - | but unspecified fields will still be used, in the order in which - | they come up in the dtype, to break ties. - | - | See Also - | -------- - | numpy.partition : Return a partitioned copy of an array. - | argpartition : Indirect partition. - | sort : Full sort. - | - | Notes - | ----- - | See ``np.partition`` for notes on the different algorithms. - | - | Examples - | -------- - | >>> a = np.array([3, 4, 2, 1]) - | >>> a.partition(3) - | >>> a - | array([2, 1, 3, 4]) - | - | >>> a.partition((1, 3)) - | >>> a - | array([1, 2, 3, 4]) - | - | prod(...) - | a.prod(axis=None, dtype=None, out=None, keepdims=False, initial=1, where=True) - | - | Return the product of the array elements over the given axis - | - | Refer to `numpy.prod` for full documentation. - | - | See Also - | -------- - | numpy.prod : equivalent function - | - | ptp(...) - | a.ptp(axis=None, out=None, keepdims=False) - | - | Peak to peak (maximum - minimum) value along a given axis. - | - | Refer to `numpy.ptp` for full documentation. - | - | See Also - | -------- - | numpy.ptp : equivalent function - | - | put(...) - | a.put(indices, values, mode='raise') - | - | Set ``a.flat[n] = values[n]`` for all `n` in indices. - | - | Refer to `numpy.put` for full documentation. - | - | See Also - | -------- - | numpy.put : equivalent function - | - | ravel(...) - | a.ravel([order]) - | - | Return a flattened array. - | - | Refer to `numpy.ravel` for full documentation. - | - | See Also - | -------- - | numpy.ravel : equivalent function - | - | ndarray.flat : a flat iterator on the array. - | - | repeat(...) - | a.repeat(repeats, axis=None) - | - | Repeat elements of an array. - | - | Refer to `numpy.repeat` for full documentation. - | - | See Also - | -------- - | numpy.repeat : equivalent function - | - | reshape(...) - | a.reshape(shape, order='C') - | - | Returns an array containing the same data with a new shape. - | - | Refer to `numpy.reshape` for full documentation. - | - | See Also - | -------- - | numpy.reshape : equivalent function - | - | Notes - | ----- - | Unlike the free function `numpy.reshape`, this method on `ndarray` allows - | the elements of the shape parameter to be passed in as separate arguments. - | For example, ``a.reshape(10, 11)`` is equivalent to - | ``a.reshape((10, 11))``. - | - | resize(...) - | a.resize(new_shape, refcheck=True) - | - | Change shape and size of array in-place. - | - | Parameters - | ---------- - | new_shape : tuple of ints, or `n` ints - | Shape of resized array. - | refcheck : bool, optional - | If False, reference count will not be checked. Default is True. - | - | Returns - | ------- - | None - | - | Raises - | ------ - | ValueError - | If `a` does not own its own data or references or views to it exist, - | and the data memory must be changed. - | PyPy only: will always raise if the data memory must be changed, since - | there is no reliable way to determine if references or views to it - | exist. - | - | SystemError - | If the `order` keyword argument is specified. This behaviour is a - | bug in NumPy. - | - | See Also - | -------- - | resize : Return a new array with the specified shape. - | - | Notes - | ----- - | This reallocates space for the data area if necessary. - | - | Only contiguous arrays (data elements consecutive in memory) can be - | resized. - | - | The purpose of the reference count check is to make sure you - | do not use this array as a buffer for another Python object and then - | reallocate the memory. However, reference counts can increase in - | other ways so if you are sure that you have not shared the memory - | for this array with another Python object, then you may safely set - | `refcheck` to False. - | - | Examples - | -------- - | Shrinking an array: array is flattened (in the order that the data are - | stored in memory), resized, and reshaped: - | - | >>> a = np.array([[0, 1], [2, 3]], order='C') - | >>> a.resize((2, 1)) - | >>> a - | array([[0], - | [1]]) - | - | >>> a = np.array([[0, 1], [2, 3]], order='F') - | >>> a.resize((2, 1)) - | >>> a - | array([[0], - | [2]]) - | - | Enlarging an array: as above, but missing entries are filled with zeros: - | - | >>> b = np.array([[0, 1], [2, 3]]) - | >>> b.resize(2, 3) # new_shape parameter doesn't have to be a tuple - | >>> b - | array([[0, 1, 2], - | [3, 0, 0]]) - | - | Referencing an array prevents resizing... - | - | >>> c = a - | >>> a.resize((1, 1)) - | Traceback (most recent call last): - | ... - | ValueError: cannot resize an array that references or is referenced ... - | - | Unless `refcheck` is False: - | - | >>> a.resize((1, 1), refcheck=False) - | >>> a - | array([[0]]) - | >>> c - | array([[0]]) - | - | round(...) - | a.round(decimals=0, out=None) - | - | Return `a` with each element rounded to the given number of decimals. - | - | Refer to `numpy.around` for full documentation. - | - | See Also - | -------- - | numpy.around : equivalent function - | - | searchsorted(...) - | a.searchsorted(v, side='left', sorter=None) - | - | Find indices where elements of v should be inserted in a to maintain order. - | - | For full documentation, see `numpy.searchsorted` - | - | See Also - | -------- - | numpy.searchsorted : equivalent function - | - | setfield(...) - | a.setfield(val, dtype, offset=0) - | - | Put a value into a specified place in a field defined by a data-type. - | - | Place `val` into `a`'s field defined by `dtype` and beginning `offset` - | bytes into the field. - | - | Parameters - | ---------- - | val : object - | Value to be placed in field. - | dtype : dtype object - | Data-type of the field in which to place `val`. - | offset : int, optional - | The number of bytes into the field at which to place `val`. - | - | Returns - | ------- - | None - | - | See Also - | -------- - | getfield - | - | Examples - | -------- - | >>> x = np.eye(3) - | >>> x.getfield(np.float64) - | array([[1., 0., 0.], - | [0., 1., 0.], - | [0., 0., 1.]]) - | >>> x.setfield(3, np.int32) - | >>> x.getfield(np.int32) - | array([[3, 3, 3], - | [3, 3, 3], - | [3, 3, 3]], dtype=int32) - | >>> x - | array([[1.0e+000, 1.5e-323, 1.5e-323], - | [1.5e-323, 1.0e+000, 1.5e-323], - | [1.5e-323, 1.5e-323, 1.0e+000]]) - | >>> x.setfield(np.eye(3), np.int32) - | >>> x - | array([[1., 0., 0.], - | [0., 1., 0.], - | [0., 0., 1.]]) - | - | setflags(...) - | a.setflags(write=None, align=None, uic=None) - | - | Set array flags WRITEABLE, ALIGNED, WRITEBACKIFCOPY, - | respectively. - | - | These Boolean-valued flags affect how numpy interprets the memory - | area used by `a` (see Notes below). The ALIGNED flag can only - | be set to True if the data is actually aligned according to the type. - | The WRITEBACKIFCOPY and flag can never be set - | to True. The flag WRITEABLE can only be set to True if the array owns its - | own memory, or the ultimate owner of the memory exposes a writeable buffer - | interface, or is a string. (The exception for string is made so that - | unpickling can be done without copying memory.) - | - | Parameters - | ---------- - | write : bool, optional - | Describes whether or not `a` can be written to. - | align : bool, optional - | Describes whether or not `a` is aligned properly for its type. - | uic : bool, optional - | Describes whether or not `a` is a copy of another "base" array. - | - | Notes - | ----- - | Array flags provide information about how the memory area used - | for the array is to be interpreted. There are 7 Boolean flags - | in use, only four of which can be changed by the user: - | WRITEBACKIFCOPY, WRITEABLE, and ALIGNED. - | - | WRITEABLE (W) the data area can be written to; - | - | ALIGNED (A) the data and strides are aligned appropriately for the hardware - | (as determined by the compiler); - | - | WRITEBACKIFCOPY (X) this array is a copy of some other array (referenced - | by .base). When the C-API function PyArray_ResolveWritebackIfCopy is - | called, the base array will be updated with the contents of this array. - | - | All flags can be accessed using the single (upper case) letter as well - | as the full name. - | - | Examples - | -------- - | >>> y = np.array([[3, 1, 7], - | ... [2, 0, 0], - | ... [8, 5, 9]]) - | >>> y - | array([[3, 1, 7], - | [2, 0, 0], - | [8, 5, 9]]) - | >>> y.flags - | C_CONTIGUOUS : True - | F_CONTIGUOUS : False - | OWNDATA : True - | WRITEABLE : True - | ALIGNED : True - | WRITEBACKIFCOPY : False - | >>> y.setflags(write=0, align=0) - | >>> y.flags - | C_CONTIGUOUS : True - | F_CONTIGUOUS : False - | OWNDATA : True - | WRITEABLE : False - | ALIGNED : False - | WRITEBACKIFCOPY : False - | >>> y.setflags(uic=1) - | Traceback (most recent call last): - | File "", line 1, in - | ValueError: cannot set WRITEBACKIFCOPY flag to True - | - | sort(...) - | a.sort(axis=-1, kind=None, order=None) - | - | Sort an array in-place. Refer to `numpy.sort` for full documentation. - | - | Parameters - | ---------- - | axis : int, optional - | Axis along which to sort. Default is -1, which means sort along the - | last axis. - | kind : {'quicksort', 'mergesort', 'heapsort', 'stable'}, optional - | Sorting algorithm. The default is 'quicksort'. Note that both 'stable' - | and 'mergesort' use timsort under the covers and, in general, the - | actual implementation will vary with datatype. The 'mergesort' option - | is retained for backwards compatibility. - | - | .. versionchanged:: 1.15.0 - | The 'stable' option was added. - | - | order : str or list of str, optional - | When `a` is an array with fields defined, this argument specifies - | which fields to compare first, second, etc. A single field can - | be specified as a string, and not all fields need be specified, - | but unspecified fields will still be used, in the order in which - | they come up in the dtype, to break ties. - | - | See Also - | -------- - | numpy.sort : Return a sorted copy of an array. - | numpy.argsort : Indirect sort. - | numpy.lexsort : Indirect stable sort on multiple keys. - | numpy.searchsorted : Find elements in sorted array. - | numpy.partition: Partial sort. - | - | Notes - | ----- - | See `numpy.sort` for notes on the different sorting algorithms. - | - | Examples - | -------- - | >>> a = np.array([[1,4], [3,1]]) - | >>> a.sort(axis=1) - | >>> a - | array([[1, 4], - | [1, 3]]) - | >>> a.sort(axis=0) - | >>> a - | array([[1, 3], - | [1, 4]]) - | - | Use the `order` keyword to specify a field to use when sorting a - | structured array: - | - | >>> a = np.array([('a', 2), ('c', 1)], dtype=[('x', 'S1'), ('y', int)]) - | >>> a.sort(order='y') - | >>> a - | array([(b'c', 1), (b'a', 2)], - | dtype=[('x', 'S1'), ('y', '>> x = np.array([[0, 1], [2, 3]], dtype='>> x.tobytes() - | b'\x00\x00\x01\x00\x02\x00\x03\x00' - | >>> x.tobytes('C') == x.tobytes() - | True - | >>> x.tobytes('F') - | b'\x00\x00\x02\x00\x01\x00\x03\x00' - | - | tofile(...) - | a.tofile(fid, sep="", format="%s") - | - | Write array to a file as text or binary (default). - | - | Data is always written in 'C' order, independent of the order of `a`. - | The data produced by this method can be recovered using the function - | fromfile(). - | - | Parameters - | ---------- - | fid : file or str or Path - | An open file object, or a string containing a filename. - | - | .. versionchanged:: 1.17.0 - | `pathlib.Path` objects are now accepted. - | - | sep : str - | Separator between array items for text output. - | If "" (empty), a binary file is written, equivalent to - | ``file.write(a.tobytes())``. - | format : str - | Format string for text file output. - | Each entry in the array is formatted to text by first converting - | it to the closest Python type, and then using "format" % item. - | - | Notes - | ----- - | This is a convenience function for quick storage of array data. - | Information on endianness and precision is lost, so this method is not a - | good choice for files intended to archive data or transport data between - | machines with different endianness. Some of these problems can be overcome - | by outputting the data as text files, at the expense of speed and file - | size. - | - | When fid is a file object, array contents are directly written to the - | file, bypassing the file object's ``write`` method. As a result, tofile - | cannot be used with files objects supporting compression (e.g., GzipFile) - | or file-like objects that do not support ``fileno()`` (e.g., BytesIO). - | - | tolist(...) - | a.tolist() - | - | Return the array as an ``a.ndim``-levels deep nested list of Python scalars. - | - | Return a copy of the array data as a (nested) Python list. - | Data items are converted to the nearest compatible builtin Python type, via - | the `~numpy.ndarray.item` function. - | - | If ``a.ndim`` is 0, then since the depth of the nested list is 0, it will - | not be a list at all, but a simple Python scalar. - | - | Parameters - | ---------- - | none - | - | Returns - | ------- - | y : object, or list of object, or list of list of object, or ... - | The possibly nested list of array elements. - | - | Notes - | ----- - | The array may be recreated via ``a = np.array(a.tolist())``, although this - | may sometimes lose precision. - | - | Examples - | -------- - | For a 1D array, ``a.tolist()`` is almost the same as ``list(a)``, - | except that ``tolist`` changes numpy scalars to Python scalars: - | - | >>> a = np.uint32([1, 2]) - | >>> a_list = list(a) - | >>> a_list - | [1, 2] - | >>> type(a_list[0]) - | - | >>> a_tolist = a.tolist() - | >>> a_tolist - | [1, 2] - | >>> type(a_tolist[0]) - | - | - | Additionally, for a 2D array, ``tolist`` applies recursively: - | - | >>> a = np.array([[1, 2], [3, 4]]) - | >>> list(a) - | [array([1, 2]), array([3, 4])] - | >>> a.tolist() - | [[1, 2], [3, 4]] - | - | The base case for this recursion is a 0D array: - | - | >>> a = np.array(1) - | >>> list(a) - | Traceback (most recent call last): - | ... - | TypeError: iteration over a 0-d array - | >>> a.tolist() - | 1 - | - | tostring(...) - | a.tostring(order='C') - | - | A compatibility alias for `tobytes`, with exactly the same behavior. - | - | Despite its name, it returns `bytes` not `str`\ s. - | - | .. deprecated:: 1.19.0 - | - | trace(...) - | a.trace(offset=0, axis1=0, axis2=1, dtype=None, out=None) - | - | Return the sum along diagonals of the array. - | - | Refer to `numpy.trace` for full documentation. - | - | See Also - | -------- - | numpy.trace : equivalent function - | - | transpose(...) - | a.transpose(*axes) - | - | Returns a view of the array with axes transposed. - | - | Refer to `numpy.transpose` for full documentation. - | - | Parameters - | ---------- - | axes : None, tuple of ints, or `n` ints - | - | * None or no argument: reverses the order of the axes. - | - | * tuple of ints: `i` in the `j`-th place in the tuple means that the - | array's `i`-th axis becomes the transposed array's `j`-th axis. - | - | * `n` ints: same as an n-tuple of the same ints (this form is - | intended simply as a "convenience" alternative to the tuple form). - | - | Returns - | ------- - | p : ndarray - | View of the array with its axes suitably permuted. - | - | See Also - | -------- - | transpose : Equivalent function. - | ndarray.T : Array property returning the array transposed. - | ndarray.reshape : Give a new shape to an array without changing its data. - | - | Examples - | -------- - | >>> a = np.array([[1, 2], [3, 4]]) - | >>> a - | array([[1, 2], - | [3, 4]]) - | >>> a.transpose() - | array([[1, 3], - | [2, 4]]) - | >>> a.transpose((1, 0)) - | array([[1, 3], - | [2, 4]]) - | >>> a.transpose(1, 0) - | array([[1, 3], - | [2, 4]]) - | - | >>> a = np.array([1, 2, 3, 4]) - | >>> a - | array([1, 2, 3, 4]) - | >>> a.transpose() - | array([1, 2, 3, 4]) - | - | var(...) - | a.var(axis=None, dtype=None, out=None, ddof=0, keepdims=False, *, where=True) - | - | Returns the variance of the array elements, along given axis. - | - | Refer to `numpy.var` for full documentation. - | - | See Also - | -------- - | numpy.var : equivalent function - | - | view(...) - | a.view([dtype][, type]) - | - | New view of array with the same data. - | - | .. note:: - | Passing None for ``dtype`` is different from omitting the parameter, - | since the former invokes ``dtype(None)`` which is an alias for - | ``dtype('float_')``. - | - | Parameters - | ---------- - | dtype : data-type or ndarray sub-class, optional - | Data-type descriptor of the returned view, e.g., float32 or int16. - | Omitting it results in the view having the same data-type as `a`. - | This argument can also be specified as an ndarray sub-class, which - | then specifies the type of the returned object (this is equivalent to - | setting the ``type`` parameter). - | type : Python type, optional - | Type of the returned view, e.g., ndarray or matrix. Again, omission - | of the parameter results in type preservation. - | - | Notes - | ----- - | ``a.view()`` is used two different ways: - | - | ``a.view(some_dtype)`` or ``a.view(dtype=some_dtype)`` constructs a view - | of the array's memory with a different data-type. This can cause a - | reinterpretation of the bytes of memory. - | - | ``a.view(ndarray_subclass)`` or ``a.view(type=ndarray_subclass)`` just - | returns an instance of `ndarray_subclass` that looks at the same array - | (same shape, dtype, etc.) This does not cause a reinterpretation of the - | memory. - | - | For ``a.view(some_dtype)``, if ``some_dtype`` has a different number of - | bytes per entry than the previous dtype (for example, converting a regular - | array to a structured array), then the last axis of ``a`` must be - | contiguous. This axis will be resized in the result. - | - | .. versionchanged:: 1.23.0 - | Only the last axis needs to be contiguous. Previously, the entire array - | had to be C-contiguous. - | - | Examples - | -------- - | >>> x = np.array([(1, 2)], dtype=[('a', np.int8), ('b', np.int8)]) - | - | Viewing array data using a different type and dtype: - | - | >>> y = x.view(dtype=np.int16, type=np.matrix) - | >>> y - | matrix([[513]], dtype=int16) - | >>> print(type(y)) - | - | - | Creating a view on a structured array so it can be used in calculations - | - | >>> x = np.array([(1, 2),(3,4)], dtype=[('a', np.int8), ('b', np.int8)]) - | >>> xv = x.view(dtype=np.int8).reshape(-1,2) - | >>> xv - | array([[1, 2], - | [3, 4]], dtype=int8) - | >>> xv.mean(0) - | array([2., 3.]) - | - | Making changes to the view changes the underlying array - | - | >>> xv[0,1] = 20 - | >>> x - | array([(1, 20), (3, 4)], dtype=[('a', 'i1'), ('b', 'i1')]) - | - | Using a view to convert an array to a recarray: - | - | >>> z = x.view(np.recarray) - | >>> z.a - | array([1, 3], dtype=int8) - | - | Views share data: - | - | >>> x[0] = (9, 10) - | >>> z[0] - | (9, 10) - | - | Views that change the dtype size (bytes per entry) should normally be - | avoided on arrays defined by slices, transposes, fortran-ordering, etc.: - | - | >>> x = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int16) - | >>> y = x[:, ::2] - | >>> y - | array([[1, 3], - | [4, 6]], dtype=int16) - | >>> y.view(dtype=[('width', np.int16), ('length', np.int16)]) - | Traceback (most recent call last): - | ... - | ValueError: To change to a dtype of a different size, the last axis must be contiguous - | >>> z = y.copy() - | >>> z.view(dtype=[('width', np.int16), ('length', np.int16)]) - | array([[(1, 3)], - | [(4, 6)]], dtype=[('width', '>> x = np.arange(2 * 3 * 4, dtype=np.int8).reshape(2, 3, 4) - | >>> x.transpose(1, 0, 2).view(np.int16) - | array([[[ 256, 770], - | [3340, 3854]], - | - | [[1284, 1798], - | [4368, 4882]], - | - | [[2312, 2826], - | [5396, 5910]]], dtype=int16) - | - | ---------------------------------------------------------------------- - | Class methods defined here: - | - | __class_getitem__(...) from builtins.type - | a.__class_getitem__(item, /) - | - | Return a parametrized wrapper around the `~numpy.ndarray` type. - | - | .. versionadded:: 1.22 - | - | Returns - | ------- - | alias : types.GenericAlias - | A parametrized `~numpy.ndarray` type. - | - | Examples - | -------- - | >>> from typing import Any - | >>> import numpy as np - | - | >>> np.ndarray[Any, np.dtype[Any]] - | numpy.ndarray[typing.Any, numpy.dtype[typing.Any]] - | - | Notes - | ----- - | This method is only available for python 3.9 and later. - | - | See Also - | -------- - | :pep:`585` : Type hinting generics in standard collections. - | numpy.typing.NDArray : An ndarray alias :term:`generic ` - | w.r.t. its `dtype.type `. - | - | ---------------------------------------------------------------------- - | Static methods defined here: - | - | __new__(*args, **kwargs) from builtins.type - | Create and return a new object. See help(type) for accurate signature. - | - | ---------------------------------------------------------------------- - | Data descriptors defined here: - | - | T - | View of the transposed array. - | - | Same as ``self.transpose()``. - | - | Examples - | -------- - | >>> a = np.array([[1, 2], [3, 4]]) - | >>> a - | array([[1, 2], - | [3, 4]]) - | >>> a.T - | array([[1, 3], - | [2, 4]]) - | - | >>> a = np.array([1, 2, 3, 4]) - | >>> a - | array([1, 2, 3, 4]) - | >>> a.T - | array([1, 2, 3, 4]) - | - | See Also - | -------- - | transpose - | - | __array_interface__ - | Array protocol: Python side. - | - | __array_priority__ - | Array priority. - | - | __array_struct__ - | Array protocol: C-struct side. - | - | base - | Base object if memory is from some other object. - | - | Examples - | -------- - | The base of an array that owns its memory is None: - | - | >>> x = np.array([1,2,3,4]) - | >>> x.base is None - | True - | - | Slicing creates a view, whose memory is shared with x: - | - | >>> y = x[2:] - | >>> y.base is x - | True - | - | ctypes - | An object to simplify the interaction of the array with the ctypes - | module. - | - | This attribute creates an object that makes it easier to use arrays - | when calling shared libraries with the ctypes module. The returned - | object has, among others, data, shape, and strides attributes (see - | Notes below) which themselves return ctypes objects that can be used - | as arguments to a shared library. - | - | Parameters - | ---------- - | None - | - | Returns - | ------- - | c : Python object - | Possessing attributes data, shape, strides, etc. - | - | See Also - | -------- - | numpy.ctypeslib - | - | Notes - | ----- - | Below are the public attributes of this object which were documented - | in "Guide to NumPy" (we have omitted undocumented public attributes, - | as well as documented private attributes): - | - | .. autoattribute:: numpy.core._internal._ctypes.data - | :noindex: - | - | .. autoattribute:: numpy.core._internal._ctypes.shape - | :noindex: - | - | .. autoattribute:: numpy.core._internal._ctypes.strides - | :noindex: - | - | .. automethod:: numpy.core._internal._ctypes.data_as - | :noindex: - | - | .. automethod:: numpy.core._internal._ctypes.shape_as - | :noindex: - | - | .. automethod:: numpy.core._internal._ctypes.strides_as - | :noindex: - | - | If the ctypes module is not available, then the ctypes attribute - | of array objects still returns something useful, but ctypes objects - | are not returned and errors may be raised instead. In particular, - | the object will still have the ``as_parameter`` attribute which will - | return an integer equal to the data attribute. - | - | Examples - | -------- - | >>> import ctypes - | >>> x = np.array([[0, 1], [2, 3]], dtype=np.int32) - | >>> x - | array([[0, 1], - | [2, 3]], dtype=int32) - | >>> x.ctypes.data - | 31962608 # may vary - | >>> x.ctypes.data_as(ctypes.POINTER(ctypes.c_uint32)) - | <__main__.LP_c_uint object at 0x7ff2fc1fc200> # may vary - | >>> x.ctypes.data_as(ctypes.POINTER(ctypes.c_uint32)).contents - | c_uint(0) - | >>> x.ctypes.data_as(ctypes.POINTER(ctypes.c_uint64)).contents - | c_ulong(4294967296) - | >>> x.ctypes.shape - | # may vary - | >>> x.ctypes.strides - | # may vary - | - | data - | Python buffer object pointing to the start of the array's data. - | - | dtype - | Data-type of the array's elements. - | - | .. warning:: - | - | Setting ``arr.dtype`` is discouraged and may be deprecated in the - | future. Setting will replace the ``dtype`` without modifying the - | memory (see also `ndarray.view` and `ndarray.astype`). - | - | Parameters - | ---------- - | None - | - | Returns - | ------- - | d : numpy dtype object - | - | See Also - | -------- - | ndarray.astype : Cast the values contained in the array to a new data-type. - | ndarray.view : Create a view of the same data but a different data-type. - | numpy.dtype - | - | Examples - | -------- - | >>> x - | array([[0, 1], - | [2, 3]]) - | >>> x.dtype - | dtype('int32') - | >>> type(x.dtype) - | - | - | flags - | Information about the memory layout of the array. - | - | Attributes - | ---------- - | C_CONTIGUOUS (C) - | The data is in a single, C-style contiguous segment. - | F_CONTIGUOUS (F) - | The data is in a single, Fortran-style contiguous segment. - | OWNDATA (O) - | The array owns the memory it uses or borrows it from another object. - | WRITEABLE (W) - | The data area can be written to. Setting this to False locks - | the data, making it read-only. A view (slice, etc.) inherits WRITEABLE - | from its base array at creation time, but a view of a writeable - | array may be subsequently locked while the base array remains writeable. - | (The opposite is not true, in that a view of a locked array may not - | be made writeable. However, currently, locking a base object does not - | lock any views that already reference it, so under that circumstance it - | is possible to alter the contents of a locked array via a previously - | created writeable view onto it.) Attempting to change a non-writeable - | array raises a RuntimeError exception. - | ALIGNED (A) - | The data and all elements are aligned appropriately for the hardware. - | WRITEBACKIFCOPY (X) - | This array is a copy of some other array. The C-API function - | PyArray_ResolveWritebackIfCopy must be called before deallocating - | to the base array will be updated with the contents of this array. - | FNC - | F_CONTIGUOUS and not C_CONTIGUOUS. - | FORC - | F_CONTIGUOUS or C_CONTIGUOUS (one-segment test). - | BEHAVED (B) - | ALIGNED and WRITEABLE. - | CARRAY (CA) - | BEHAVED and C_CONTIGUOUS. - | FARRAY (FA) - | BEHAVED and F_CONTIGUOUS and not C_CONTIGUOUS. - | - | Notes - | ----- - | The `flags` object can be accessed dictionary-like (as in ``a.flags['WRITEABLE']``), - | or by using lowercased attribute names (as in ``a.flags.writeable``). Short flag - | names are only supported in dictionary access. - | - | Only the WRITEBACKIFCOPY, WRITEABLE, and ALIGNED flags can be - | changed by the user, via direct assignment to the attribute or dictionary - | entry, or by calling `ndarray.setflags`. - | - | The array flags cannot be set arbitrarily: - | - | - WRITEBACKIFCOPY can only be set ``False``. - | - ALIGNED can only be set ``True`` if the data is truly aligned. - | - WRITEABLE can only be set ``True`` if the array owns its own memory - | or the ultimate owner of the memory exposes a writeable buffer - | interface or is a string. - | - | Arrays can be both C-style and Fortran-style contiguous simultaneously. - | This is clear for 1-dimensional arrays, but can also be true for higher - | dimensional arrays. - | - | Even for contiguous arrays a stride for a given dimension - | ``arr.strides[dim]`` may be *arbitrary* if ``arr.shape[dim] == 1`` - | or the array has no elements. - | It does *not* generally hold that ``self.strides[-1] == self.itemsize`` - | for C-style contiguous arrays or ``self.strides[0] == self.itemsize`` for - | Fortran-style contiguous arrays is true. - | - | flat - | A 1-D iterator over the array. - | - | This is a `numpy.flatiter` instance, which acts similarly to, but is not - | a subclass of, Python's built-in iterator object. - | - | See Also - | -------- - | flatten : Return a copy of the array collapsed into one dimension. - | - | flatiter - | - | Examples - | -------- - | >>> x = np.arange(1, 7).reshape(2, 3) - | >>> x - | array([[1, 2, 3], - | [4, 5, 6]]) - | >>> x.flat[3] - | 4 - | >>> x.T - | array([[1, 4], - | [2, 5], - | [3, 6]]) - | >>> x.T.flat[3] - | 5 - | >>> type(x.flat) - | - | - | An assignment example: - | - | >>> x.flat = 3; x - | array([[3, 3, 3], - | [3, 3, 3]]) - | >>> x.flat[[1,4]] = 1; x - | array([[3, 1, 3], - | [3, 1, 3]]) - | - | imag - | The imaginary part of the array. - | - | Examples - | -------- - | >>> x = np.sqrt([1+0j, 0+1j]) - | >>> x.imag - | array([ 0. , 0.70710678]) - | >>> x.imag.dtype - | dtype('float64') - | - | itemsize - | Length of one array element in bytes. - | - | Examples - | -------- - | >>> x = np.array([1,2,3], dtype=np.float64) - | >>> x.itemsize - | 8 - | >>> x = np.array([1,2,3], dtype=np.complex128) - | >>> x.itemsize - | 16 - | - | nbytes - | Total bytes consumed by the elements of the array. - | - | Notes - | ----- - | Does not include memory consumed by non-element attributes of the - | array object. - | - | Examples - | -------- - | >>> x = np.zeros((3,5,2), dtype=np.complex128) - | >>> x.nbytes - | 480 - | >>> np.prod(x.shape) * x.itemsize - | 480 - | - | ndim - | Number of array dimensions. - | - | Examples - | -------- - | >>> x = np.array([1, 2, 3]) - | >>> x.ndim - | 1 - | >>> y = np.zeros((2, 3, 4)) - | >>> y.ndim - | 3 - | - | real - | The real part of the array. - | - | Examples - | -------- - | >>> x = np.sqrt([1+0j, 0+1j]) - | >>> x.real - | array([ 1. , 0.70710678]) - | >>> x.real.dtype - | dtype('float64') - | - | See Also - | -------- - | numpy.real : equivalent function - | - | shape - | Tuple of array dimensions. - | - | The shape property is usually used to get the current shape of an array, - | but may also be used to reshape the array in-place by assigning a tuple of - | array dimensions to it. As with `numpy.reshape`, one of the new shape - | dimensions can be -1, in which case its value is inferred from the size of - | the array and the remaining dimensions. Reshaping an array in-place will - | fail if a copy is required. - | - | .. warning:: - | - | Setting ``arr.shape`` is discouraged and may be deprecated in the - | future. Using `ndarray.reshape` is the preferred approach. - | - | Examples - | -------- - | >>> x = np.array([1, 2, 3, 4]) - | >>> x.shape - | (4,) - | >>> y = np.zeros((2, 3, 4)) - | >>> y.shape - | (2, 3, 4) - | >>> y.shape = (3, 8) - | >>> y - | array([[ 0., 0., 0., 0., 0., 0., 0., 0.], - | [ 0., 0., 0., 0., 0., 0., 0., 0.], - | [ 0., 0., 0., 0., 0., 0., 0., 0.]]) - | >>> y.shape = (3, 6) - | Traceback (most recent call last): - | File "", line 1, in - | ValueError: total size of new array must be unchanged - | >>> np.zeros((4,2))[::2].shape = (-1,) - | Traceback (most recent call last): - | File "", line 1, in - | AttributeError: Incompatible shape for in-place modification. Use - | `.reshape()` to make a copy with the desired shape. - | - | See Also - | -------- - | numpy.shape : Equivalent getter function. - | numpy.reshape : Function similar to setting ``shape``. - | ndarray.reshape : Method similar to setting ``shape``. - | - | size - | Number of elements in the array. - | - | Equal to ``np.prod(a.shape)``, i.e., the product of the array's - | dimensions. - | - | Notes - | ----- - | `a.size` returns a standard arbitrary precision Python integer. This - | may not be the case with other methods of obtaining the same value - | (like the suggested ``np.prod(a.shape)``, which returns an instance - | of ``np.int_``), and may be relevant if the value is used further in - | calculations that may overflow a fixed size integer type. - | - | Examples - | -------- - | >>> x = np.zeros((3, 5, 2), dtype=np.complex128) - | >>> x.size - | 30 - | >>> np.prod(x.shape) - | 30 - | - | strides - | Tuple of bytes to step in each dimension when traversing an array. - | - | The byte offset of element ``(i[0], i[1], ..., i[n])`` in an array `a` - | is:: - | - | offset = sum(np.array(i) * a.strides) - | - | A more detailed explanation of strides can be found in the - | "ndarray.rst" file in the NumPy reference guide. - | - | .. warning:: - | - | Setting ``arr.strides`` is discouraged and may be deprecated in the - | future. `numpy.lib.stride_tricks.as_strided` should be preferred - | to create a new view of the same data in a safer way. - | - | Notes - | ----- - | Imagine an array of 32-bit integers (each 4 bytes):: - | - | x = np.array([[0, 1, 2, 3, 4], - | [5, 6, 7, 8, 9]], dtype=np.int32) - | - | This array is stored in memory as 40 bytes, one after the other - | (known as a contiguous block of memory). The strides of an array tell - | us how many bytes we have to skip in memory to move to the next position - | along a certain axis. For example, we have to skip 4 bytes (1 value) to - | move to the next column, but 20 bytes (5 values) to get to the same - | position in the next row. As such, the strides for the array `x` will be - | ``(20, 4)``. - | - | See Also - | -------- - | numpy.lib.stride_tricks.as_strided - | - | Examples - | -------- - | >>> y = np.reshape(np.arange(2*3*4), (2,3,4)) - | >>> y - | array([[[ 0, 1, 2, 3], - | [ 4, 5, 6, 7], - | [ 8, 9, 10, 11]], - | [[12, 13, 14, 15], - | [16, 17, 18, 19], - | [20, 21, 22, 23]]]) - | >>> y.strides - | (48, 16, 4) - | >>> y[1,1,1] - | 17 - | >>> offset=sum(y.strides * np.array((1,1,1))) - | >>> offset/y.itemsize - | 17 - | - | >>> x = np.reshape(np.arange(5*6*7*8), (5,6,7,8)).transpose(2,3,1,0) - | >>> x.strides - | (32, 4, 224, 1344) - | >>> i = np.array([3,5,2,2]) - | >>> offset = sum(i * x.strides) - | >>> x[3,5,2,2] - | 813 - | >>> offset / x.itemsize - | 813 - | - | ---------------------------------------------------------------------- - | Data and other attributes defined here: - | - | __hash__ = None + /** + * Cast {@code long} to {@code int}. + * + * @throws IllegalArgumentException if the value is too large to fit in an integer. */ + private static int safeInt(final long value) { + if (value > Integer.MAX_VALUE) throw new IllegalArgumentException("value too large"); + return (int) value; + } + + /** + * The shape of a multi-dimensional array. + */ + public static class Shape { + + /** + * The order of the array elements, where + *
    + *
  • {@code C_ORDER} means fastest-moving dimension first (as in NumPy)
    • + *
  • {@code F_ORDER} means fastest-moving dimension last (as in ImgLib2)
    • + *
+ * See ArrayOrder + */ + public enum Order {C_ORDER, F_ORDER} + + /** + * native order + */ + private final Order order; + + /** + * dimensions along each axis, arranged in the native order. + */ + private final int[] shape; + + /** + * Construct a {@code Shape} with the specified order and dimensions. + * + * @param order order of the axes. + * @param shape size along each axis. + */ + public Shape(final Order order, final int... shape) { + this.order = order; + this.shape = shape; + } + + /** + * Get the size along at the specified dimension (in the native {@link + * #order()} of this {@code Shape}) + * + * @param d axis index + * @return size along dimension {@code d}. + */ + public int get(final int d) { + return shape[d]; + } + + /** + * Get the number of dimensions. + */ + public int length() { + return shape.length; + } + + /** + * Get the native order of this {@code Shape}, that is the order in + * which axes are arranged when accessed through {@link #get(int)}, + * {@link #toIntArray()}, {@link #toLongArray()}. + */ + public Order order() { + return order; + } + + /** + * Get the total number of elements in the shape. + */ + public long numElements() { + long n = 1; + for (int s : shape) { + n *= s; + } + return n; + } + + /** + * Get the shape dimensions as an array in the native {@link #order()} + * of this {@code Shape}. + * + * @return dimensions array + */ + public int[] toIntArray() { + return shape; + } + + /** + * Get the shape dimensions as an array in the specified order. + * + * @return dimensions array + */ + public int[] toIntArray(final Order order) { + if (order.equals(this.order)) { + return shape; + } + else { + final int[] ishape = new int[shape.length]; + Arrays.setAll(ishape, i -> shape[shape.length - i - 1]); + return ishape; + } + } + + /** + * Get the shape dimensions as an array in the native {@link #order()} + * of this {@code Shape}. + * + * @return dimensions array + */ + public long[] toLongArray() { + return toLongArray(order); + } + + /** + * Get the shape dimensions as an array in the specified order. + * + * @return dimensions array + */ + public long[] toLongArray(final Order order) { + final long[] lshape = new long[shape.length]; + if (order.equals(this.order)) { + Arrays.setAll(lshape, i -> shape[i]); + } + else { + Arrays.setAll(lshape, i -> shape[shape.length - i - 1]); + } + return lshape; + } + + /** + * Returns representation of this {@code Shape} with the given native {@code order}. + */ + public Shape to(final Order order) { + if (order.equals(this.order)) { + return this; + } + else { + return new Shape(order, toIntArray(order)); + } + } - /* - void T(); - void __abs__(); - void __add__(); - void __and__(); - void __array__(); - void __array_finalize__(); - void __array_function__(); - void __array_interface__(); - void __array_prepare__(); - void __array_priority__(); - void __array_struct__(); - void __array_ufunc__(); - void __array_wrap__(); - void __bool__(); - void __class__(); - void __class_getitem__(); - void __complex__(); - void __contains__(); - void __copy__(); - void __deepcopy__(); - void __delattr__(); - void __delitem__(); - void __dir__(); - void __divmod__(); - void __dlpack__(); - void __dlpack_device__(); - void __doc__(); - void __eq__(); - void __float__(); - void __floordiv__(); - void __format__(); - void __ge__(); - void __getattribute__(); - void __getitem__(); - void __gt__(); - void __hash__(); - void __iadd__(); - void __iand__(); - void __ifloordiv__(); - void __ilshift__(); - void __imatmul__(); - void __imod__(); - void __imul__(); - void __index__(); - void __init__(); - void __init_subclass__(); - void __int__(); - void __invert__(); - void __ior__(); - void __ipow__(); - void __irshift__(); - void __isub__(); - void __iter__(); - void __itruediv__(); - void __ixor__(); - void __le__(); - void __len__(); - void __lshift__(); - void __lt__(); - void __matmul__(); - void __mod__(); - void __mul__(); - void __ne__(); - void __neg__(); - void __new__(); - void __or__(); - void __pos__(); - void __pow__(); - void __radd__(); - void __rand__(); - void __rdivmod__(); - void __reduce__(); - void __reduce_ex__(); - void __repr__(); - void __rfloordiv__(); - void __rlshift__(); - void __rmatmul__(); - void __rmod__(); - void __rmul__(); - void __ror__(); - void __rpow__(); - void __rrshift__(); - void __rshift__(); - void __rsub__(); - void __rtruediv__(); - void __rxor__(); - void __setattr__(); - void __setitem__(); - void __setstate__(); - void __sizeof__(); - void __str__(); - void __sub__(); - void __subclasshook__(); - void __truediv__(); - void __xor__(); - void all(); - void any(); - void argmax(); - void argmin(); - void argpartition(); - void argsort(); - void astype(); - void base(); - void byteswap(); - void choose(); - void clip(); - void compress(); - void conj(); - void conjugate(); - void copy(); - void ctypes(); - void cumprod(); - void cumsum(); - void data(); - void diagonal(); - void dot(); - void dtype(); - void dump(); - void dumps(); - void fill(); - void flags(); - void flat(); - void flatten(); - void getfield(); - void imag(); - void item(); - void itemset(); - void itemsize(); - void max(); - void mean(); - void min(); - void nbytes(); - void ndim(); - void newbyteorder(); - void nonzero(); - void partition(); - void prod(); - void ptp(); - void put(); - void ravel(); - void real(); - void repeat(); - void reshape(); - void resize(); - void round(); - void searchsorted(); - void setfield(); - void setflags(); - void shape(); - void size(); - void sort(); - void squeeze(); - void std(); - void strides(); - void sum(); - void swapaxes(); - void take(); - void tobytes(); - void tofile(); - void tolist(); - void tostring(); - void trace(); - void transpose(); - void var(); - void view(); - */ + /** + * Returns a string representation of this Shape, including its order and dimensions. + * + * @return A string representation of this Shape. + */ + @Override + public String toString() { + return "Shape{" + + "order=" + order + + ", shape=" + Arrays.toString(shape) + + '}'; + } + } } diff --git a/src/main/java/org/apposed/appose/SharedMemory.java b/src/main/java/org/apposed/appose/SharedMemory.java index 9c215c1..37d2599 100644 --- a/src/main/java/org/apposed/appose/SharedMemory.java +++ b/src/main/java/org/apposed/appose/SharedMemory.java @@ -30,317 +30,88 @@ package org.apposed.appose; import com.sun.jna.Pointer; -import com.sun.jna.platform.linux.LibRT; -import com.sun.jna.platform.unix.LibCUtil; -import com.sun.jna.platform.win32.Kernel32; -import com.sun.jna.platform.win32.WinBase; -import com.sun.jna.platform.win32.WinError; -import com.sun.jna.platform.win32.WinNT; +import org.apposed.appose.shm.ShmImpl; /** - * Unfinished port of Python's - * {@code multiprocess.shared_memory.SharedMemory} class. + * A shared memory block. *

- * Original source code in Python can be found here. - *

+ * Each shared memory block is assigned a unique name. In this way, one process + * can create a shared memory block with a particular name and a different + * process can attach to that same shared memory block using that same name. */ -public class SharedMemory { - - private static final boolean USE_POSIX = - System.getProperty("os.name").indexOf("Windows") < 0; - - public static final int O_RDONLY = 0; - public static final int O_WRONLY = 1; - public static final int O_RDWR = 2; - public static final int O_NONBLOCK = 4; - public static final int O_APPEND = 8; - public static final int O_SHLOCK = 16; - public static final int O_EXLOCK = 32; - public static final int O_ASYNC = 64; - public static final int O_SYNC = 128; - public static final int O_FSYNC = 128; - public static final int O_NOFOLLOW = 256; - public static final int O_CREAT = 512; - public static final int O_TRUNC = 1024; - public static final int O_EXCL = 2048; - - public static final int O_ACCMODE = 3; - public static final int O_NDELAY = 4; - - public static final int O_EVTONLY = 32768; - public static final int O_NOCTTY = 131072; - public static final int O_DIRECTORY = 1048576; - public static final int O_SYMLINK = 2097152; - public static final int O_DSYNC = 4194304; - public static final int O_CLOEXEC = 16777216; - public static final int O_NOFOLLOW_ANY = 536870912; - - private static final int O_CREX = O_CREAT | O_EXCL; - - /** FreeBSD (and perhaps other BSDs) limit names to 14 characters. */ - private static final int SHM_SAFE_NAME_LENGTH = 14; - - /** Shared memory block name prefix. */ - private static final String SHM_NAME_PREFIX = USE_POSIX ? "/psm_" : "wnsm_"; - - public static class MemoryView { - public MemoryView(Pointer mmap) { - throw new UnsupportedOperationException("Unimplemented"); - } - - public void release() { - throw new UnsupportedOperationException("Unimplemented"); - } - } - - private static String token_hex(long nbytes) { - StringBuilder sb = new StringBuilder(); - for (int b=0; b= 1 && s.length() <= 2; - if (s.length() == 1) sb.append("0"); - sb.append(s); - } - return sb.toString(); - } - - private static long sizeFromFileDescriptor(int fd) { - throw new UnsupportedOperationException("Unimplemented"); - } - - private Pointer mmap(int i, long size) { - String tagName = null; // FIXME - return mmap(i, size, tagName); - } - - private Pointer mmap(int i, long size, String tagName) { - throw new UnsupportedOperationException("Unimplemented"); - } - - private void osClose(int fd) { - throw new UnsupportedOperationException("Unimplemented"); - } - - private void register(String name) { - throw new UnsupportedOperationException("Unimplemented"); - } - - private static void unregister(String name) { - throw new UnsupportedOperationException("Unimplemented"); - } - - /** Creates a random filename for the shared memory object. */ - private static String make_filename() { - // number of random bytes to use for name - long nbytes = (SHM_SAFE_NAME_LENGTH - SHM_NAME_PREFIX.length()) / 2; - assert nbytes >= 2; // 'SHM_NAME_PREFIX too long' - String name = SHM_NAME_PREFIX + token_hex(nbytes); - assert name.length() <= SHM_SAFE_NAME_LENGTH; - return name; - } - - private String name; - private long size; - private int fd = -1; - private Pointer mmap; - private MemoryView buf; - private int flags = O_RDWR; - private int mode = 0600; - private boolean prepend_leading_slash = USE_POSIX; - - public SharedMemory(String name, boolean create, long size) { - // NB: Would be great to use LArray for this instead. But it - // doesn't support an equivalent of Python's shared_memory: - // https://github.com/xerial/larray/issues/78 - - if (size < 0) { - throw new IllegalArgumentException("'size' must be a positive integer"); - } - if (create) { - this.flags = O_CREX | O_RDWR; - if (size == 0) { - throw new IllegalArgumentException("'size' must be a positive number different from zero"); - } - } - if (name == null && (this.flags & O_EXCL) != 0) { - throw new IllegalArgumentException("'name' can only be null if create=true"); - } - - if (USE_POSIX) { - // POSIX Shared Memory - - if (name == null) { - while (true) { - name = make_filename(); - this.fd = LibRT.INSTANCE.shm_open( - name, - this.flags, - this.mode - ); - this.name = name; - break; - } - } - else { - name = this.prepend_leading_slash ? "/" + name : name; - this.fd = LibRT.INSTANCE.shm_open( - name, - this.flags, - mode - ); - this.name = name; - } - try { - if (create && size != 0) { - LibCUtil.ftruncate(this.fd, size); - } - size = sizeFromFileDescriptor(this.fd); - //LibCUtil.mmap(Pointer addr, long length, int prot, int flags, int fd, long offset); - this.mmap = mmap(this.fd, size); - } - finally { - this.unlink(); - } - - register(this.name); - } - else { - // Windows Named Shared Memory - - if (create) { - while (true) { - String temp_name = name == null ? make_filename() : name; - // Create and reserve shared memory block with this name - // until it can be attached to by mmap. - Kernel32.HANDLE h_map = Kernel32.INSTANCE.CreateFileMapping( - WinBase.INVALID_HANDLE_VALUE, - null, - WinNT.PAGE_READWRITE, - (int) ((size >> 32) & 0xFFFFFFFF), - (int) (size & 0xFFFFFFFF), - temp_name - ); - try { - int last_error_code = Kernel32.INSTANCE.GetLastError(); - if (last_error_code == WinError.ERROR_ALREADY_EXISTS) { - if (name != null) { - throw new RuntimeException("File already exists: " + name); - } - continue; - } - //LibCUtil.mmap(Pointer addr, long length, int prot, int flags, int fd, long offset); - this.mmap = mmap(-1, size, temp_name); - } - finally { - Kernel32.INSTANCE.CloseHandle(h_map); - } - this.name = temp_name; - break; - } - } - else { - this.name = name; - // Dynamically determine the existing named shared memory - // block's size which is likely a multiple of mmap.PAGESIZE. - Kernel32.HANDLE h_map = Kernel32.INSTANCE.OpenFileMapping( - WinBase.FILE_MAP_READ, - false, - name - ); - Pointer p_buf; - try { - p_buf = Kernel32.INSTANCE.MapViewOfFile( - h_map, - WinBase.FILE_MAP_READ, - 0, - 0, - 0 - ); - } - finally { - Kernel32.INSTANCE.CloseHandle(h_map); - } - try { - //SIZE_T size = Kernel32.INSTANCE.VirtualQueryEx(HANDLE hProcess, Pointer lpAddress, MEMORY_BASIC_INFORMATION lpBuffer, SIZE_T dwLength); - if (getClass() == getClass()) throw new UnsupportedOperationException(); - //size = Kernel32.INSTANCE.VirtualQuerySize(p_buf); - } - finally { - Kernel32.INSTANCE.UnmapViewOfFile(p_buf); - } - //LibCUtil.mmap(Pointer addr, long length, int prot, int flags, int fd, long offset); - this.mmap = mmap(-1, size, name); - } - } - - this.size = size; - this.buf = new MemoryView(this.mmap); - } - - /** - * A memoryview of contents of the shared memory block. - * - * @return The memoryview buffer. - */ - public MemoryView buf() { - return this.buf; - } - - /** - * Unique name that identifies the shared memory block. - * - * @return The name of the shared memory. - */ - public String name() { - String reported_name = this.name; - if (USE_POSIX && this.prepend_leading_slash) { - if (this.name.startsWith("/")) { - reported_name = this.name.substring(1); - } - } - return reported_name; - } - - /** - * Size in bytes. - * - * @return The length in bytes of the shared memory. - */ - public long size() { - return this.size; - } - - /** - * Closes access to the shared memory from this instance but does - * not destroy the shared memory block. - */ - public void close() { - if (this.buf != null) { - this.buf.release(); - this.buf = null; - } - if (this.mmap != null ){ - //this._mmap.close(); - this.mmap = null; - } - if (USE_POSIX && this.fd >= 0) { - osClose(this.fd); - this.fd = -1; - } - } - - /** - * Requests that the underlying shared memory block be destroyed. - * In order to ensure proper cleanup of resources, unlink should be - * called once (and only once) across all processes which have access - * to the shared memory block. - */ - public void unlink() { - if (USE_POSIX && this.name != null) { - LibRT.INSTANCE.shm_unlink(this.name); - unregister(this.name); - } - } +public final class SharedMemory implements AutoCloseable { + + private final ShmImpl impl; + + /** + * Creates a new shared memory block or attaches to an existing shared + * memory block. + * + * @param name the unique name for the requested shared memory, specified + * as a string. If {@code create==true} a new shared memory + * block, if {@code null} is supplied for the name, a novel + * name will be generated. + * @param create whether a new shared memory block is created ({@code true}) + * or an existing one is attached to ({@code false}). + * @param size size in bytes + */ + public SharedMemory(String name, boolean create, int size) { + if (size < 0) { + throw new IllegalArgumentException("'size' must be a positive integer"); + } + if (create && size == 0) { + throw new IllegalArgumentException("'size' must be a positive number different from zero"); + } + if (!create && name == null) { + throw new IllegalArgumentException("'name' can only be null if create=true"); + } + impl = ShmImpl.create(name, create, size); + } + + /** + * Unique name that identifies the shared memory block. + * + * @return The name of the shared memory. + */ + public String name() { + return impl.name(); + } + + /** + * Size in bytes. + * + * @return The length in bytes of the shared memory. + */ + public long size() { + return impl.size(); + } + + /** + * JNA pointer to the shared memory segment. + * + * @return the pointer to the shared memory segment + */ + public Pointer pointer() { + return impl.pointer(); + } + + /** + * Closes access to the shared memory from this instance but does + * not destroy the shared memory block. + */ + @Override + public void close() throws Exception { + try { + impl.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public String toString() { + return "SharedMemory{impl=" + impl + '}'; + } } + diff --git a/src/main/java/org/apposed/appose/ShmNDArray.java b/src/main/java/org/apposed/appose/ShmNDArray.java deleted file mode 100644 index 7e123ef..0000000 --- a/src/main/java/org/apposed/appose/ShmNDArray.java +++ /dev/null @@ -1,59 +0,0 @@ -/*- - * #%L - * Appose: multi-language interprocess cooperation with shared memory. - * %% - * Copyright (C) 2023 Appose developers. - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ - -package org.apposed.appose; - -/** - * Implementation of {@link NDArray} backed by {@link SharedMemory}. - * - * @param The type of each array element. - */ -public abstract class ShmNDArray implements NDArray { - - // Mark really wants this to be called ShmagePlus! :-P - - /** - * TODO - * - * @param shape tuple of ints : Shape of created array. - * @param dtype data-type, optional : Any object that can be interpreted as a - * numpy data type. - * @param buffer object exposing buffer interface, optional : Used to fill the - * array with data. - * @param offset int, optional : Offset of array data in buffer. - * @param strides tuple of ints, optional : Strides of data in memory. - * @param cOrder {'C', 'F'}, optional : If true, row-major (C-style) order; if - * false, column-major (Fortran-style) order. - */ - public ShmNDArray(long[] shape, Class dtype, SharedMemory buffer, - long offset, long[] strides, boolean cOrder) - { - - } -} diff --git a/src/main/java/org/apposed/appose/Types.java b/src/main/java/org/apposed/appose/Types.java index bcc7768..aec9df2 100644 --- a/src/main/java/org/apposed/appose/Types.java +++ b/src/main/java/org/apposed/appose/Types.java @@ -28,12 +28,18 @@ */ package org.apposed.appose; +import groovy.json.JsonGenerator; +import groovy.json.JsonSlurper; + import java.io.PrintWriter; import java.io.StringWriter; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; -import groovy.json.JsonOutput; -import groovy.json.JsonSlurper; +import static org.apposed.appose.NDArray.Shape.Order.C_ORDER; public final class Types { @@ -42,12 +48,11 @@ private Types() { } public static String encode(Map data) { - return JsonOutput.toJson(data); + return GENERATOR.toJson(data); } - @SuppressWarnings("unchecked") public static Map decode(String json) { - return (Map) new JsonSlurper().parseText(json); + return postProcess(new JsonSlurper().parseText(json)); } /** Dumps the given exception, including stack trace, to a string. */ @@ -56,4 +61,129 @@ public static String stackTrace(Throwable t) { t.printStackTrace(new PrintWriter(sw)); return sw.toString(); } + + + // == serialization ======================================================= + + /* + SharedMemory is represented in JSON as + { + appose_type:shm + name:psm_4812f794 + size:16384 + } + + where + "name" is the unique name of the shared memory segment + "size" is the size in bytes. + (as required for python shared_memory.SharedMemory constructor) + + + NDArray is represented in JSON as + { + appose_type:ndarray + shm:{...} + dtype:float32 + shape:[2, 3, 4] + } + + where + "shm" is the representation of the underlying shared memory segment (as described above) + "dtype" is the data type of the array elements + "shape" is the array dimensions in C-Order + */ + + /** + * Helper to create a {@link JsonGenerator.Converter}. + *

+ * The converter objects of a specified class {@code clz} into a {@code + * Map}. The given {@code appose_type} string is put into + * the map with key {@code "appose_type"}. The map and value to be converted + * are passed to the given {@code BiConsumer} which should serialize the + * value into the map somehow. + * + * @param clz the converter will handle objects of this class (or sub-classes) + * @param appose_type the value for key "appose_type" in the returned Map + * @param converter accepts a map and a value, and serializes the value into the map somehow. + * @return a new converter + * @param object type handled by this converter + */ + private static JsonGenerator.Converter convert(final Class clz, final String appose_type, final BiConsumer, T> converter) { + return new JsonGenerator.Converter() { + + @Override + public boolean handles(final Class type) { + return clz.isAssignableFrom(type); + } + + @SuppressWarnings("unchecked") + @Override + public Object convert(final Object value, final String key) { + final Map map = new LinkedHashMap<>(); + map.put("appose_type", appose_type); + converter.accept(map, (T) value); + return map; + } + }; + } + + static final JsonGenerator GENERATOR = new JsonGenerator.Options() // + .addConverter(convert(SharedMemory.class, "shm", (map, shm) -> { + map.put("name", shm.name()); + map.put("size", shm.size()); + })).addConverter(convert(NDArray.class, "ndarray", (map, ndArray) -> { + map.put("shm", ndArray.shm()); + map.put("dtype", ndArray.dType().label()); + map.put("shape", ndArray.shape().toIntArray(C_ORDER)); + })).build(); + + + // == deserialization ===================================================== + + @SuppressWarnings("unchecked") + private static Map postProcess(Object parseResult) { + return processMap((Map) parseResult); + } + + private static Map processMap(Map map) { + map.entrySet().forEach(entry -> entry.setValue(processValue(entry.getValue()))); + return map; + } + + @SuppressWarnings("unchecked") + private static Object processValue(Object value) { + if (value instanceof Map) { + final Map map = processMap((Map) value); + final Object v = map.get("appose_type"); + if (v instanceof String) { + final String appose_type = (String) v; + switch (appose_type) { + case "shm": + final String name = (String) map.get("name"); + final int size = (int) map.get("size"); + return new SharedMemory(name, false, size); + case "ndarray": + final SharedMemory shm = (SharedMemory) map.get("shm"); + final DType dType = toDType((String) map.get("dtype")); + final NDArray.Shape shape = toShape((List) map.get("shape")); + return new NDArray(shm, dType, shape); + default: + System.err.println("unknown appose_type \"" + appose_type + "\""); + } + } + return map; + } else { + return value; + } + } + + private static DType toDType(final String dtype) { + return DType.fromLabel(dtype); + } + + private static NDArray.Shape toShape(final List shape) { + final int[] ints = new int[shape.size()]; + Arrays.setAll(ints, shape::get); + return new NDArray.Shape(C_ORDER, ints); + } } diff --git a/src/main/java/org/apposed/appose/shm/CLibrary.java b/src/main/java/org/apposed/appose/shm/CLibrary.java new file mode 100644 index 0000000..fb5299e --- /dev/null +++ b/src/main/java/org/apposed/appose/shm/CLibrary.java @@ -0,0 +1,40 @@ +/*- + * #%L + * Use deep learning frameworks from Java in an agnostic and isolated way. + * %% + * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package org.apposed.appose.shm; + +import com.sun.jna.Library; +import com.sun.jna.Native; +import com.sun.jna.Pointer; + +interface CLibrary extends Library { + CLibrary INSTANCE = Native.load("c", CLibrary.class); + + int shm_open(String name, int oflag, int mode); + int ftruncate(int fd, int length); + Pointer mmap(Pointer addr, int length, int prot, int flags, int fd, int offset); + int munmap(Pointer addr, int length); + int close(int fd); + int shm_unlink(String name); + + int SEEK_SET = 0; + int SEEK_CUR = 1; + int SEEK_END = 2; + long lseek(int fd, long offset, int whence); +} diff --git a/src/main/java/org/apposed/appose/shm/LibRt.java b/src/main/java/org/apposed/appose/shm/LibRt.java new file mode 100644 index 0000000..a108933 --- /dev/null +++ b/src/main/java/org/apposed/appose/shm/LibRt.java @@ -0,0 +1,41 @@ +/*- + * #%L + * Use deep learning frameworks from Java in an agnostic and isolated way. + * %% + * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package org.apposed.appose.shm; + +import com.sun.jna.Library; +import com.sun.jna.Native; +import com.sun.jna.Pointer; + + +interface LibRt extends Library { + LibRt INSTANCE = Native.load("rt", LibRt.class); + + int shm_open(String name, int oflag, int mode); + int ftruncate(int fd, int length); + Pointer mmap(Pointer addr, int length, int prot, int flags, int fd, int offset); + int munmap(Pointer addr, int length); + int close(int fd); + int shm_unlink(String name); + + int SEEK_SET = 0; + int SEEK_CUR = 1; + int SEEK_END = 2; + long lseek(int fd, long offset, int whence); +} diff --git a/src/main/java/org/apposed/appose/shm/MacosHelpers.java b/src/main/java/org/apposed/appose/shm/MacosHelpers.java new file mode 100644 index 0000000..bd3fe0d --- /dev/null +++ b/src/main/java/org/apposed/appose/shm/MacosHelpers.java @@ -0,0 +1,61 @@ +/*- + * #%L + * Use deep learning frameworks from Java in an agnostic and isolated way. + * %% + * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +package org.apposed.appose.shm; + +import com.sun.jna.Library; +import com.sun.jna.Native; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + +interface MacosHelpers extends Library { + + String LIBRARY_NAME = "ShmCreate"; + + String LIBRARY_NAME_SUF = ".dylib"; + + // Load the native library + MacosHelpers INSTANCE = loadLibrary(); + + static MacosHelpers loadLibrary() { + try( + final InputStream in = MacosHelpers.class.getResourceAsStream(LIBRARY_NAME + LIBRARY_NAME_SUF);) { + final File tempFile = File.createTempFile(LIBRARY_NAME, LIBRARY_NAME_SUF); + tempFile.deleteOnExit(); + Files.copy(in, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + return Native.load(tempFile.getAbsolutePath(), MacosHelpers.class); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + // Declare methods corresponding to the native functions + int create_shared_memory(String name, int size); + + void unlink_shared_memory(String name); + + long get_shared_memory_size(int fd); +} + + diff --git a/src/main/java/org/apposed/appose/shm/ShmImpl.java b/src/main/java/org/apposed/appose/shm/ShmImpl.java new file mode 100644 index 0000000..63281c7 --- /dev/null +++ b/src/main/java/org/apposed/appose/shm/ShmImpl.java @@ -0,0 +1,55 @@ +package org.apposed.appose.shm; + +import com.sun.jna.Pointer; + +public interface ShmImpl { + static ShmImpl create(String name, boolean create, int size) { + switch (ShmUtils.os) { + case OSX: + return new ShmMacOS(name, create, size); + case LINUX: + return new ShmLinux(name, create, size); + case WINDOWS: + return new ShmWindows(name, create, size); + default: + throw new UnsupportedOperationException("not implemented for " + ShmUtils.os); + } + } + + /** + * Unique name that identifies the shared memory block. + * + * @return The name of the shared memory. + */ + String name(); + + /** + * Size in bytes. + * + * @return The length in bytes of the shared memory. + */ + int size(); + + /** + * JNA pointer to the shared memory segment. + * + * @return the pointer to the shared memory segment + */ + Pointer pointer(); + + /** + * Closes access to the shared memory from this instance but does + * not destroy the shared memory block. + */ + void close() throws Exception; + + /** + * Requests that the underlying shared memory block be destroyed. + * In order to ensure proper cleanup of resources, unlink should be + * called once (and only once) across all processes which have access + * to the shared memory block. + */ + default void unlink() throws Exception { + throw new UnsupportedOperationException(); + } +} diff --git a/src/main/java/org/apposed/appose/shm/ShmLinux.java b/src/main/java/org/apposed/appose/shm/ShmLinux.java new file mode 100644 index 0000000..1f6671c --- /dev/null +++ b/src/main/java/org/apposed/appose/shm/ShmLinux.java @@ -0,0 +1,294 @@ +/*- + * #%L + * Use deep learning frameworks from Java in an agnostic and isolated way. + * %% + * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +package org.apposed.appose.shm; + +import com.sun.jna.Native; +import com.sun.jna.Pointer; + +import static org.apposed.appose.shm.ShmUtils.MAP_SHARED; +import static org.apposed.appose.shm.ShmUtils.O_CREAT; +import static org.apposed.appose.shm.ShmUtils.O_RDONLY; +import static org.apposed.appose.shm.ShmUtils.O_RDWR; +import static org.apposed.appose.shm.ShmUtils.PROT_READ; +import static org.apposed.appose.shm.ShmUtils.PROT_WRITE; +import static org.apposed.appose.shm.ShmUtils.SHM_NAME_PREFIX_POSIX; +import static org.apposed.appose.shm.ShmUtils.SHM_SAFE_NAME_LENGTH; +import static org.apposed.appose.shm.ShmUtils.withLeadingSlash; +import static org.apposed.appose.shm.ShmUtils.withoutLeadingSlash; + +/** + * TODO separate unlink and close + * + * @author Carlos Garcia Lopez de Haro + * @author Tobias Pietzsch + */ +class ShmLinux implements ShmImpl { + + + /** + * File descriptor + */ + private final int fd; + + /** + * Size in bytes + */ + private final int size; + + /** + * Pointer referencing the shared memory + */ + private final Pointer pointer; + + /** + * Unique name that identifies the shared memory segment. + */ + private final String name; + + /** + * Whether the memory block has been closed and unlinked + */ + private boolean unlinked = false; + + @Override + public String name() { + return name; + } + + @Override + public Pointer pointer() { + return pointer; + } + + @Override + public int size() { + return size; + } + + /** + * Unmap and close the shared memory. Necessary to eliminate the shared memory block + */ + @Override + public synchronized void close() { + if (unlinked) { + return; + } + + // Unmap the shared memory + if (this.pointer != Pointer.NULL && LibRtOrC.munmap(this.pointer, size) == -1) { + throw new RuntimeException("munmap failed. Errno: " + Native.getLastError()); + } + + // Close the file descriptor + if (LibRtOrC.close(this.fd) == -1) { + throw new RuntimeException("close failed. Errno: " + Native.getLastError()); + } + + // Unlink the shared memory object + LibRtOrC.shm_unlink(this.name); + unlinked = true; + } + + // name without leading slash + ShmLinux(final String name, final boolean create, final int size) { + String shm_name; + long prevSize; + if (name == null) { + do { + shm_name = ShmUtils.make_filename(SHM_SAFE_NAME_LENGTH, SHM_NAME_PREFIX_POSIX); + prevSize = getSHMSize(shm_name); + } while (prevSize >= 0); + } else { + shm_name = withLeadingSlash(name); + prevSize = getSHMSize(shm_name); + } + + final boolean alreadyExists = prevSize >= 0; + if (alreadyExists && prevSize < size) { + throw new RuntimeException("Shared memory segment already exists with smaller dimensions, data type or format. " + + "Size of the existing shared memory segment cannot be smaller than the size of the proposed object. " + + "Size of existing shared memory segment: " + prevSize + ", size of proposed object: " + size); + } + final int shmFd = LibRtOrC.shm_open(shm_name, O_CREAT | O_RDWR, 0666); + if (shmFd < 0) { + throw new RuntimeException("shm_open failed, errno: " + Native.getLastError()); + } + if (create) { + if ( LibRtOrC.ftruncate(shmFd, (int) size) == -1 ) { + LibRtOrC.close(shmFd); + throw new RuntimeException("ftruncate failed, errno: " + Native.getLastError()); + } + } + final int shm_size = (int) getSHMSize(shmFd); + + Pointer pointer = LibRtOrC.mmap(Pointer.NULL, shm_size, PROT_READ | PROT_WRITE, MAP_SHARED, shmFd, 0); + if (pointer == Pointer.NULL) { + LibRtOrC.close(shmFd); + LibRtOrC.shm_unlink(shm_name); + throw new RuntimeException("mmap failed, errno: " + Native.getLastError()); + } + + this.size = shm_size; + this.name = withoutLeadingSlash(shm_name); + this.fd = shmFd; + this.pointer = pointer; + } + + /** + * Try to open {@code name} and get its size in bytes. + * + * @param name name with leading slash + * @return size in bytes, or -1 if the shared memory segment couuld not be opened + */ + private static long getSHMSize(final String name) { + final int shmFd = LibRtOrC.shm_open(name, O_RDONLY, 0700); + if (shmFd < 0) { + return -1; + } else { + return getSHMSize(shmFd); + } + } + + /** + * Method to find the size of an already created shared memory segment + * + * @param shmFd + * the shared memory segment identifier + * + * @return the size in bytes of the shared memory segment + */ + private static long getSHMSize(final int shmFd) { + if (shmFd < 0) { + throw new RuntimeException("Invalid shmFd. It should be bigger than 0."); + } + + final long size = LibRtOrC.lseek(shmFd, 0, LibRtOrC.SEEK_END); + if (size == -1 ) { + // TODO remove LibRtOrC.close(shmFd); + throw new RuntimeException("Failed to get shared memory segment size. Errno: " + Native.getLastError()); + } + return size; + } + + @Override + public String toString() { + return "ShmLinux{" + + "fd=" + fd + + ", size=" + size + + ", pointer=" + pointer + + ", name='" + name + '\'' + + ", unlinked=" + unlinked + + '}'; + } + + + + private static class LibRtOrC { + + /** + * Depending on the computer, some might work with LibRT or LibC to create SHM segments. + * Thus if true use librt if false, use libc instance + */ + private static boolean useLibRT = true; + + static final int SEEK_SET = 0; + static final int SEEK_CUR = 1; + static final int SEEK_END = 2; + + static int shm_open(String name, int oflag, int mode) { + if (useLibRT) { + try { + return LibRt.INSTANCE.shm_open(name, oflag, mode); + } catch (Exception ex) { + useLibRT = false; + } + } + return CLibrary.INSTANCE.shm_open(name, oflag, mode); + } + + static long lseek(int fd, long offset, int whence) { + if (useLibRT) { + try { + return LibRt.INSTANCE.lseek(fd, offset, whence); + } catch (Exception ex) { + useLibRT = false; + } + } + return CLibrary.INSTANCE.lseek( fd, offset, whence); + } + + static int ftruncate(int fd, int length) { + if (useLibRT) { + try { + return LibRt.INSTANCE.ftruncate(fd, length); + } catch (Exception ex) { + useLibRT = false; + } + } + return CLibrary.INSTANCE.ftruncate(fd, length); + } + + + static Pointer mmap(Pointer addr, int length, int prot, int flags, int fd, int offset) { + if (useLibRT) { + try { + return LibRt.INSTANCE.mmap(addr, length, prot, flags, fd, offset); + } catch (Exception ex) { + useLibRT = false; + } + } + return CLibrary.INSTANCE.mmap(addr, length, prot, flags, fd, offset); + } + + static int munmap(Pointer addr, int length) { + if (useLibRT) { + try { + return LibRt.INSTANCE.munmap(addr, length); + } catch (Exception ex) { + useLibRT = false; + } + } + return CLibrary.INSTANCE.munmap(addr, length); + } + + static int close(int fd) { + if (useLibRT) { + try { + return LibRt.INSTANCE.close(fd); + } catch (Exception ex) { + useLibRT = false; + } + } + return CLibrary.INSTANCE.close(fd); + } + + static int shm_unlink(String name) { + if (useLibRT) { + try { + return LibRt.INSTANCE.shm_unlink(name); + } catch (Exception ex) { + useLibRT = false; + } + } + return CLibrary.INSTANCE.shm_unlink(name); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/apposed/appose/shm/ShmMacOS.java b/src/main/java/org/apposed/appose/shm/ShmMacOS.java new file mode 100644 index 0000000..884d4fb --- /dev/null +++ b/src/main/java/org/apposed/appose/shm/ShmMacOS.java @@ -0,0 +1,193 @@ +/*- + * #%L + * Use deep learning frameworks from Java in an agnostic and isolated way. + * %% + * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +package org.apposed.appose.shm; + +import com.sun.jna.Native; +import com.sun.jna.Pointer; + +import static org.apposed.appose.shm.ShmUtils.MAP_SHARED; +import static org.apposed.appose.shm.ShmUtils.O_RDONLY; +import static org.apposed.appose.shm.ShmUtils.PROT_READ; +import static org.apposed.appose.shm.ShmUtils.PROT_WRITE; +import static org.apposed.appose.shm.ShmUtils.SHM_NAME_PREFIX_POSIX; +import static org.apposed.appose.shm.ShmUtils.SHM_SAFE_NAME_LENGTH; +import static org.apposed.appose.shm.ShmUtils.withLeadingSlash; +import static org.apposed.appose.shm.ShmUtils.withoutLeadingSlash; + +/** + * TODO separate unlink and close + * + * @author Carlos Garcia Lopez de Haro + * @author Tobias Pietzsch + */ +class ShmMacOS implements ShmImpl { + + /** + * File descriptor + */ + private final int fd; + + /** + * Size in bytes + */ + private final int size; + + /** + * Pointer referencing the shared memory + */ + private final Pointer pointer; + + /** + * Unique name that identifies the shared memory segment. + */ + private final String name; + + /** + * Whether the memory block has been closed and unlinked + */ + private boolean unlinked = false; + + @Override + public String name() { + return name; + } + + @Override + public Pointer pointer() { + return pointer; + } + + @Override + public int size() { + return size; + } + + /** + * Unmap and close the shared memory. Necessary to eliminate the shared memory block + */ + @Override + public synchronized void close() { + if (unlinked) { + return; + } + + // Unmap the shared memory + if (this.pointer != Pointer.NULL && CLibrary.INSTANCE.munmap(this.pointer, size) == -1) { + throw new RuntimeException("munmap failed. Errno: " + Native.getLastError()); + } + + // Close the file descriptor + if (CLibrary.INSTANCE.close(this.fd) == -1) { + throw new RuntimeException("close failed. Errno: " + Native.getLastError()); + } + + // Unlink the shared memory object + CLibrary.INSTANCE.shm_unlink(this.name); + unlinked = true; + } + + // name without leading slash + ShmMacOS(final String name, final boolean create, final int size) { + String shm_name; + long prevSize; + if (name == null) { + do { + shm_name = ShmUtils.make_filename(SHM_SAFE_NAME_LENGTH, SHM_NAME_PREFIX_POSIX); + prevSize = getSHMSize(shm_name); + } while (prevSize >= 0); + } else { + shm_name = withLeadingSlash(name); + prevSize = getSHMSize(shm_name); + } + + final boolean alreadyExists = prevSize >= 0; + if (alreadyExists && prevSize < size) { + throw new RuntimeException("Shared memory segment already exists with smaller dimensions, data type or format. " + + "Size of the existing shared memory segment cannot be smaller than the size of the proposed object. " + + "Size of existing shared memory segment: " + prevSize + ", size of proposed object: " + size); + } + // shmFd = INSTANCE.shm_open(this.memoryName, O_RDWR, 0666); + final int shmFd = MacosHelpers.INSTANCE.create_shared_memory(shm_name, size); + if (shmFd < 0) { + throw new RuntimeException("shm_open failed, errno: " + Native.getLastError()); + } + final int shm_size = (int) getSHMSize(shmFd); + + Pointer pointer = CLibrary.INSTANCE.mmap(Pointer.NULL, shm_size, PROT_READ | PROT_WRITE, MAP_SHARED, shmFd, 0); + if (pointer == Pointer.NULL) { + CLibrary.INSTANCE.close(shmFd); + CLibrary.INSTANCE.shm_unlink(shm_name); + throw new RuntimeException("mmap failed, errno: " + Native.getLastError()); + } + + this.size = shm_size; + this.name = withoutLeadingSlash(shm_name); + this.fd = shmFd; + this.pointer = pointer; + } + + /** + * Try to open {@code name} and get its size in bytes. + * + * @param name name with leading slash + * @return size in bytes, or -1 if the shared memory segment couuld not be opened + */ + private static long getSHMSize(final String name) { + final int shmFd = CLibrary.INSTANCE.shm_open(name, O_RDONLY, 0700); + if (shmFd < 0) { + return -1; + } else { + return getSHMSize(shmFd); + } + } + + /** + * Method to find the size of an already created shared memory segment + * + * @param shmFd + * the shared memory segment identifier + * + * @return the size in bytes of the shared memory segment + */ + private static long getSHMSize(final int shmFd) { + if (shmFd < 0) { + throw new RuntimeException("Invalid shmFd. It should be bigger than 0."); + } + + final long size = MacosHelpers.INSTANCE.get_shared_memory_size(shmFd); + if (size == -1) { + // TODO remove macosInstance.unlink_shared_memory(null);; + throw new RuntimeException("Failed to get shared memory segment size. Errno: " + Native.getLastError()); + } + return size; + } + + @Override + public String toString() { + return "ShmMacOS{" + + "fd=" + fd + + ", size=" + size + + ", pointer=" + pointer + + ", name='" + name + '\'' + + ", unlinked=" + unlinked + + '}'; + } +} \ No newline at end of file diff --git a/src/main/java/org/apposed/appose/shm/ShmUtils.java b/src/main/java/org/apposed/appose/shm/ShmUtils.java new file mode 100644 index 0000000..9560e44 --- /dev/null +++ b/src/main/java/org/apposed/appose/shm/ShmUtils.java @@ -0,0 +1,124 @@ +package org.apposed.appose.shm; + +import java.util.Random; + +import static org.apposed.appose.shm.ShmUtils.OS.LINUX; +import static org.apposed.appose.shm.ShmUtils.OS.OSX; +import static org.apposed.appose.shm.ShmUtils.OS.WINDOWS; + +/** + * Utilities used in all platform-specific {@code SharedMemory} implementations. + */ +class ShmUtils { + + /** + * Constant to specify that the shared memory segment that is going to be + * open is only for reading + */ + static int O_RDONLY = 0; + + /** + * Constant to specify that the shared memory segment that is going to be + * open is for reading and/or writing + */ + static int O_RDWR = 2; + + /** + * Constant to specify that the shared memory segment that is going to be + * open will be created if it does not exist + */ + static int O_CREAT = 64; + + /** + * Constant to specify that the shared memory regions mapped can be read but + * not written + */ + static int PROT_READ = 0x1; + + /** + * Constant to specify that the shared memory regions mapped can be written + */ + static int PROT_WRITE = 0x2; + + /** + * Constant to specify that the shared memory regions mapped can be shared + * with other processes + */ + static int MAP_SHARED = 0x01; + + /** + * The detected OS. + */ + public static final OS os = detect_os(); + + enum OS {WINDOWS, OSX, LINUX} + + private static OS detect_os() { + final String os_name = System.getProperty("os.name").toLowerCase(); + if (os_name.startsWith("windows")) { + return WINDOWS; + } else if (os_name.startsWith("mac")) { + return OSX; + } else if (os_name.contains("linux") || os_name.endsWith("ix")) { + return LINUX; + } + throw new RuntimeException("OS detection failed. System.getProperty(\"os.name\") = " + System.getProperty("os.name")); + } + + /** + * Add leading slash if {@code name} doesn't have one. + */ + static String withLeadingSlash(String name) { + return name.startsWith("/") ? name : ("/" + name); + } + + /** + * Remove leading slash if {@code name} has one. + */ + static String withoutLeadingSlash(String name) { + return name.startsWith("/") ? name.substring(1) : name; + } + + /** + * FreeBSD (and perhaps other BSDs) limit names to 14 characters. + */ + static final int SHM_SAFE_NAME_LENGTH = 14; + + /** + * Shared memory block name prefix for Mac and Linux + */ + static final String SHM_NAME_PREFIX_POSIX = "/psm_"; + + /** + * Shared memory block name prefix for Windows + */ + static final String SHM_NAME_PREFIX_WIN = "wnsm_"; + + /** + * Creates a random filename for the shared memory object. + * + * @param maxLength maximum length of the generated name + * @param prefix prefix of the generated filename + * @return a random filename with the given {@code prefix}. + */ + static String make_filename(int maxLength, String prefix) { + // number of random bytes to use for name + final int nbytes = (maxLength - prefix.length()) / 2; + if (nbytes < 2) { + throw new IllegalArgumentException("prefix too long"); + } + final String name = prefix + token_hex(nbytes); + assert name.length() <= maxLength; + return name; + } + + private static String token_hex(int nbytes) { + final byte[] bytes = new byte[nbytes]; + new Random().nextBytes(bytes); + StringBuilder sb = new StringBuilder(nbytes * 2); + for (byte b : bytes) { + sb.append(String.format("%02x", b & 0xff)); + } + return sb.toString(); + } +} diff --git a/src/main/java/org/apposed/appose/shm/ShmWindows.java b/src/main/java/org/apposed/appose/shm/ShmWindows.java new file mode 100644 index 0000000..d6ac31d --- /dev/null +++ b/src/main/java/org/apposed/appose/shm/ShmWindows.java @@ -0,0 +1,246 @@ +/*- + * #%L + * Use deep learning frameworks from Java in an agnostic and isolated way. + * %% + * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +package org.apposed.appose.shm; + +import com.sun.jna.Pointer; +import com.sun.jna.platform.win32.BaseTSD; +import com.sun.jna.platform.win32.Kernel32; +import com.sun.jna.platform.win32.WinBase; +import com.sun.jna.platform.win32.WinNT; + +import static org.apposed.appose.shm.ShmUtils.SHM_NAME_PREFIX_WIN; +import static org.apposed.appose.shm.ShmUtils.SHM_SAFE_NAME_LENGTH; + +/** + * TODO separate unlink and close + * + * @author Carlos Garcia Lopez de Haro + * @author Tobias Pietzsch + */ +class ShmWindows implements ShmImpl { + + /** + * reference to the file that covers the shared memory region + */ + private WinNT.HANDLE hMapFile; + + /** + * Size in bytes + */ + private final int size; + + /** + * Pointer referencing the shared memory + */ + private final Pointer pointer; + + private final Pointer writePointer; + + /** + * Unique name that identifies the shared memory segment. + */ + private final String name; + + /** + * Whether the memory block has been closed and unlinked + */ + private boolean unlinked = false; + + @Override + public String name() { + return name; + } + + @Override + public Pointer pointer() { + return pointer; + } + + @Override + public int size() { + return size; + } + + /** + * Unmap and close the shared memory. Necessary to eliminate the shared memory block + */ + @Override + public synchronized void close() { + if (unlinked) { + return; + } + if (writePointer != null) { + Kernel32.INSTANCE.UnmapViewOfFile(this.writePointer); + } + Kernel32.INSTANCE.UnmapViewOfFile(pointer); + Kernel32.INSTANCE.CloseHandle(hMapFile); + unlinked = true; + } + + // name is WITHOUT prefix etc + ShmWindows(final String name, final boolean create, final int size) { + String shm_name; + long prevSize; + if (name == null) { + do { + shm_name = ShmUtils.make_filename(SHM_SAFE_NAME_LENGTH, SHM_NAME_PREFIX_WIN); + prevSize = getSHMSize(shm_name); + } while (prevSize >= 0); + } else { + shm_name = nameMangle_TODO(name); + prevSize = getSHMSize(shm_name); + } + + final boolean alreadyExists = prevSize >= 0; + if (alreadyExists && prevSize < size) { + throw new RuntimeException("Shared memory segment already exists with smaller dimensions, data type or format. " + + "Size of the existing shared memory segment cannot be smaller than the size of the proposed object. " + + "Size of existing shared memory segment: " + prevSize + ", size of proposed object: " + size); + } + + final WinNT.HANDLE shm_hMapFile = Kernel32.INSTANCE.CreateFileMapping( + WinBase.INVALID_HANDLE_VALUE, + null, + WinNT.PAGE_READWRITE, + 0, + size, + shm_name + ); + if (hMapFile == null) { + throw new RuntimeException("Error creating shared memory array. CreateFileMapping failed: " + Kernel32.INSTANCE.GetLastError()); + } + final int shm_size = (int) getSHMSize(shm_hMapFile); + + // Map the shared memory + Pointer pointer = Kernel32.INSTANCE.MapViewOfFile( + hMapFile, + WinNT.FILE_MAP_WRITE, + 0, + 0, + size + ); + if (pointer == null) { + Kernel32.INSTANCE.CloseHandle(hMapFile); + throw new RuntimeException("Error creating shared memory array. " + Kernel32.INSTANCE.GetLastError()); + } + + Pointer writePointer = Kernel32.INSTANCE.VirtualAllocEx(Kernel32.INSTANCE.GetCurrentProcess(), + pointer, new BaseTSD.SIZE_T(size), WinNT.MEM_COMMIT, WinNT.PAGE_READWRITE); + if (writePointer == null) { + close(); + throw new RuntimeException("Error committing to the shared memory pages. Errno: " + Kernel32.INSTANCE.GetLastError()); + } + + this.size = shm_size; + this.name = nameUnmangle_TODO(shm_name); + this.hMapFile = shm_hMapFile; + this.pointer = pointer; + this.writePointer = writePointer; + } + + // TODO equivalent of removing slash + private String nameUnmangle_TODO(String memoryName) { + return memoryName; + } + + // TODO equivalent of adding slash + // Do we need the "Local\" prefix? + private String nameMangle_TODO(String memoryName) { +// if (!memoryName.startsWith("Local" + File.separator) && !memoryName.startsWith("Global" + File.separator)) +// memoryName = "Local" + File.separator + memoryName; + return memoryName; + } + + // name is WITH prefix etc + private static boolean checkSHMExists(final String name) { + final WinNT.HANDLE hMapFile = Kernel32.INSTANCE.OpenFileMapping(WinNT.FILE_MAP_READ, false, name); + if (hMapFile == null) { + return false; + } else { + Kernel32.INSTANCE.CloseHandle(hMapFile); + return true; + } + } + + /** + * Try to open {@code name} and get its size in bytes. + * + * @param name name with leading slash + * @return size in bytes, or -1 if the shared memory segment couuld not be opened + */ + // name is WITH prefix etc + private static long getSHMSize(final String name) { + WinNT.HANDLE hMapFile = Kernel32.INSTANCE.OpenFileMapping( WinNT.FILE_MAP_READ, false, name); + if (hMapFile == null) { + return -1; + } else { + return getSHMSize(hMapFile); + } + } + + /** + * Method to find the size of an already created shared memory segment + * + * @param hMapFile + * the shared memory segment handle + * + * @return the size in bytes of the shared memory segment + */ + private static long getSHMSize(final WinNT.HANDLE hMapFile) { + if (hMapFile == null) { + throw new NullPointerException("hMapFile is null."); + } + + // Map the shared memory object into the current process's address space + final Pointer pSharedMemory = Kernel32.INSTANCE.MapViewOfFile(hMapFile, WinNT.FILE_MAP_READ, 0, 0, 0); + if (pSharedMemory == null) { + Kernel32.INSTANCE.CloseHandle(hMapFile); + throw new RuntimeException("MapViewOfFile failed with error: " + Kernel32.INSTANCE.GetLastError()); + } + final Kernel32.MEMORY_BASIC_INFORMATION mbi = new Kernel32.MEMORY_BASIC_INFORMATION(); + + if ( + Kernel32.INSTANCE.VirtualQueryEx( + Kernel32.INSTANCE.GetCurrentProcess(), pSharedMemory, + mbi, new BaseTSD.SIZE_T((long) mbi.size()) + ).intValue() == 0) { + throw new RuntimeException("Failed to get shared memory segment size. Errno: " + Kernel32.INSTANCE.GetLastError()); + } + final int size = mbi.regionSize.intValue(); + + Kernel32.INSTANCE.UnmapViewOfFile(pSharedMemory); + Kernel32.INSTANCE.CloseHandle(hMapFile); + + return size; + } + + @Override + public String toString() { + return "ShmWindows{" + + "hMapFile=" + hMapFile + + ", size=" + size + + ", pointer=" + pointer + + ", writePointer=" + writePointer + + ", name='" + name + '\'' + + ", unlinked=" + unlinked + + '}'; + } +} \ No newline at end of file diff --git a/src/main/resources/org/apposed/appose/shm/ShmCreate.dylib b/src/main/resources/org/apposed/appose/shm/ShmCreate.dylib new file mode 100755 index 0000000000000000000000000000000000000000..191fa08fdeb5b545aab397765df2b9240b5ae98a GIT binary patch literal 99378 zcmeI*e{9s{9S88|?m%n#(T)*C-O`JS6A+3*D!N#&l*^9E0BvV*!k(7)N^kb=*xhwf zO7);%EEQ}QvJvBE!xn}3Bh08NE|as$W(Jl{WGeB(`}l-s4$l{kBp^5L|p2ZzaMn~1YgxKL{I z?Kn)9D;5e+fC3bt00k&O0SZun0u-PC1t>rP3Q&Lo6rcbFC_n)UP=Epypa2CZKmiI+ zfC3bt00k&O0SZun0u-PC1t>rP3Q&Lo6rcbFC_n)UP=Epypa2CfufY4~Kfj2?e-{$} z)fK3c;NOQ}iz+q`x)#SLkpHiBwSPsmX&Qa-jSkKB0caX;>o|K`>qKx}Vx({Wczt&} z_ntL-4PXw4v%fWW#;N4UuVp^*MDyzSa8v$xHTO8TX4)PS+o4TMfgMlQEJf?Ocp?^x ztZ~BsSnpOl-Yk1R8{1I1vlqj^Z;m_?iqz{+q#=s7{Q33T@dCEL#Q9_74k_k?;quTo|0@_@tocQ#fPu{ufpBtO@_dWV@%{198!;5Ox=)pD`sB$guLaFm^`MWgpH?gzP zhM3D(GIvjX3H}SYRB3ZOR*Dr%t!jxUg5fy}vj+0hn^9$)X_#LD&NFN*>t>+<1t>rP z3Q&Lo6rcbFC_n)UP=Epypa2CZKmiI+fC3bt00k&O0SZun0u-PC1t>rP3Q&Lo6rcbF zC_n)UP=Epypa2CZKmiI+fC3cw{|o#u(D89#<7Yj1a!1=2oo!zPw!K!9$z)PbA=|Wl zFjKU}Waw_Y`&WolkK+({$3KV;nKP0u8tC-*1v>nLI1dIk9*|*H^*78ZD%p%HWVLpg z@!LO}8|Y{|8|Y|G1v=Z#zUfAW>Km_Pti39ACr%yyf%dimRkTgAR|DJbz5{Kk>u`9- zf5ALz71}%722x8A;c-U;?QKU@(RNA3?eq_H_%EFKuG1sX*_=up!?l^Z7Z(`r98RhC zaF8Khz&SgtByT&v@AO{?v>%wes^8AH81u!V+6Gh4qK_F&K4p)=UdAc)s9d)nC?4+b z#gJ%2hw(e*)Zsr1r<9}x%Zkmk&Rxu8hF4mMUi+~^mGa^s6Ptpw^i8#3k+=0>c<-wI zuFUkO(Vm(9j5+?=9DB@hmpT5{9G}CnA)csBsCZ*oN7n`;sv!|;j?^LX*)%mrnnIC> zRcW{d9njh25`RlBP;$?bJ1n{L$d5KPn+GqkI8|MN2IC{BpMW@<|i&KmiI+fC3bt00k&O0SZun0u-PC1t>rP3Q&Lo6rcbFC_n)U zP=Epypa2CZKmiI+fC3bt00k&O0SZun0u-PC1t>rP3Q&Lo6rcbFC_n)UP=EpypulAl z@M0wShI>7#d;{It3jh}=&OQLh{C7h0eR5TdT>yNx0VnD2!o~qEbaG)c0@q!r%Ta4k z6S%4CnxgTb(!q71gwkfCgPRsFUAD+yHXq^eMhALrFcyo(lpfv$!CcwB4=|1tmsaD| zxHZ89?$ySE^*S63M`JBI9$Ft%_tlwi#gD$F9&kp;YkeqO8;YnJkLqj1Dtk8IaHtK3 zq-+j>#+tpZxz4xPi?Bd%xwJapXO~#^SdfA)N@8`7L`s#RN_@!nUxFB+^f09g$K3eA zyjZ?npL;)^7q{lcTk>MruR-=|kbN5DUCJgOSE0&&6<>DGiMwKP=Epypa2CZKmiI+fC3bt00k&O0SZun0u-PC1t>rP3Q&Lo6rcbFC_n)UP=Epy zpa2CZKmiI+fC3bt00k&O0SZun0u-PC1t>rP3Q&LoUkicv&wqY#qRmg2obe*mLrD7H zVl)1)Mf?O3(zUMkuc#h_#Fy7*W6436Dmmv;v`z%q$zA}q9*=wr9Za*FWaxrc+yOV*(ElEq8!^)FhS1y7<; ziq9NX_U+wn{~%;O_Enn>lW>&16sG01S2cz!&AtS)Yh&TE`E%-9nnJ75XFf)h^~kup zahhl2-H00zi=~V;ZKF~Xa8{d?>cUB`mrI;eWJ*$Hzl2MbcQMZ7F>YRuHhIpjYp$NL zuKu>|n}71sUyl6v-GjHI&5LT*=)qqZs$4t!X`FY@#I>0}&Qj#>P|Rg4nY*XH1pj?p zsrP z3Q&Lo6rcbFC_n)UP=Epypa2CZKmiI+fC3bt00k&O0SZun0u-PC1t>rP3Q&Lo6rcbF zC_n)UP=Epypa2CZKmiI+fC67nflR;q@K9^vo}qSkcTZ(WZ%?J}J?&B5k0`YR`Ny}p zT&fx=nNy+DrRJ79?JD{6-R&iV$VZ);n&(T)we;y}*Mmc?-`|5VyZb!)!;Hs;u|0c+ z`c-$2r)2PSkxKV?^k9WAdH)$a6wUE|wV;8H&=%zb8{y!%Ho z7nPi{yT7CtpKhOe3jIFFWPW+-N|jDs=}Y4`=U!6IJY@{&H^uflUF1uD&#t8+cig7( z?nUOgzJ2%}`tbN>O?C{krheT!)LOX*YjM`puCON8o}o(BEzduN=OZW9SiNQEV61kO zci)P38DprmVo#=BIlu3h-@D#T7f z)@WR|+VQ&ZFSQ*y4a&bhwMxo(IjY7;8n7}n9X!H8-|#F`^@wTYlrP3Q&Lo6rcbFC_n)UP=Epypa2CZKmiI+ zfC3bt00k&O0SZun0u-PC1t>rP3Q&Lo6rcbFC_n)UP=EpyxNHJmj1)kvN0o1&J9`1( z0>#+}0GaA)gW+hbMaM(ygX+FI z^Nsk?x6}j92zjj!g=<3*RpU{|*JE|PtvEc{hQq-&dCc+kEB`KYmwgb5u&`e2r65JV z%Pz6(tsrGCs>BD&O{lQXg^8cE@e;(ijekz5%8MV&i)-@Yhx6iiUM!Q%9bfiokUbh? ze+GGn@-6tQPzzDN?4B1dXDmB?32ANg^q8`j<29fEXks5$whPnF+D%29{B> Date: Wed, 15 May 2024 14:23:40 +0200 Subject: [PATCH 02/16] Add Python example with NDArray (requires branch 'schmarrn' in appose-python to work) --- .../apposed/appose/NDArrayExamplePython.java | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 src/test/java/org/apposed/appose/NDArrayExamplePython.java diff --git a/src/test/java/org/apposed/appose/NDArrayExamplePython.java b/src/test/java/org/apposed/appose/NDArrayExamplePython.java new file mode 100644 index 0000000..0f9949f --- /dev/null +++ b/src/test/java/org/apposed/appose/NDArrayExamplePython.java @@ -0,0 +1,43 @@ +package org.apposed.appose; + +import java.nio.FloatBuffer; +import java.util.HashMap; +import java.util.Map; + +import static org.apposed.appose.NDArray.Shape.Order.F_ORDER; + +public class NDArrayExamplePython { + + public static void main(String[] args) throws Exception { + + // create a FLOAT32 NDArray with shape (4,3,2) in F_ORDER + // respectively (2,3,4) in C_ORDER + final DType dType = DType.FLOAT32; + final NDArray ndArray = new NDArray(dType, new NDArray.Shape(F_ORDER, 4, 3, 2)); + + // fill with values 0..23 in flat iteration order + final FloatBuffer buf = ndArray.buffer().asFloatBuffer(); + final long len = ndArray.shape().numElements(); + for ( int i = 0; i < len; ++i ) { + buf.put(i, i); + } + + // pass to python (will be wrapped as numpy ndarray + final Environment env = Appose.base( "/opt/homebrew/Caskroom/miniforge/base/envs/appose/" ).build(); + try ( Service service = env.python() ) { + final Map< String, Object > inputs = new HashMap<>(); + inputs.put( "img", ndArray); + Service.Task task = service.task(PRINT_INPUT, inputs ); + task.waitFor(); + final String result = ( String ) task.outputs.get( "result" ); + System.out.println( "result = \n" + result ); + } + ndArray.close(); + } + + + private static final String PRINT_INPUT = "" + // + "import numpy as np\n" + // + "task.outputs['result'] = str(img.ndarray())"; + +} From db763259a83abbfdbaf0c14404e326b145739db2 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 15 May 2024 14:49:30 +0200 Subject: [PATCH 03/16] Add C src for ShmCreate.dylib Co-authored-by: Carlos Garcia Lopez de Haro <100329787@alumnos.uc3m.es> --- src/main/c/ShmCreate.c | 67 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 src/main/c/ShmCreate.c diff --git a/src/main/c/ShmCreate.c b/src/main/c/ShmCreate.c new file mode 100644 index 0000000..236e0c2 --- /dev/null +++ b/src/main/c/ShmCreate.c @@ -0,0 +1,67 @@ +#include +#include +#include +#include +#include +#include + +// Function to get the size of a shared memory segment given its file descriptor +long get_shared_memory_size(int fd) { + struct stat shm_stat; + if (fstat(fd, &shm_stat) == -1) { + perror("fstat"); + return -1; + } + return (long)shm_stat.st_size; +} + +// Function to create a shared memory segment, modified to accept a long for size +int create_shared_memory(const char *name, long size) { + int fd = shm_open(name, O_CREAT | O_RDWR, 0666); + if (fd < 0) { + perror("shm_open"); + return -1; + } + long already_size = get_shared_memory_size(fd); + if (already_size > 0) { + return fd; + } + + if (ftruncate(fd, size) == -1) { + perror("ftruncate"); + close(fd); + return -1; + } + + return fd; +} + +// Function to unlink a shared memory segment +void unlink_shared_memory(const char *name) { + if (shm_unlink(name) == -1) { + perror("shm_unlink"); + } +} + +int main() { + const char *shm_name = "/myshm"; + size_t shm_size = 1024; + + // Create shared memory + int shm_fd = create_shared_memory(shm_name, shm_size); + if (shm_fd < 0) { + exit(EXIT_FAILURE); + } + + // Perform operations with shared memory here + // ... + + // Close the shared memory file descriptor + close(shm_fd); + + // Unlink shared memory + unlink_shared_memory(shm_name); + + return 0; +} + From 1f09ca00414ce24820ea0095317dc7c96f027641 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 11 Jul 2024 13:18:31 -0500 Subject: [PATCH 04/16] POM: credit people who contributed --- pom.xml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 263aacb..3bea011 100644 --- a/pom.xml +++ b/pom.xml @@ -46,7 +46,19 @@ - None + Tobias Pietzsch + https://github.com/tpietzsch + tpietzsch + + + Carlos Garcia Lopez de Haro + https://github.com/carlosuc3m + carlosuc3m + + + Mark Kittisopikul + https://github.com/mkitti + mkitti From 321d79ea76ef5973e89561a1ea4e66514a83450a Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 10 Jul 2024 15:06:47 -0500 Subject: [PATCH 05/16] Refactor platform-specific shm logic to plugins This makes the SharedMemory class into an interface with three implementations, one per supported platform, but leaves the door open for additional platforms to add support by implementing their own ShmFactory that produces appropriate SharedMemory objects. The plugins are discovered using Java's ServiceLoader mechanism. --- src/main/java/org/apposed/appose/NDArray.java | 7 +- .../java/org/apposed/appose/SharedMemory.java | 135 +++--- .../java/org/apposed/appose/ShmFactory.java | 5 + src/main/java/org/apposed/appose/Types.java | 26 +- .../java/org/apposed/appose/shm/CLibrary.java | 22 +- .../java/org/apposed/appose/shm/LibRt.java | 23 +- .../java/org/apposed/appose/shm/ShmImpl.java | 55 --- .../java/org/apposed/appose/shm/ShmLinux.java | 404 +++++++++--------- .../java/org/apposed/appose/shm/ShmMacOS.java | 278 ++++++------ .../java/org/apposed/appose/shm/ShmUtils.java | 220 +++++----- .../org/apposed/appose/shm/ShmWindows.java | 389 +++++++++-------- .../services/org.apposed.appose.ShmFactory | 3 + 12 files changed, 783 insertions(+), 784 deletions(-) create mode 100644 src/main/java/org/apposed/appose/ShmFactory.java delete mode 100644 src/main/java/org/apposed/appose/shm/ShmImpl.java create mode 100644 src/main/resources/META-INF/services/org.apposed.appose.ShmFactory diff --git a/src/main/java/org/apposed/appose/NDArray.java b/src/main/java/org/apposed/appose/NDArray.java index 92f2d38..61b21cb 100644 --- a/src/main/java/org/apposed/appose/NDArray.java +++ b/src/main/java/org/apposed/appose/NDArray.java @@ -76,9 +76,8 @@ public NDArray(final SharedMemory sharedMemory, final DType dType, final Shape s * @param shape array shape */ public NDArray(final DType dType, final Shape shape) { - this( - new SharedMemory(null,true,safeInt(shape.numElements() * dType.bytesPerElement())), - dType, shape); + this(SharedMemory.create(null, true, + safeInt(shape.numElements() * dType.bytesPerElement())), dType, shape); } /** @@ -151,7 +150,7 @@ public static class Shape { *

  • {@code F_ORDER} means fastest-moving dimension last (as in ImgLib2)
    • * * See ArrayOrder - */ + */ public enum Order {C_ORDER, F_ORDER} /** diff --git a/src/main/java/org/apposed/appose/SharedMemory.java b/src/main/java/org/apposed/appose/SharedMemory.java index 37d2599..9a8a841 100644 --- a/src/main/java/org/apposed/appose/SharedMemory.java +++ b/src/main/java/org/apposed/appose/SharedMemory.java @@ -30,7 +30,8 @@ package org.apposed.appose; import com.sun.jna.Pointer; -import org.apposed.appose.shm.ShmImpl; + +import java.util.ServiceLoader; /** * A shared memory block. @@ -39,79 +40,77 @@ * can create a shared memory block with a particular name and a different * process can attach to that same shared memory block using that same name. */ -public final class SharedMemory implements AutoCloseable { +public interface SharedMemory extends AutoCloseable { - private final ShmImpl impl; + /** + * Creates a new shared memory block or attaches to an existing shared + * memory block. + * + * @param name the unique name for the requested shared memory, specified + * as a string. If {@code create==true} a new shared memory + * block, if {@code null} is supplied for the name, a novel + * name will be generated. + * @param create whether a new shared memory block is created ({@code true}) + * or an existing one is attached to ({@code false}). + * @param size size in bytes + */ + static SharedMemory create(String name, boolean create, int size) { + if (size < 0) { + throw new IllegalArgumentException("'size' must be a positive integer"); + } + if (create && size == 0) { + throw new IllegalArgumentException("'size' must be a positive number different from zero"); + } + if (!create && name == null) { + throw new IllegalArgumentException("'name' can only be null if create=true"); + } - /** - * Creates a new shared memory block or attaches to an existing shared - * memory block. - * - * @param name the unique name for the requested shared memory, specified - * as a string. If {@code create==true} a new shared memory - * block, if {@code null} is supplied for the name, a novel - * name will be generated. - * @param create whether a new shared memory block is created ({@code true}) - * or an existing one is attached to ({@code false}). - * @param size size in bytes - */ - public SharedMemory(String name, boolean create, int size) { - if (size < 0) { - throw new IllegalArgumentException("'size' must be a positive integer"); - } - if (create && size == 0) { - throw new IllegalArgumentException("'size' must be a positive number different from zero"); - } - if (!create && name == null) { - throw new IllegalArgumentException("'name' can only be null if create=true"); - } - impl = ShmImpl.create(name, create, size); - } + ServiceLoader loader = ServiceLoader.load(ShmFactory.class); + for (ShmFactory factory: loader) { + SharedMemory shm = factory.create(name, create, size); + if (shm != null) return shm; + } + throw new UnsupportedOperationException("No SharedMemory support for this platform"); + } - /** - * Unique name that identifies the shared memory block. - * - * @return The name of the shared memory. - */ - public String name() { - return impl.name(); - } + /** + * Unique name that identifies the shared memory block. + * + * @return The name of the shared memory. + */ + String name(); - /** - * Size in bytes. - * - * @return The length in bytes of the shared memory. - */ - public long size() { - return impl.size(); - } + /** + * Size in bytes. + * + * @return The length in bytes of the shared memory. + */ + long size(); - /** - * JNA pointer to the shared memory segment. - * - * @return the pointer to the shared memory segment - */ - public Pointer pointer() { - return impl.pointer(); - } + /** + * JNA pointer to the shared memory segment. + * + * @return the pointer to the shared memory segment + */ + Pointer pointer(); - /** - * Closes access to the shared memory from this instance but does - * not destroy the shared memory block. - */ - @Override - public void close() throws Exception { - try { - impl.close(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } + /** + * Requests that the underlying shared memory block be destroyed. + * In order to ensure proper cleanup of resources, unlink should be + * called once (and only once) across all processes which have access + * to the shared memory block. + */ + default void unlink() throws Exception { + throw new UnsupportedOperationException(); + } - @Override - public String toString() { - return "SharedMemory{impl=" + impl + '}'; - } + /** + * Closes access to the shared memory from this instance but does + * not destroy the shared memory block. + */ + @Override + void close(); + @Override + String toString(); } - diff --git a/src/main/java/org/apposed/appose/ShmFactory.java b/src/main/java/org/apposed/appose/ShmFactory.java new file mode 100644 index 0000000..2b54f2d --- /dev/null +++ b/src/main/java/org/apposed/appose/ShmFactory.java @@ -0,0 +1,5 @@ +package org.apposed.appose; + +public interface ShmFactory { +SharedMemory create(String name, boolean create, int size); +} diff --git a/src/main/java/org/apposed/appose/Types.java b/src/main/java/org/apposed/appose/Types.java index aec9df2..2b495f8 100644 --- a/src/main/java/org/apposed/appose/Types.java +++ b/src/main/java/org/apposed/appose/Types.java @@ -68,29 +68,29 @@ public static String stackTrace(Throwable t) { /* SharedMemory is represented in JSON as { - appose_type:shm - name:psm_4812f794 - size:16384 + appose_type:shm + name:psm_4812f794 + size:16384 } where - "name" is the unique name of the shared memory segment - "size" is the size in bytes. + "name" is the unique name of the shared memory segment + "size" is the size in bytes. (as required for python shared_memory.SharedMemory constructor) NDArray is represented in JSON as { - appose_type:ndarray - shm:{...} - dtype:float32 - shape:[2, 3, 4] + appose_type:ndarray + shm:{...} + dtype:float32 + shape:[2, 3, 4] } where - "shm" is the representation of the underlying shared memory segment (as described above) - "dtype" is the data type of the array elements - "shape" is the array dimensions in C-Order + "shm" is the representation of the underlying shared memory segment (as described above) + "dtype" is the data type of the array elements + "shape" is the array dimensions in C-Order */ /** @@ -161,7 +161,7 @@ private static Object processValue(Object value) { case "shm": final String name = (String) map.get("name"); final int size = (int) map.get("size"); - return new SharedMemory(name, false, size); + return SharedMemory.create(name, false, size); case "ndarray": final SharedMemory shm = (SharedMemory) map.get("shm"); final DType dType = toDType((String) map.get("dtype")); diff --git a/src/main/java/org/apposed/appose/shm/CLibrary.java b/src/main/java/org/apposed/appose/shm/CLibrary.java index fb5299e..f3a0214 100644 --- a/src/main/java/org/apposed/appose/shm/CLibrary.java +++ b/src/main/java/org/apposed/appose/shm/CLibrary.java @@ -24,17 +24,17 @@ import com.sun.jna.Pointer; interface CLibrary extends Library { - CLibrary INSTANCE = Native.load("c", CLibrary.class); + CLibrary INSTANCE = Native.load("c", CLibrary.class); - int shm_open(String name, int oflag, int mode); - int ftruncate(int fd, int length); - Pointer mmap(Pointer addr, int length, int prot, int flags, int fd, int offset); - int munmap(Pointer addr, int length); - int close(int fd); - int shm_unlink(String name); + int shm_open(String name, int oflag, int mode); + int ftruncate(int fd, int length); + Pointer mmap(Pointer addr, int length, int prot, int flags, int fd, int offset); + int munmap(Pointer addr, int length); + int close(int fd); + int shm_unlink(String name); - int SEEK_SET = 0; - int SEEK_CUR = 1; - int SEEK_END = 2; - long lseek(int fd, long offset, int whence); + int SEEK_SET = 0; + int SEEK_CUR = 1; + int SEEK_END = 2; + long lseek(int fd, long offset, int whence); } diff --git a/src/main/java/org/apposed/appose/shm/LibRt.java b/src/main/java/org/apposed/appose/shm/LibRt.java index a108933..81ed482 100644 --- a/src/main/java/org/apposed/appose/shm/LibRt.java +++ b/src/main/java/org/apposed/appose/shm/LibRt.java @@ -23,19 +23,18 @@ import com.sun.jna.Native; import com.sun.jna.Pointer; - interface LibRt extends Library { - LibRt INSTANCE = Native.load("rt", LibRt.class); + LibRt INSTANCE = Native.load("rt", LibRt.class); - int shm_open(String name, int oflag, int mode); - int ftruncate(int fd, int length); - Pointer mmap(Pointer addr, int length, int prot, int flags, int fd, int offset); - int munmap(Pointer addr, int length); - int close(int fd); - int shm_unlink(String name); + int shm_open(String name, int oflag, int mode); + int ftruncate(int fd, int length); + Pointer mmap(Pointer addr, int length, int prot, int flags, int fd, int offset); + int munmap(Pointer addr, int length); + int close(int fd); + int shm_unlink(String name); - int SEEK_SET = 0; - int SEEK_CUR = 1; - int SEEK_END = 2; - long lseek(int fd, long offset, int whence); + int SEEK_SET = 0; + int SEEK_CUR = 1; + int SEEK_END = 2; + long lseek(int fd, long offset, int whence); } diff --git a/src/main/java/org/apposed/appose/shm/ShmImpl.java b/src/main/java/org/apposed/appose/shm/ShmImpl.java deleted file mode 100644 index 63281c7..0000000 --- a/src/main/java/org/apposed/appose/shm/ShmImpl.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.apposed.appose.shm; - -import com.sun.jna.Pointer; - -public interface ShmImpl { - static ShmImpl create(String name, boolean create, int size) { - switch (ShmUtils.os) { - case OSX: - return new ShmMacOS(name, create, size); - case LINUX: - return new ShmLinux(name, create, size); - case WINDOWS: - return new ShmWindows(name, create, size); - default: - throw new UnsupportedOperationException("not implemented for " + ShmUtils.os); - } - } - - /** - * Unique name that identifies the shared memory block. - * - * @return The name of the shared memory. - */ - String name(); - - /** - * Size in bytes. - * - * @return The length in bytes of the shared memory. - */ - int size(); - - /** - * JNA pointer to the shared memory segment. - * - * @return the pointer to the shared memory segment - */ - Pointer pointer(); - - /** - * Closes access to the shared memory from this instance but does - * not destroy the shared memory block. - */ - void close() throws Exception; - - /** - * Requests that the underlying shared memory block be destroyed. - * In order to ensure proper cleanup of resources, unlink should be - * called once (and only once) across all processes which have access - * to the shared memory block. - */ - default void unlink() throws Exception { - throw new UnsupportedOperationException(); - } -} diff --git a/src/main/java/org/apposed/appose/shm/ShmLinux.java b/src/main/java/org/apposed/appose/shm/ShmLinux.java index 1f6671c..a9dd8f5 100644 --- a/src/main/java/org/apposed/appose/shm/ShmLinux.java +++ b/src/main/java/org/apposed/appose/shm/ShmLinux.java @@ -22,6 +22,8 @@ import com.sun.jna.Native; import com.sun.jna.Pointer; +import org.apposed.appose.SharedMemory; +import org.apposed.appose.ShmFactory; import static org.apposed.appose.shm.ShmUtils.MAP_SHARED; import static org.apposed.appose.shm.ShmUtils.O_CREAT; @@ -35,260 +37,268 @@ import static org.apposed.appose.shm.ShmUtils.withoutLeadingSlash; /** + * Linux-specific shared memory implementation. + *

    * TODO separate unlink and close + *

    * * @author Carlos Garcia Lopez de Haro * @author Tobias Pietzsch */ -class ShmLinux implements ShmImpl { +public class ShmLinux implements ShmFactory { + @Override + public SharedMemory create(final String name, final boolean create, final int size) { + if (ShmUtils.os != ShmUtils.OS.LINUX) return null; // wrong platform + return new SharedMemoryLinux(name, create, size); + } - /** - * File descriptor - */ - private final int fd; + private static class SharedMemoryLinux implements SharedMemory { - /** - * Size in bytes - */ - private final int size; + /** + * File descriptor + */ + private final int fd; - /** - * Pointer referencing the shared memory - */ - private final Pointer pointer; + /** + * Size in bytes + */ + private final int size; - /** - * Unique name that identifies the shared memory segment. - */ - private final String name; + /** + * Pointer referencing the shared memory + */ + private final Pointer pointer; - /** - * Whether the memory block has been closed and unlinked - */ - private boolean unlinked = false; + /** + * Unique name that identifies the shared memory segment. + */ + private final String name; - @Override - public String name() { - return name; - } + /** + * Whether the memory block has been closed and unlinked + */ + private boolean unlinked = false; + + // name without leading slash + private SharedMemoryLinux(final String name, final boolean create, final int size) { + String shm_name; + long prevSize; + if (name == null) { + do { + shm_name = ShmUtils.make_filename(SHM_SAFE_NAME_LENGTH, SHM_NAME_PREFIX_POSIX); + prevSize = getSHMSize(shm_name); + } while (prevSize >= 0); + } else { + shm_name = withLeadingSlash(name); + prevSize = getSHMSize(shm_name); + } - @Override - public Pointer pointer() { - return pointer; - } + final boolean alreadyExists = prevSize >= 0; + if (alreadyExists && prevSize < size) { + throw new RuntimeException("Shared memory segment already exists with smaller dimensions, data type or format. " + + "Size of the existing shared memory segment cannot be smaller than the size of the proposed object. " + + "Size of existing shared memory segment: " + prevSize + ", size of proposed object: " + size); + } + final int shmFd = LibRtOrC.shm_open(shm_name, O_CREAT | O_RDWR, 0666); + if (shmFd < 0) { + throw new RuntimeException("shm_open failed, errno: " + Native.getLastError()); + } + if (create) { + if (LibRtOrC.ftruncate(shmFd, (int) size) == -1) { + LibRtOrC.close(shmFd); + throw new RuntimeException("ftruncate failed, errno: " + Native.getLastError()); + } + } + final int shm_size = (int) getSHMSize(shmFd); - @Override - public int size() { - return size; - } + Pointer pointer = LibRtOrC.mmap(Pointer.NULL, shm_size, PROT_READ | PROT_WRITE, MAP_SHARED, shmFd, 0); + if (pointer == Pointer.NULL) { + LibRtOrC.close(shmFd); + LibRtOrC.shm_unlink(shm_name); + throw new RuntimeException("mmap failed, errno: " + Native.getLastError()); + } - /** - * Unmap and close the shared memory. Necessary to eliminate the shared memory block - */ - @Override - public synchronized void close() { - if (unlinked) { - return; + this.size = shm_size; + this.name = withoutLeadingSlash(shm_name); + this.fd = shmFd; + this.pointer = pointer; } - // Unmap the shared memory - if (this.pointer != Pointer.NULL && LibRtOrC.munmap(this.pointer, size) == -1) { - throw new RuntimeException("munmap failed. Errno: " + Native.getLastError()); + @Override + public String name() { + return name; } - // Close the file descriptor - if (LibRtOrC.close(this.fd) == -1) { - throw new RuntimeException("close failed. Errno: " + Native.getLastError()); + @Override + public Pointer pointer() { + return pointer; } - // Unlink the shared memory object - LibRtOrC.shm_unlink(this.name); - unlinked = true; - } - - // name without leading slash - ShmLinux(final String name, final boolean create, final int size) { - String shm_name; - long prevSize; - if (name == null) { - do { - shm_name = ShmUtils.make_filename(SHM_SAFE_NAME_LENGTH, SHM_NAME_PREFIX_POSIX); - prevSize = getSHMSize(shm_name); - } while (prevSize >= 0); - } else { - shm_name = withLeadingSlash(name); - prevSize = getSHMSize(shm_name); + @Override + public long size() { + return size; } - final boolean alreadyExists = prevSize >= 0; - if (alreadyExists && prevSize < size) { - throw new RuntimeException("Shared memory segment already exists with smaller dimensions, data type or format. " - + "Size of the existing shared memory segment cannot be smaller than the size of the proposed object. " - + "Size of existing shared memory segment: " + prevSize + ", size of proposed object: " + size); - } - final int shmFd = LibRtOrC.shm_open(shm_name, O_CREAT | O_RDWR, 0666); - if (shmFd < 0) { - throw new RuntimeException("shm_open failed, errno: " + Native.getLastError()); - } - if (create) { - if ( LibRtOrC.ftruncate(shmFd, (int) size) == -1 ) { - LibRtOrC.close(shmFd); - throw new RuntimeException("ftruncate failed, errno: " + Native.getLastError()); + /** + * Unmap and close the shared memory. Necessary to eliminate the shared memory block + */ + @Override + public synchronized void close() { + if (unlinked) { + return; } - } - final int shm_size = (int) getSHMSize(shmFd); - Pointer pointer = LibRtOrC.mmap(Pointer.NULL, shm_size, PROT_READ | PROT_WRITE, MAP_SHARED, shmFd, 0); - if (pointer == Pointer.NULL) { - LibRtOrC.close(shmFd); - LibRtOrC.shm_unlink(shm_name); - throw new RuntimeException("mmap failed, errno: " + Native.getLastError()); - } + // Unmap the shared memory + if (this.pointer != Pointer.NULL && LibRtOrC.munmap(this.pointer, size) == -1) { + throw new RuntimeException("munmap failed. Errno: " + Native.getLastError()); + } - this.size = shm_size; - this.name = withoutLeadingSlash(shm_name); - this.fd = shmFd; - this.pointer = pointer; - } + // Close the file descriptor + if (LibRtOrC.close(this.fd) == -1) { + throw new RuntimeException("close failed. Errno: " + Native.getLastError()); + } - /** - * Try to open {@code name} and get its size in bytes. - * - * @param name name with leading slash - * @return size in bytes, or -1 if the shared memory segment couuld not be opened - */ - private static long getSHMSize(final String name) { - final int shmFd = LibRtOrC.shm_open(name, O_RDONLY, 0700); - if (shmFd < 0) { - return -1; - } else { - return getSHMSize(shmFd); + // Unlink the shared memory object + LibRtOrC.shm_unlink(this.name); + unlinked = true; } - } - /** - * Method to find the size of an already created shared memory segment - * - * @param shmFd - * the shared memory segment identifier - * - * @return the size in bytes of the shared memory segment - */ - private static long getSHMSize(final int shmFd) { - if (shmFd < 0) { - throw new RuntimeException("Invalid shmFd. It should be bigger than 0."); + /** + * Try to open {@code name} and get its size in bytes. + * + * @param name name with leading slash + * @return size in bytes, or -1 if the shared memory segment couuld not be opened + */ + private static long getSHMSize(final String name) { + final int shmFd = LibRtOrC.shm_open(name, O_RDONLY, 0700); + if (shmFd < 0) { + return -1; + } else { + return getSHMSize(shmFd); + } } - final long size = LibRtOrC.lseek(shmFd, 0, LibRtOrC.SEEK_END); - if (size == -1 ) { - // TODO remove LibRtOrC.close(shmFd); - throw new RuntimeException("Failed to get shared memory segment size. Errno: " + Native.getLastError()); + /** + * Method to find the size of an already created shared memory segment + * + * @param shmFd the shared memory segment identifier + * @return the size in bytes of the shared memory segment + */ + private static long getSHMSize(final int shmFd) { + if (shmFd < 0) { + throw new RuntimeException("Invalid shmFd. It should be bigger than 0."); + } + + final long size = LibRtOrC.lseek(shmFd, 0, LibRtOrC.SEEK_END); + if (size == -1) { + // TODO remove LibRtOrC.close(shmFd); + throw new RuntimeException("Failed to get shared memory segment size. Errno: " + Native.getLastError()); + } + return size; } - return size; - } - @Override - public String toString() { - return "ShmLinux{" + - "fd=" + fd + - ", size=" + size + - ", pointer=" + pointer + - ", name='" + name + '\'' + - ", unlinked=" + unlinked + - '}'; - } + @Override + public String toString() { + return "ShmLinux{" + + "fd=" + fd + + ", size=" + size + + ", pointer=" + pointer + + ", name='" + name + '\'' + + ", unlinked=" + unlinked + + '}'; + } + private static class LibRtOrC { - private static class LibRtOrC { + /** + * Depending on the computer, some might work with LibRT or LibC to create SHM segments. + * Thus if true use librt if false, use libc instance + */ + private static boolean useLibRT = true; - /** - * Depending on the computer, some might work with LibRT or LibC to create SHM segments. - * Thus if true use librt if false, use libc instance - */ - private static boolean useLibRT = true; - - static final int SEEK_SET = 0; - static final int SEEK_CUR = 1; - static final int SEEK_END = 2; - - static int shm_open(String name, int oflag, int mode) { - if (useLibRT) { - try { - return LibRt.INSTANCE.shm_open(name, oflag, mode); - } catch (Exception ex) { - useLibRT = false; + static final int SEEK_SET = 0; + static final int SEEK_CUR = 1; + static final int SEEK_END = 2; + + static int shm_open(String name, int oflag, int mode) { + if (useLibRT) { + try { + return LibRt.INSTANCE.shm_open(name, oflag, mode); + } catch (Exception ex) { + useLibRT = false; + } } + return CLibrary.INSTANCE.shm_open(name, oflag, mode); } - return CLibrary.INSTANCE.shm_open(name, oflag, mode); - } - static long lseek(int fd, long offset, int whence) { - if (useLibRT) { - try { - return LibRt.INSTANCE.lseek(fd, offset, whence); - } catch (Exception ex) { - useLibRT = false; + static long lseek(int fd, long offset, int whence) { + if (useLibRT) { + try { + return LibRt.INSTANCE.lseek(fd, offset, whence); + } catch (Exception ex) { + useLibRT = false; + } } + return CLibrary.INSTANCE.lseek(fd, offset, whence); } - return CLibrary.INSTANCE.lseek( fd, offset, whence); - } - static int ftruncate(int fd, int length) { - if (useLibRT) { - try { - return LibRt.INSTANCE.ftruncate(fd, length); - } catch (Exception ex) { - useLibRT = false; + static int ftruncate(int fd, int length) { + if (useLibRT) { + try { + return LibRt.INSTANCE.ftruncate(fd, length); + } catch (Exception ex) { + useLibRT = false; + } } + return CLibrary.INSTANCE.ftruncate(fd, length); } - return CLibrary.INSTANCE.ftruncate(fd, length); - } - static Pointer mmap(Pointer addr, int length, int prot, int flags, int fd, int offset) { - if (useLibRT) { - try { - return LibRt.INSTANCE.mmap(addr, length, prot, flags, fd, offset); - } catch (Exception ex) { - useLibRT = false; + static Pointer mmap(Pointer addr, int length, int prot, int flags, int fd, int offset) { + if (useLibRT) { + try { + return LibRt.INSTANCE.mmap(addr, length, prot, flags, fd, offset); + } catch (Exception ex) { + useLibRT = false; + } } + return CLibrary.INSTANCE.mmap(addr, length, prot, flags, fd, offset); } - return CLibrary.INSTANCE.mmap(addr, length, prot, flags, fd, offset); - } - static int munmap(Pointer addr, int length) { - if (useLibRT) { - try { - return LibRt.INSTANCE.munmap(addr, length); - } catch (Exception ex) { - useLibRT = false; + static int munmap(Pointer addr, int length) { + if (useLibRT) { + try { + return LibRt.INSTANCE.munmap(addr, length); + } catch (Exception ex) { + useLibRT = false; + } } + return CLibrary.INSTANCE.munmap(addr, length); } - return CLibrary.INSTANCE.munmap(addr, length); - } - static int close(int fd) { - if (useLibRT) { - try { - return LibRt.INSTANCE.close(fd); - } catch (Exception ex) { - useLibRT = false; + static int close(int fd) { + if (useLibRT) { + try { + return LibRt.INSTANCE.close(fd); + } catch (Exception ex) { + useLibRT = false; + } } + return CLibrary.INSTANCE.close(fd); } - return CLibrary.INSTANCE.close(fd); - } - static int shm_unlink(String name) { - if (useLibRT) { - try { - return LibRt.INSTANCE.shm_unlink(name); - } catch (Exception ex) { - useLibRT = false; + static int shm_unlink(String name) { + if (useLibRT) { + try { + return LibRt.INSTANCE.shm_unlink(name); + } catch (Exception ex) { + useLibRT = false; + } } + return CLibrary.INSTANCE.shm_unlink(name); } - return CLibrary.INSTANCE.shm_unlink(name); } } -} \ No newline at end of file +} diff --git a/src/main/java/org/apposed/appose/shm/ShmMacOS.java b/src/main/java/org/apposed/appose/shm/ShmMacOS.java index 884d4fb..5f5115b 100644 --- a/src/main/java/org/apposed/appose/shm/ShmMacOS.java +++ b/src/main/java/org/apposed/appose/shm/ShmMacOS.java @@ -22,6 +22,8 @@ import com.sun.jna.Native; import com.sun.jna.Pointer; +import org.apposed.appose.SharedMemory; +import org.apposed.appose.ShmFactory; import static org.apposed.appose.shm.ShmUtils.MAP_SHARED; import static org.apposed.appose.shm.ShmUtils.O_RDONLY; @@ -33,161 +35,171 @@ import static org.apposed.appose.shm.ShmUtils.withoutLeadingSlash; /** + * MacOS-specific shared memory implementation. + *

    * TODO separate unlink and close + *

    * * @author Carlos Garcia Lopez de Haro * @author Tobias Pietzsch */ -class ShmMacOS implements ShmImpl { - - /** - * File descriptor - */ - private final int fd; - - /** - * Size in bytes - */ - private final int size; - - /** - * Pointer referencing the shared memory - */ - private final Pointer pointer; - - /** - * Unique name that identifies the shared memory segment. - */ - private final String name; - - /** - * Whether the memory block has been closed and unlinked - */ - private boolean unlinked = false; +public class ShmMacOS implements ShmFactory { @Override - public String name() { - return name; + public SharedMemory create(final String name, final boolean create, final int size) { + if (ShmUtils.os != ShmUtils.OS.OSX) return null; // wrong platform + return new SharedMemoryMacOS(name, create, size); } - @Override - public Pointer pointer() { - return pointer; - } - - @Override - public int size() { - return size; - } - - /** - * Unmap and close the shared memory. Necessary to eliminate the shared memory block - */ - @Override - public synchronized void close() { - if (unlinked) { - return; - } - - // Unmap the shared memory - if (this.pointer != Pointer.NULL && CLibrary.INSTANCE.munmap(this.pointer, size) == -1) { - throw new RuntimeException("munmap failed. Errno: " + Native.getLastError()); + private static class SharedMemoryMacOS implements SharedMemory { + private SharedMemoryMacOS(final String name, final boolean create, final int size) { + String shm_name; + long prevSize; + if (name == null) { + do { + shm_name = ShmUtils.make_filename(SHM_SAFE_NAME_LENGTH, SHM_NAME_PREFIX_POSIX); + prevSize = getSHMSize(shm_name); + } while (prevSize >= 0); + } else { + shm_name = withLeadingSlash(name); + prevSize = getSHMSize(shm_name); + } + + final boolean alreadyExists = prevSize >= 0; + if (alreadyExists && prevSize < size) { + throw new RuntimeException("Shared memory segment already exists with smaller dimensions, data type or format. " + + "Size of the existing shared memory segment cannot be smaller than the size of the proposed object. " + + "Size of existing shared memory segment: " + prevSize + ", size of proposed object: " + size); + } + // shmFd = INSTANCE.shm_open(this.memoryName, O_RDWR, 0666); + final int shmFd = MacosHelpers.INSTANCE.create_shared_memory(shm_name, size); + if (shmFd < 0) { + throw new RuntimeException("shm_open failed, errno: " + Native.getLastError()); + } + final int shm_size = (int) getSHMSize(shmFd); + + Pointer pointer = CLibrary.INSTANCE.mmap(Pointer.NULL, shm_size, PROT_READ | PROT_WRITE, MAP_SHARED, shmFd, 0); + if (pointer == Pointer.NULL) { + CLibrary.INSTANCE.close(shmFd); + CLibrary.INSTANCE.shm_unlink(shm_name); + throw new RuntimeException("mmap failed, errno: " + Native.getLastError()); + } + + this.size = shm_size; + this.name = withoutLeadingSlash(shm_name); + this.fd = shmFd; + this.pointer = pointer; } - // Close the file descriptor - if (CLibrary.INSTANCE.close(this.fd) == -1) { - throw new RuntimeException("close failed. Errno: " + Native.getLastError()); + /** + * File descriptor + */ + private final int fd; + + /** + * Size in bytes + */ + private final int size; + + /** + * Pointer referencing the shared memory + */ + private final Pointer pointer; + + /** + * Unique name that identifies the shared memory segment. + */ + private final String name; + + /** + * Whether the memory block has been closed and unlinked + */ + private boolean unlinked = false; + + @Override + public String name() { + return name; } - // Unlink the shared memory object - CLibrary.INSTANCE.shm_unlink(this.name); - unlinked = true; - } - - // name without leading slash - ShmMacOS(final String name, final boolean create, final int size) { - String shm_name; - long prevSize; - if (name == null) { - do { - shm_name = ShmUtils.make_filename(SHM_SAFE_NAME_LENGTH, SHM_NAME_PREFIX_POSIX); - prevSize = getSHMSize(shm_name); - } while (prevSize >= 0); - } else { - shm_name = withLeadingSlash(name); - prevSize = getSHMSize(shm_name); + @Override + public Pointer pointer() { + return pointer; } - final boolean alreadyExists = prevSize >= 0; - if (alreadyExists && prevSize < size) { - throw new RuntimeException("Shared memory segment already exists with smaller dimensions, data type or format. " - + "Size of the existing shared memory segment cannot be smaller than the size of the proposed object. " - + "Size of existing shared memory segment: " + prevSize + ", size of proposed object: " + size); - } - // shmFd = INSTANCE.shm_open(this.memoryName, O_RDWR, 0666); - final int shmFd = MacosHelpers.INSTANCE.create_shared_memory(shm_name, size); - if (shmFd < 0) { - throw new RuntimeException("shm_open failed, errno: " + Native.getLastError()); + @Override + public long size() { + return size; } - final int shm_size = (int) getSHMSize(shmFd); - Pointer pointer = CLibrary.INSTANCE.mmap(Pointer.NULL, shm_size, PROT_READ | PROT_WRITE, MAP_SHARED, shmFd, 0); - if (pointer == Pointer.NULL) { - CLibrary.INSTANCE.close(shmFd); - CLibrary.INSTANCE.shm_unlink(shm_name); - throw new RuntimeException("mmap failed, errno: " + Native.getLastError()); + /** + * Unmap and close the shared memory. Necessary to eliminate the shared memory block + */ + @Override + public synchronized void close() { + if (unlinked) { + return; + } + + // Unmap the shared memory + if (this.pointer != Pointer.NULL && CLibrary.INSTANCE.munmap(this.pointer, size) == -1) { + throw new RuntimeException("munmap failed. Errno: " + Native.getLastError()); + } + + // Close the file descriptor + if (CLibrary.INSTANCE.close(this.fd) == -1) { + throw new RuntimeException("close failed. Errno: " + Native.getLastError()); + } + + // Unlink the shared memory object + CLibrary.INSTANCE.shm_unlink(this.name); + unlinked = true; } - this.size = shm_size; - this.name = withoutLeadingSlash(shm_name); - this.fd = shmFd; - this.pointer = pointer; - } - - /** - * Try to open {@code name} and get its size in bytes. - * - * @param name name with leading slash - * @return size in bytes, or -1 if the shared memory segment couuld not be opened - */ - private static long getSHMSize(final String name) { - final int shmFd = CLibrary.INSTANCE.shm_open(name, O_RDONLY, 0700); - if (shmFd < 0) { - return -1; - } else { - return getSHMSize(shmFd); + // name without leading slash + + /** + * Try to open {@code name} and get its size in bytes. + * + * @param name name with leading slash + * @return size in bytes, or -1 if the shared memory segment couuld not be opened + */ + private static long getSHMSize(final String name) { + final int shmFd = CLibrary.INSTANCE.shm_open(name, O_RDONLY, 0700); + if (shmFd < 0) { + return -1; + } else { + return getSHMSize(shmFd); + } } - } - /** - * Method to find the size of an already created shared memory segment - * - * @param shmFd - * the shared memory segment identifier - * - * @return the size in bytes of the shared memory segment - */ - private static long getSHMSize(final int shmFd) { - if (shmFd < 0) { - throw new RuntimeException("Invalid shmFd. It should be bigger than 0."); + /** + * Method to find the size of an already created shared memory segment + * + * @param shmFd the shared memory segment identifier + * @return the size in bytes of the shared memory segment + */ + private static long getSHMSize(final int shmFd) { + if (shmFd < 0) { + throw new RuntimeException("Invalid shmFd. It should be bigger than 0."); + } + + final long size = MacosHelpers.INSTANCE.get_shared_memory_size(shmFd); + if (size == -1) { + // TODO remove macosInstance.unlink_shared_memory(null);; + throw new RuntimeException("Failed to get shared memory segment size. Errno: " + Native.getLastError()); + } + return size; } - final long size = MacosHelpers.INSTANCE.get_shared_memory_size(shmFd); - if (size == -1) { - // TODO remove macosInstance.unlink_shared_memory(null);; - throw new RuntimeException("Failed to get shared memory segment size. Errno: " + Native.getLastError()); + @Override + public String toString() { + return "ShmMacOS{" + + "fd=" + fd + + ", size=" + size + + ", pointer=" + pointer + + ", name='" + name + '\'' + + ", unlinked=" + unlinked + + '}'; } - return size; - } - - @Override - public String toString() { - return "ShmMacOS{" + - "fd=" + fd + - ", size=" + size + - ", pointer=" + pointer + - ", name='" + name + '\'' + - ", unlinked=" + unlinked + - '}'; } -} \ No newline at end of file +} diff --git a/src/main/java/org/apposed/appose/shm/ShmUtils.java b/src/main/java/org/apposed/appose/shm/ShmUtils.java index 9560e44..72fd144 100644 --- a/src/main/java/org/apposed/appose/shm/ShmUtils.java +++ b/src/main/java/org/apposed/appose/shm/ShmUtils.java @@ -11,114 +11,114 @@ */ class ShmUtils { - /** - * Constant to specify that the shared memory segment that is going to be - * open is only for reading - */ - static int O_RDONLY = 0; - - /** - * Constant to specify that the shared memory segment that is going to be - * open is for reading and/or writing - */ - static int O_RDWR = 2; - - /** - * Constant to specify that the shared memory segment that is going to be - * open will be created if it does not exist - */ - static int O_CREAT = 64; - - /** - * Constant to specify that the shared memory regions mapped can be read but - * not written - */ - static int PROT_READ = 0x1; - - /** - * Constant to specify that the shared memory regions mapped can be written - */ - static int PROT_WRITE = 0x2; - - /** - * Constant to specify that the shared memory regions mapped can be shared - * with other processes - */ - static int MAP_SHARED = 0x01; - - /** - * The detected OS. - */ - public static final OS os = detect_os(); - - enum OS {WINDOWS, OSX, LINUX} - - private static OS detect_os() { - final String os_name = System.getProperty("os.name").toLowerCase(); - if (os_name.startsWith("windows")) { - return WINDOWS; - } else if (os_name.startsWith("mac")) { - return OSX; - } else if (os_name.contains("linux") || os_name.endsWith("ix")) { - return LINUX; - } - throw new RuntimeException("OS detection failed. System.getProperty(\"os.name\") = " + System.getProperty("os.name")); - } - - /** - * Add leading slash if {@code name} doesn't have one. - */ - static String withLeadingSlash(String name) { - return name.startsWith("/") ? name : ("/" + name); - } - - /** - * Remove leading slash if {@code name} has one. - */ - static String withoutLeadingSlash(String name) { - return name.startsWith("/") ? name.substring(1) : name; - } - - /** - * FreeBSD (and perhaps other BSDs) limit names to 14 characters. - */ - static final int SHM_SAFE_NAME_LENGTH = 14; - - /** - * Shared memory block name prefix for Mac and Linux - */ - static final String SHM_NAME_PREFIX_POSIX = "/psm_"; - - /** - * Shared memory block name prefix for Windows - */ - static final String SHM_NAME_PREFIX_WIN = "wnsm_"; - - /** - * Creates a random filename for the shared memory object. - * - * @param maxLength maximum length of the generated name - * @param prefix prefix of the generated filename - * @return a random filename with the given {@code prefix}. - */ - static String make_filename(int maxLength, String prefix) { - // number of random bytes to use for name - final int nbytes = (maxLength - prefix.length()) / 2; - if (nbytes < 2) { - throw new IllegalArgumentException("prefix too long"); - } - final String name = prefix + token_hex(nbytes); - assert name.length() <= maxLength; - return name; - } - - private static String token_hex(int nbytes) { - final byte[] bytes = new byte[nbytes]; - new Random().nextBytes(bytes); - StringBuilder sb = new StringBuilder(nbytes * 2); - for (byte b : bytes) { - sb.append(String.format("%02x", b & 0xff)); - } - return sb.toString(); - } + /** + * Constant to specify that the shared memory segment that is going to be + * open is only for reading + */ + static int O_RDONLY = 0; + + /** + * Constant to specify that the shared memory segment that is going to be + * open is for reading and/or writing + */ + static int O_RDWR = 2; + + /** + * Constant to specify that the shared memory segment that is going to be + * open will be created if it does not exist + */ + static int O_CREAT = 64; + + /** + * Constant to specify that the shared memory regions mapped can be read but + * not written + */ + static int PROT_READ = 0x1; + + /** + * Constant to specify that the shared memory regions mapped can be written + */ + static int PROT_WRITE = 0x2; + + /** + * Constant to specify that the shared memory regions mapped can be shared + * with other processes + */ + static int MAP_SHARED = 0x01; + + /** + * The detected OS. + */ + public static final OS os = detect_os(); + + enum OS {WINDOWS, OSX, LINUX} + + private static OS detect_os() { + final String os_name = System.getProperty("os.name").toLowerCase(); + if (os_name.startsWith("windows")) { + return WINDOWS; + } else if (os_name.startsWith("mac")) { + return OSX; + } else if (os_name.contains("linux") || os_name.endsWith("ix")) { + return LINUX; + } + throw new RuntimeException("OS detection failed. System.getProperty(\"os.name\") = " + System.getProperty("os.name")); + } + + /** + * Add leading slash if {@code name} doesn't have one. + */ + static String withLeadingSlash(String name) { + return name.startsWith("/") ? name : ("/" + name); + } + + /** + * Remove leading slash if {@code name} has one. + */ + static String withoutLeadingSlash(String name) { + return name.startsWith("/") ? name.substring(1) : name; + } + + /** + * FreeBSD (and perhaps other BSDs) limit names to 14 characters. + */ + static final int SHM_SAFE_NAME_LENGTH = 14; + + /** + * Shared memory block name prefix for Mac and Linux + */ + static final String SHM_NAME_PREFIX_POSIX = "/psm_"; + + /** + * Shared memory block name prefix for Windows + */ + static final String SHM_NAME_PREFIX_WIN = "wnsm_"; + + /** + * Creates a random filename for the shared memory object. + * + * @param maxLength maximum length of the generated name + * @param prefix prefix of the generated filename + * @return a random filename with the given {@code prefix}. + */ + static String make_filename(int maxLength, String prefix) { + // number of random bytes to use for name + final int nbytes = (maxLength - prefix.length()) / 2; + if (nbytes < 2) { + throw new IllegalArgumentException("prefix too long"); + } + final String name = prefix + token_hex(nbytes); + assert name.length() <= maxLength; + return name; + } + + private static String token_hex(int nbytes) { + final byte[] bytes = new byte[nbytes]; + new Random().nextBytes(bytes); + StringBuilder sb = new StringBuilder(nbytes * 2); + for (byte b : bytes) { + sb.append(String.format("%02x", b & 0xff)); + } + return sb.toString(); + } } diff --git a/src/main/java/org/apposed/appose/shm/ShmWindows.java b/src/main/java/org/apposed/appose/shm/ShmWindows.java index d6ac31d..0bcb1d8 100644 --- a/src/main/java/org/apposed/appose/shm/ShmWindows.java +++ b/src/main/java/org/apposed/appose/shm/ShmWindows.java @@ -25,222 +25,249 @@ import com.sun.jna.platform.win32.Kernel32; import com.sun.jna.platform.win32.WinBase; import com.sun.jna.platform.win32.WinNT; +import org.apposed.appose.SharedMemory; +import org.apposed.appose.ShmFactory; import static org.apposed.appose.shm.ShmUtils.SHM_NAME_PREFIX_WIN; import static org.apposed.appose.shm.ShmUtils.SHM_SAFE_NAME_LENGTH; /** + * Windows-specific shared memory implementation. + *

    * TODO separate unlink and close + *

    * * @author Carlos Garcia Lopez de Haro * @author Tobias Pietzsch */ -class ShmWindows implements ShmImpl { - - /** - * reference to the file that covers the shared memory region - */ - private WinNT.HANDLE hMapFile; - - /** - * Size in bytes - */ - private final int size; - - /** - * Pointer referencing the shared memory - */ - private final Pointer pointer; - - private final Pointer writePointer; - - /** - * Unique name that identifies the shared memory segment. - */ - private final String name; - - /** - * Whether the memory block has been closed and unlinked - */ - private boolean unlinked = false; - - @Override - public String name() { - return name; - } - - @Override - public Pointer pointer() { - return pointer; - } +public class ShmWindows implements ShmFactory { - @Override - public int size() { - return size; - } - - /** - * Unmap and close the shared memory. Necessary to eliminate the shared memory block - */ - @Override - public synchronized void close() { - if (unlinked) { - return; - } - if (writePointer != null) { - Kernel32.INSTANCE.UnmapViewOfFile(this.writePointer); - } - Kernel32.INSTANCE.UnmapViewOfFile(pointer); - Kernel32.INSTANCE.CloseHandle(hMapFile); - unlinked = true; + // name is WITHOUT prefix etc + public SharedMemory create(final String name, final boolean create, final int size) { + if (ShmUtils.os != ShmUtils.OS.WINDOWS) return null; // wrong platform + return new SharedMemoryWindows(name, create, size); } - // name is WITHOUT prefix etc - ShmWindows(final String name, final boolean create, final int size) { + private static class SharedMemoryWindows implements SharedMemory { String shm_name; long prevSize; - if (name == null) { - do { - shm_name = ShmUtils.make_filename(SHM_SAFE_NAME_LENGTH, SHM_NAME_PREFIX_WIN); + private SharedMemoryWindows(final String name, final boolean create, final int size) { + if(name ==null) + + { + do { + shm_name = ShmUtils.make_filename(SHM_SAFE_NAME_LENGTH, SHM_NAME_PREFIX_WIN); + prevSize = getSHMSize(shm_name); + } while (prevSize >= 0); + } else + + { + shm_name = nameMangle_TODO(name); prevSize = getSHMSize(shm_name); - } while (prevSize >= 0); - } else { - shm_name = nameMangle_TODO(name); - prevSize = getSHMSize(shm_name); - } + } - final boolean alreadyExists = prevSize >= 0; - if (alreadyExists && prevSize < size) { - throw new RuntimeException("Shared memory segment already exists with smaller dimensions, data type or format. " - + "Size of the existing shared memory segment cannot be smaller than the size of the proposed object. " - + "Size of existing shared memory segment: " + prevSize + ", size of proposed object: " + size); - } + final boolean alreadyExists = prevSize >= 0; + if(alreadyExists &&prevSize loader = ServiceLoader.load(ShmFactory.class); for (ShmFactory factory: loader) { SharedMemory shm = factory.create(name, create, size); diff --git a/src/main/java/org/apposed/appose/Types.java b/src/main/java/org/apposed/appose/Types.java index 2b495f8..4a869ef 100644 --- a/src/main/java/org/apposed/appose/Types.java +++ b/src/main/java/org/apposed/appose/Types.java @@ -161,7 +161,7 @@ private static Object processValue(Object value) { case "shm": final String name = (String) map.get("name"); final int size = (int) map.get("size"); - return SharedMemory.create(name, false, size); + return SharedMemory.attach(name, size); case "ndarray": final SharedMemory shm = (SharedMemory) map.get("shm"); final DType dType = toDType((String) map.get("dtype")); From d529957cf5144009e8e5ba77c24bf040b9dd7b3f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 10 Jul 2024 15:31:48 -0500 Subject: [PATCH 07/16] Add unit tests for SharedMemory functionality --- .../org/apposed/appose/SharedMemoryTest.java | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 src/test/java/org/apposed/appose/SharedMemoryTest.java diff --git a/src/test/java/org/apposed/appose/SharedMemoryTest.java b/src/test/java/org/apposed/appose/SharedMemoryTest.java new file mode 100644 index 0000000..be8b13c --- /dev/null +++ b/src/test/java/org/apposed/appose/SharedMemoryTest.java @@ -0,0 +1,147 @@ +/*- + * #%L + * Appose: multi-language interprocess cooperation with shared memory. + * %% + * Copyright (C) 2023 Appose developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.apposed.appose; + +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.nio.ByteBuffer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Tests {@link SharedMemory}. + * + * @author Curtis Rueden + */ +public class SharedMemoryTest { + + @Test + public void testShmCreate() throws IOException { + int size = 456; + try (SharedMemory shm = SharedMemory.create(null, size)) { + assertNotNull(shm.name()); + assertEquals(size, shm.size()); + assertNotNull(shm.pointer()); + + // Modify the memory contents. + ByteBuffer buffer = shm.pointer().getByteBuffer(0, size); + for (int i = 0; i < size; i++) { + buffer.put(i, (byte) (size - i)); + } + + // Assert that values have been modified as expected. + byte[] b = new byte[size]; + buffer.get(b); + for (int i = 0; i < size; i++) { + assertEquals((byte) (size - i), b[i]); + } + + // Assert that another process is able to read the values. + String output = runPython( + "from multiprocessing.shared_memory import SharedMemory\n" + + "from sys import stdout\n" + + "shm = SharedMemory(name='" + shm.name() + "', size=" + shm.size() + ")\n" + + "matches = sum(1 for i in range(shm.size) if shm.buf[i] == (shm.size - i) % 256)\n" + + "stdout.write(f'{matches}\\n')\n" + + "stdout.flush()\n" + + "shm.unlink()\n" // HACK: to satisfy Python's overly aggressive resource tracker + ); + assertEquals("" + size, output); + } + } + + @Test + public void testShmAttach() throws IOException { + // Create a named shared memory block in a separate process. + // NB: I originally tried passing the Python script as an argument to `-c`, + // but it gets truncated (don't know why) and the program fails to execute. + // So instead, the program is passed in via stdin. But that means the + // program itself cannot read from stdin as a means of waiting for Java to + // signal its completion of the test asserts; we use a hacky sleep instead. + String output = runPython( + "from multiprocessing.shared_memory import SharedMemory\n" + + "from sys import stdout\n" + + "shm = SharedMemory(create=True, size=345)\n" + + "shm.buf[0] = 12\n" + + "shm.buf[100] = 234\n" + + "shm.buf[344] = 7\n" + + "stdout.write(f'{shm.name}|{shm.size}\\n')\n" + + "stdout.flush()\n" + + "import time; time.sleep(0.2)\n" + // HACK: horrible, but keeps things simple + "shm.unlink()\n" + ); + + // Parse the output into the name and size of the shared memory block. + String[] shmInfo = output.split("\\|"); + assertEquals(2, shmInfo.length); + String shmName = shmInfo[0]; + assertNotNull(shmName); + assertFalse(shmName.isEmpty()); + int shmSize = Integer.parseInt(shmInfo[1]); + assertEquals(345, shmSize); + + // Attach to the shared memory and verify it matches expectations. + try (SharedMemory shm = SharedMemory.attach(shmName, shmSize)) { + assertNotNull(shm); + assertEquals(shmName, shm.name()); + assertEquals(shmSize, shm.size()); + ByteBuffer buf = shm.pointer().getByteBuffer(0, shmSize); + assertEquals(12, buf.get(0)); + assertEquals((byte) 234, buf.get(100)); + assertEquals(7, buf.get(344)); + } + + // NB: No need to clean up the shared memory explicitly, + // since the Python program will unlink it and terminate + // upon completion of the sleep instruction. + } + + private static String runPython(String script) throws IOException { + boolean isWindows = System.getProperty("os.name").startsWith("Win"); + String pythonCommand = isWindows ? "python.exe" : "python"; + ProcessBuilder pb = new ProcessBuilder().command(pythonCommand); + Process p = pb.start(); + try (BufferedWriter os = new BufferedWriter(new OutputStreamWriter(p.getOutputStream()))) { + os.write(script); + os.flush(); + } + BufferedReader is = new BufferedReader(new InputStreamReader(p.getInputStream())); + String line = is.readLine(); + assertNotNull(line, "Python program returned no output"); + return line; + } +} From f855dc6a23cbf56438003a941c81cb932d0ea0d6 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 15 Jul 2024 14:21:04 -0500 Subject: [PATCH 08/16] Make DType enum a static inner class of NDArray The DType and Shape classes define the two major pieces of metadata needed by NDArray. One was an inner class of NDArray and the other was not. This change makes it consistent: now both are inner classes. --- src/main/java/org/apposed/appose/DType.java | 63 ------------------- src/main/java/org/apposed/appose/NDArray.java | 62 ++++++++++++++++++ src/main/java/org/apposed/appose/Types.java | 6 +- .../apposed/appose/NDArrayExampleGroovy.java | 2 +- .../apposed/appose/NDArrayExamplePython.java | 2 +- 5 files changed, 67 insertions(+), 68 deletions(-) delete mode 100644 src/main/java/org/apposed/appose/DType.java diff --git a/src/main/java/org/apposed/appose/DType.java b/src/main/java/org/apposed/appose/DType.java deleted file mode 100644 index 1c229ef..0000000 --- a/src/main/java/org/apposed/appose/DType.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.apposed.appose; - -/** - * Enumerates possible data type of {@link NDArray} elements. - */ -public enum DType { - INT8("int8", Byte.BYTES), // - INT16("int16", Short.BYTES), // - INT32("int32", Integer.BYTES), // - INT64("int64", Long.BYTES), // - UINT8("uint8", Byte.BYTES), // - UINT16("uint16", Short.BYTES), // - UINT32("uint32", Integer.BYTES), // - UINT64("uint64", Long.BYTES), // - FLOAT32("float32", Float.BYTES), // - FLOAT64("float64", Double.BYTES), // - COMPLEX64("complex64", Float.BYTES * 2), // - COMPLEX128("complex128", Double.BYTES * 2), // - BOOL("bool", 1); - - private final String label; - - private final int bytesPerElement; - - DType(final String label, final int bytesPerElement) - { - this.label = label; - this.bytesPerElement = bytesPerElement; - } - - /** - * Get the number of bytes per element for this data type. - */ - public int bytesPerElement() - { - return bytesPerElement; - } - - /** - * Get the label of this {@code DType}. - *

    - * The label can used as a {@code dtype} in Python. It is also used for JSON - * serialization. - * - * @return the label. - */ - public String label() - { - return label; - } - - /** - * Returns the {@code DType} corresponding to the given {@code label}. - * - * @param label a label. - * @return {@code DType} corresponding to {@code label}. - * @throws IllegalArgumentException if no {@code DType} corresponds to the given label. - */ - public static DType fromLabel(final String label) throws IllegalArgumentException - { - return valueOf( label.toUpperCase() ); - } -} diff --git a/src/main/java/org/apposed/appose/NDArray.java b/src/main/java/org/apposed/appose/NDArray.java index 5bcc956..e86084b 100644 --- a/src/main/java/org/apposed/appose/NDArray.java +++ b/src/main/java/org/apposed/appose/NDArray.java @@ -138,6 +138,68 @@ private static int safeInt(final long value) { return (int) value; } + /** + * Enumerates possible data type of {@link NDArray} elements. + */ + public static enum DType { + INT8("int8", Byte.BYTES), // + INT16("int16", Short.BYTES), // + INT32("int32", Integer.BYTES), // + INT64("int64", Long.BYTES), // + UINT8("uint8", Byte.BYTES), // + UINT16("uint16", Short.BYTES), // + UINT32("uint32", Integer.BYTES), // + UINT64("uint64", Long.BYTES), // + FLOAT32("float32", Float.BYTES), // + FLOAT64("float64", Double.BYTES), // + COMPLEX64("complex64", Float.BYTES * 2), // + COMPLEX128("complex128", Double.BYTES * 2), // + BOOL("bool", 1); + + private final String label; + + private final int bytesPerElement; + + DType(final String label, final int bytesPerElement) + { + this.label = label; + this.bytesPerElement = bytesPerElement; + } + + /** + * Get the number of bytes per element for this data type. + */ + public int bytesPerElement() + { + return bytesPerElement; + } + + /** + * Get the label of this {@code DType}. + *

    + * The label can used as a {@code dtype} in Python. It is also used for JSON + * serialization. + * + * @return the label. + */ + public String label() + { + return label; + } + + /** + * Returns the {@code DType} corresponding to the given {@code label}. + * + * @param label a label. + * @return {@code DType} corresponding to {@code label}. + * @throws IllegalArgumentException if no {@code DType} corresponds to the given label. + */ + public static DType fromLabel(final String label) throws IllegalArgumentException + { + return valueOf( label.toUpperCase() ); + } + } + /** * The shape of a multi-dimensional array. */ diff --git a/src/main/java/org/apposed/appose/Types.java b/src/main/java/org/apposed/appose/Types.java index 4a869ef..811ff0c 100644 --- a/src/main/java/org/apposed/appose/Types.java +++ b/src/main/java/org/apposed/appose/Types.java @@ -164,7 +164,7 @@ private static Object processValue(Object value) { return SharedMemory.attach(name, size); case "ndarray": final SharedMemory shm = (SharedMemory) map.get("shm"); - final DType dType = toDType((String) map.get("dtype")); + final NDArray.DType dType = toDType((String) map.get("dtype")); final NDArray.Shape shape = toShape((List) map.get("shape")); return new NDArray(shm, dType, shape); default: @@ -177,8 +177,8 @@ private static Object processValue(Object value) { } } - private static DType toDType(final String dtype) { - return DType.fromLabel(dtype); + private static NDArray.DType toDType(final String dtype) { + return NDArray.DType.fromLabel(dtype); } private static NDArray.Shape toShape(final List shape) { diff --git a/src/test/java/org/apposed/appose/NDArrayExampleGroovy.java b/src/test/java/org/apposed/appose/NDArrayExampleGroovy.java index 999e541..1467656 100644 --- a/src/test/java/org/apposed/appose/NDArrayExampleGroovy.java +++ b/src/test/java/org/apposed/appose/NDArrayExampleGroovy.java @@ -12,7 +12,7 @@ public static void main(String[] args) throws Exception { // create a FLOAT32 NDArray with shape (4,3,2) in F_ORDER // respectively (2,3,4) in C_ORDER - final DType dType = DType.FLOAT32; + final NDArray.DType dType = NDArray.DType.FLOAT32; final NDArray ndArray = new NDArray(dType, new NDArray.Shape(F_ORDER, 4, 3, 2)); // fill with values 0..23 in flat iteration order diff --git a/src/test/java/org/apposed/appose/NDArrayExamplePython.java b/src/test/java/org/apposed/appose/NDArrayExamplePython.java index 0f9949f..4a80b39 100644 --- a/src/test/java/org/apposed/appose/NDArrayExamplePython.java +++ b/src/test/java/org/apposed/appose/NDArrayExamplePython.java @@ -12,7 +12,7 @@ public static void main(String[] args) throws Exception { // create a FLOAT32 NDArray with shape (4,3,2) in F_ORDER // respectively (2,3,4) in C_ORDER - final DType dType = DType.FLOAT32; + final NDArray.DType dType = NDArray.DType.FLOAT32; final NDArray ndArray = new NDArray(dType, new NDArray.Shape(F_ORDER, 4, 3, 2)); // fill with values 0..23 in flat iteration order From 0ed61124bf7197ddcfcac839d4a679e9e3517577 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 15 Jul 2024 14:22:19 -0500 Subject: [PATCH 09/16] Add tests for the encode/decode methods of Types --- .../java/org/apposed/appose/TypesTest.java | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 src/test/java/org/apposed/appose/TypesTest.java diff --git a/src/test/java/org/apposed/appose/TypesTest.java b/src/test/java/org/apposed/appose/TypesTest.java new file mode 100644 index 0000000..12374aa --- /dev/null +++ b/src/test/java/org/apposed/appose/TypesTest.java @@ -0,0 +1,134 @@ +/*- + * #%L + * Appose: multi-language interprocess cooperation with shared memory. + * %% + * Copyright (C) 2023 Appose developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.apposed.appose; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Tests {@link Types}. + * + * @author Curtis Rueden + */ +public class TypesTest { + + private static String JSON = "{" + + "\"posByte\":123,\"negByte\":-98," + + "\"posDouble\":9.876543210123456,\"negDouble\":-1.234567890987654E302," + + "\"posFloat\":9.876543,\"negFloat\":-1.2345678," + + "\"posInt\":1234567890,\"negInt\":-987654321," + + "\"posLong\":12345678987654321,\"negLong\":-98765432123456789," + + "\"posShort\":32109,\"negShort\":-23456," + + "\"trueBoolean\":true,\"falseBoolean\":false," + + "\"aChar\":\"\\u0000\"," + + "\"aString\":\"-=[]\\\\;',./_+{}|:\\\"<>?" + + "AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz" + + "~!@#$%^&*()\"," + + "\"numbers\":[1,1,2,3,5,8]," + + "\"words\":[\"quick\",\"brown\",\"fox\"]" + + "}"; + + private static String STRING = "-=[]\\;',./_+{}|:\"<>?" + + "AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz" + + "~!@#$%^&*()"; + + private static int[] NUMBERS = {1, 1, 2, 3, 5, 8}; + + private static String[] WORDS = {"quick", "brown", "fox"}; + + @Test + public void testEncode() { + Map data = new LinkedHashMap<>(); + data.put("posByte", (byte) 123); + data.put("negByte", (byte) -98); + data.put("posDouble", 9.876543210123456); + data.put("negDouble", -1.234567890987654E302); + data.put("posFloat", 9.876543f); + data.put("negFloat", -1.2345678f); + data.put("posInt", 1234567890); + data.put("negInt", -987654321); + data.put("posLong", 12345678987654321L); + data.put("negLong", -98765432123456789L); + data.put("posShort", (short) 32109); + data.put("negShort", (short) -23456); + data.put("trueBoolean", true); + data.put("falseBoolean", false); + data.put("aChar", '\0'); + data.put("aString", STRING); + data.put("numbers", NUMBERS); + data.put("words", WORDS); + String json = Types.encode(data); + assertNotNull(json); + assertEquals(JSON, json); + } + + @Test + public void testDecode() throws IOException { + Map data = Types.decode(JSON); + assertNotNull(data); + assertEquals(18, data.size()); + assertEquals(123, data.get("posByte")); // NB: decodes back to int + assertEquals(-98, data.get("negByte")); // NB: decodes back to int + assertEquals(9.876543210123456, bd(data.get("posDouble")).doubleValue()); + assertEquals(-1.234567890987654E302, bd(data.get("negDouble")).doubleValue()); + assertEquals(9.876543f, bd(data.get("posFloat")).floatValue()); + assertEquals(-1.2345678f, bd(data.get("negFloat")).floatValue()); + assertEquals(1234567890, data.get("posInt")); + assertEquals(-987654321, data.get("negInt")); + assertEquals(12345678987654321L, data.get("posLong")); + assertEquals(-98765432123456789L, data.get("negLong")); + assertEquals(32109, data.get("posShort")); // NB: decodes back to int + assertEquals(-23456, data.get("negShort")); // NB: decodes back to int + assertEquals(true, data.get("trueBoolean")); + assertEquals(false, data.get("falseBoolean")); + assertEquals("\0", data.get("aChar")); + assertEquals(STRING, data.get("aString")); + List numbersList = Arrays.stream(NUMBERS) + .boxed() + .collect(Collectors.toList()); + assertEquals(numbersList, data.get("numbers")); + assertEquals(Arrays.asList(WORDS), data.get("words")); + } + + private BigDecimal bd(Object posDouble) { + if (posDouble instanceof BigDecimal) return ((BigDecimal) posDouble); + throw new IllegalArgumentException("Not a BigDecimal: " + posDouble.getClass().getName()); + } +} From 33f7cddc7042cbf8490ddbe688c56abe561c4a71 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 15 Jul 2024 15:38:41 -0500 Subject: [PATCH 10/16] Migrate common shm size checking logic to helper --- src/main/java/org/apposed/appose/shm/ShmLinux.java | 7 +------ src/main/java/org/apposed/appose/shm/ShmMacOS.java | 7 +------ src/main/java/org/apposed/appose/shm/ShmUtils.java | 9 +++++++++ src/main/java/org/apposed/appose/shm/ShmWindows.java | 10 +--------- 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/apposed/appose/shm/ShmLinux.java b/src/main/java/org/apposed/appose/shm/ShmLinux.java index a9dd8f5..087b1f4 100644 --- a/src/main/java/org/apposed/appose/shm/ShmLinux.java +++ b/src/main/java/org/apposed/appose/shm/ShmLinux.java @@ -93,13 +93,8 @@ private SharedMemoryLinux(final String name, final boolean create, final int siz shm_name = withLeadingSlash(name); prevSize = getSHMSize(shm_name); } + ShmUtils.checkSize(shm_name, prevSize, size); - final boolean alreadyExists = prevSize >= 0; - if (alreadyExists && prevSize < size) { - throw new RuntimeException("Shared memory segment already exists with smaller dimensions, data type or format. " - + "Size of the existing shared memory segment cannot be smaller than the size of the proposed object. " - + "Size of existing shared memory segment: " + prevSize + ", size of proposed object: " + size); - } final int shmFd = LibRtOrC.shm_open(shm_name, O_CREAT | O_RDWR, 0666); if (shmFd < 0) { throw new RuntimeException("shm_open failed, errno: " + Native.getLastError()); diff --git a/src/main/java/org/apposed/appose/shm/ShmMacOS.java b/src/main/java/org/apposed/appose/shm/ShmMacOS.java index 5f5115b..7837b33 100644 --- a/src/main/java/org/apposed/appose/shm/ShmMacOS.java +++ b/src/main/java/org/apposed/appose/shm/ShmMacOS.java @@ -64,13 +64,8 @@ private SharedMemoryMacOS(final String name, final boolean create, final int siz shm_name = withLeadingSlash(name); prevSize = getSHMSize(shm_name); } + ShmUtils.checkSize(shm_name, prevSize, size); - final boolean alreadyExists = prevSize >= 0; - if (alreadyExists && prevSize < size) { - throw new RuntimeException("Shared memory segment already exists with smaller dimensions, data type or format. " - + "Size of the existing shared memory segment cannot be smaller than the size of the proposed object. " - + "Size of existing shared memory segment: " + prevSize + ", size of proposed object: " + size); - } // shmFd = INSTANCE.shm_open(this.memoryName, O_RDWR, 0666); final int shmFd = MacosHelpers.INSTANCE.create_shared_memory(shm_name, size); if (shmFd < 0) { diff --git a/src/main/java/org/apposed/appose/shm/ShmUtils.java b/src/main/java/org/apposed/appose/shm/ShmUtils.java index 72fd144..3476fc3 100644 --- a/src/main/java/org/apposed/appose/shm/ShmUtils.java +++ b/src/main/java/org/apposed/appose/shm/ShmUtils.java @@ -112,6 +112,15 @@ static String make_filename(int maxLength, String prefix) { return name; } + static void checkSize(String shmName, long prevSize, int size) { + final boolean alreadyExists = prevSize >= 0; + if (alreadyExists && prevSize < size) { + throw new RuntimeException("Shared memory segment '" + shmName + "' already exists with smaller size. " + + "Size of the existing shared memory segment cannot be smaller than the size of the proposed object. " + + "Size of existing shared memory segment: " + prevSize + ", size of proposed object: " + size); + } + } + private static String token_hex(int nbytes) { final byte[] bytes = new byte[nbytes]; new Random().nextBytes(bytes); diff --git a/src/main/java/org/apposed/appose/shm/ShmWindows.java b/src/main/java/org/apposed/appose/shm/ShmWindows.java index 0bcb1d8..9c3e470 100644 --- a/src/main/java/org/apposed/appose/shm/ShmWindows.java +++ b/src/main/java/org/apposed/appose/shm/ShmWindows.java @@ -65,15 +65,7 @@ private SharedMemoryWindows(final String name, final boolean create, final int s shm_name = nameMangle_TODO(name); prevSize = getSHMSize(shm_name); } - - final boolean alreadyExists = prevSize >= 0; - if(alreadyExists &&prevSize Date: Mon, 15 Jul 2024 16:32:07 -0500 Subject: [PATCH 11/16] Make shm the third, not first, NDArray attribute So that it matches the Python implementation of Appose. The reason it's third in Python is so that it can have the default parameter value of None, to provide feature parity with the Java version's two-argument NDAray constructor that creates the SharedMemory. And create the memory even if the three-argument constructor is used. --- src/main/java/org/apposed/appose/NDArray.java | 32 ++++++++++--------- src/main/java/org/apposed/appose/Types.java | 6 ++-- .../apposed/appose/NDArrayExampleGroovy.java | 3 +- .../apposed/appose/NDArrayExamplePython.java | 3 +- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/apposed/appose/NDArray.java b/src/main/java/org/apposed/appose/NDArray.java index e86084b..7e0bdde 100644 --- a/src/main/java/org/apposed/appose/NDArray.java +++ b/src/main/java/org/apposed/appose/NDArray.java @@ -43,7 +43,7 @@ public class NDArray implements AutoCloseable { /** * shared memory containing the flattened array data. */ - private final SharedMemory sharedMemory; + private final SharedMemory shm; /** * data type of the array elements. @@ -56,16 +56,19 @@ public class NDArray implements AutoCloseable { private final Shape shape; /** - * Constructs an {@code NDArray} with the specified {@code SharedMemory}. + * Constructs an {@code NDArray} with the specified data type, shape, + * and {@code SharedMemory}. * - * @param sharedMemory the flattened array data. * @param dType element data type * @param shape array shape + * @param shm the flattened array data. */ - public NDArray(final SharedMemory sharedMemory, final DType dType, final Shape shape) { - this.sharedMemory = sharedMemory; + public NDArray(final DType dType, final Shape shape, final SharedMemory shm) { this.dType = dType; this.shape = shape; + this.shm = shm == null + ? SharedMemory.create(null, safeInt(shape.numElements() * dType.bytesPerElement())) + : shm; } /** @@ -76,8 +79,7 @@ public NDArray(final SharedMemory sharedMemory, final DType dType, final Shape s * @param shape array shape */ public NDArray(final DType dType, final Shape shape) { - this(SharedMemory.create(null, - safeInt(shape.numElements() * dType.bytesPerElement())), dType, shape); + this(dType, shape, null); } /** @@ -98,7 +100,7 @@ public Shape shape() { * @return The shared memory block containing the array data. */ public SharedMemory shm() { - return sharedMemory; + return shm; } /** @@ -108,7 +110,7 @@ public SharedMemory shm() { */ public ByteBuffer buffer() { final long length = shape.numElements() * dType.bytesPerElement(); - return sharedMemory.pointer().getByteBuffer(0, length); + return shm.pointer().getByteBuffer(0, length); } /** @@ -116,16 +118,16 @@ public ByteBuffer buffer() { */ @Override public void close() throws Exception { - sharedMemory.close(); + shm.close(); } @Override public String toString() { - return "NDArray{" + - "sharedMemory=" + sharedMemory + - ", dType=" + dType + - ", shape=" + shape + - '}'; + return "NDArray(" + + "dType=" + dType + + ", shape=" + shape + + ", shm=" + shm + + ")"; } /** diff --git a/src/main/java/org/apposed/appose/Types.java b/src/main/java/org/apposed/appose/Types.java index 811ff0c..f89bae4 100644 --- a/src/main/java/org/apposed/appose/Types.java +++ b/src/main/java/org/apposed/appose/Types.java @@ -132,9 +132,9 @@ public Object convert(final Object value, final String key) { map.put("name", shm.name()); map.put("size", shm.size()); })).addConverter(convert(NDArray.class, "ndarray", (map, ndArray) -> { - map.put("shm", ndArray.shm()); map.put("dtype", ndArray.dType().label()); map.put("shape", ndArray.shape().toIntArray(C_ORDER)); + map.put("shm", ndArray.shm()); })).build(); @@ -163,10 +163,10 @@ private static Object processValue(Object value) { final int size = (int) map.get("size"); return SharedMemory.attach(name, size); case "ndarray": - final SharedMemory shm = (SharedMemory) map.get("shm"); final NDArray.DType dType = toDType((String) map.get("dtype")); final NDArray.Shape shape = toShape((List) map.get("shape")); - return new NDArray(shm, dType, shape); + final SharedMemory shm = (SharedMemory) map.get("shm"); + return new NDArray(dType, shape, shm); default: System.err.println("unknown appose_type \"" + appose_type + "\""); } diff --git a/src/test/java/org/apposed/appose/NDArrayExampleGroovy.java b/src/test/java/org/apposed/appose/NDArrayExampleGroovy.java index 1467656..0948605 100644 --- a/src/test/java/org/apposed/appose/NDArrayExampleGroovy.java +++ b/src/test/java/org/apposed/appose/NDArrayExampleGroovy.java @@ -13,7 +13,8 @@ public static void main(String[] args) throws Exception { // create a FLOAT32 NDArray with shape (4,3,2) in F_ORDER // respectively (2,3,4) in C_ORDER final NDArray.DType dType = NDArray.DType.FLOAT32; - final NDArray ndArray = new NDArray(dType, new NDArray.Shape(F_ORDER, 4, 3, 2)); + final NDArray.Shape shape = new NDArray.Shape(F_ORDER, 4, 3, 2); + final NDArray ndArray = new NDArray(dType, shape); // fill with values 0..23 in flat iteration order final FloatBuffer buf = ndArray.buffer().asFloatBuffer(); diff --git a/src/test/java/org/apposed/appose/NDArrayExamplePython.java b/src/test/java/org/apposed/appose/NDArrayExamplePython.java index 4a80b39..4512377 100644 --- a/src/test/java/org/apposed/appose/NDArrayExamplePython.java +++ b/src/test/java/org/apposed/appose/NDArrayExamplePython.java @@ -13,7 +13,8 @@ public static void main(String[] args) throws Exception { // create a FLOAT32 NDArray with shape (4,3,2) in F_ORDER // respectively (2,3,4) in C_ORDER final NDArray.DType dType = NDArray.DType.FLOAT32; - final NDArray ndArray = new NDArray(dType, new NDArray.Shape(F_ORDER, 4, 3, 2)); + final NDArray.Shape shape = new NDArray.Shape(F_ORDER, 4, 3, 2); + final NDArray ndArray = new NDArray(dType, shape); // fill with values 0..23 in flat iteration order final FloatBuffer buf = ndArray.buffer().asFloatBuffer(); From 42052dff1770df99756f84c7bb9f9a16ca2668c6 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 17 Jul 2024 14:37:20 -0500 Subject: [PATCH 12/16] Remove blanket `throws Exception` declarations --- src/main/java/org/apposed/appose/NDArray.java | 2 +- src/main/java/org/apposed/appose/SharedMemory.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apposed/appose/NDArray.java b/src/main/java/org/apposed/appose/NDArray.java index 7e0bdde..2f3e3c2 100644 --- a/src/main/java/org/apposed/appose/NDArray.java +++ b/src/main/java/org/apposed/appose/NDArray.java @@ -117,7 +117,7 @@ public ByteBuffer buffer() { * Release resources ({@code SharedMemory}) associated with this {@code NDArray}. */ @Override - public void close() throws Exception { + public void close() { shm.close(); } diff --git a/src/main/java/org/apposed/appose/SharedMemory.java b/src/main/java/org/apposed/appose/SharedMemory.java index d68d13b..a9041e9 100644 --- a/src/main/java/org/apposed/appose/SharedMemory.java +++ b/src/main/java/org/apposed/appose/SharedMemory.java @@ -121,7 +121,7 @@ static SharedMemory createOrAttach(String name, boolean create, int size) { * called once (and only once) across all processes which have access * to the shared memory block. */ - default void unlink() throws Exception { + default void unlink() { throw new UnsupportedOperationException(); } From 136618a77c2668920ceee852d0c4b3f0ccbfed9f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 17 Jul 2024 14:21:27 -0500 Subject: [PATCH 13/16] Do some minor cleanups --- .../java/org/apposed/appose/SharedMemory.java | 2 +- .../java/org/apposed/appose/ApposeTest.java | 46 +++++++++---------- .../java/org/apposed/appose/TypesTest.java | 11 ++--- 3 files changed, 29 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/apposed/appose/SharedMemory.java b/src/main/java/org/apposed/appose/SharedMemory.java index a9041e9..47dcf7b 100644 --- a/src/main/java/org/apposed/appose/SharedMemory.java +++ b/src/main/java/org/apposed/appose/SharedMemory.java @@ -87,7 +87,7 @@ static SharedMemory createOrAttach(String name, boolean create, int size) { throw new IllegalArgumentException("'name' can only be null if create=true"); } ServiceLoader loader = ServiceLoader.load(ShmFactory.class); - for (ShmFactory factory: loader) { + for (ShmFactory factory : loader) { SharedMemory shm = factory.create(name, create, size); if (shm != null) return shm; } diff --git a/src/test/java/org/apposed/appose/ApposeTest.java b/src/test/java/org/apposed/appose/ApposeTest.java index 0804b42..894e733 100644 --- a/src/test/java/org/apposed/appose/ApposeTest.java +++ b/src/test/java/org/apposed/appose/ApposeTest.java @@ -45,27 +45,27 @@ public class ApposeTest { - private static final String COLLATZ_GROOVY = "" + // - "// Computes the stopping time of a given value\n" + // - "// according to the Collatz conjecture sequence.\n" + // - "time = 0\n" + // + private static final String COLLATZ_GROOVY = + "// Computes the stopping time of a given value\n" + + "// according to the Collatz conjecture sequence.\n" + + "time = 0\n" + "BigInteger v = 9999\n" + - "while (v != 1) {\n" + // - " v = v%2==0 ? v/2 : 3*v+1\n" + // - " task.update(\"[${time}] -> ${v}\", time, null)\n" + // - " time++\n" + // - "}\n" + // + "while (v != 1) {\n" + + " v = v%2==0 ? v/2 : 3*v+1\n" + + " task.update(\"[${time}] -> ${v}\", time, null)\n" + + " time++\n" + + "}\n" + "return time\n"; - private static final String COLLATZ_PYTHON = "" + // - "# Computes the stopping time of a given value\n" + // - "# according to the Collatz conjecture sequence.\n" + // - "time = 0\n" + // + private static final String COLLATZ_PYTHON = + "# Computes the stopping time of a given value\n" + + "# according to the Collatz conjecture sequence.\n" + + "time = 0\n" + "v = 9999\n" + - "while v != 1:\n" + // - " v = v//2 if v%2==0 else 3*v+1\n" + // - " task.update(f\"[{time}] -> {v}\", current=time)\n" + // - " time += 1\n" + // + "while v != 1:\n" + + " v = v//2 if v%2==0 else 3*v+1\n" + + " task.update(f\"[{time}] -> {v}\", current=time)\n" + + " time += 1\n" + "task.outputs[\"result\"] = time\n"; @Test @@ -108,12 +108,12 @@ public void executeAndAssert(Service service, String script) // Record the state of the task for each event that occurs. class TaskState { - ResponseType responseType; - TaskStatus status; - String message; - Long current; - Long maximum; - String error; + final ResponseType responseType; + final TaskStatus status; + final String message; + final Long current; + final Long maximum; + final String error; TaskState(TaskEvent event) { responseType = event.responseType; status = event.task.status; diff --git a/src/test/java/org/apposed/appose/TypesTest.java b/src/test/java/org/apposed/appose/TypesTest.java index 12374aa..14badbb 100644 --- a/src/test/java/org/apposed/appose/TypesTest.java +++ b/src/test/java/org/apposed/appose/TypesTest.java @@ -31,7 +31,6 @@ import org.junit.jupiter.api.Test; -import java.io.IOException; import java.math.BigDecimal; import java.util.Arrays; import java.util.LinkedHashMap; @@ -49,7 +48,7 @@ */ public class TypesTest { - private static String JSON = "{" + + private static final String JSON = "{" + "\"posByte\":123,\"negByte\":-98," + "\"posDouble\":9.876543210123456,\"negDouble\":-1.234567890987654E302," + "\"posFloat\":9.876543,\"negFloat\":-1.2345678," + @@ -65,13 +64,13 @@ public class TypesTest { "\"words\":[\"quick\",\"brown\",\"fox\"]" + "}"; - private static String STRING = "-=[]\\;',./_+{}|:\"<>?" + + private static final String STRING = "-=[]\\;',./_+{}|:\"<>?" + "AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz" + "~!@#$%^&*()"; - private static int[] NUMBERS = {1, 1, 2, 3, 5, 8}; + private static final int[] NUMBERS = {1, 1, 2, 3, 5, 8}; - private static String[] WORDS = {"quick", "brown", "fox"}; + private static final String[] WORDS = {"quick", "brown", "fox"}; @Test public void testEncode() { @@ -100,7 +99,7 @@ public void testEncode() { } @Test - public void testDecode() throws IOException { + public void testDecode() { Map data = Types.decode(JSON); assertNotNull(data); assertEquals(18, data.size()); From 7edf5124e24dcd9724a65c76102571058eab341f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 17 Jul 2024 15:40:23 -0500 Subject: [PATCH 14/16] Update license headers for 2024 Thanks, license-maven-plugin! --- LICENSE.txt | 2 +- src/main/c/ShmCreate.c | 66 +++++++++++++------ src/main/java/org/apposed/appose/Appose.java | 2 +- src/main/java/org/apposed/appose/Builder.java | 2 +- .../java/org/apposed/appose/Environment.java | 2 +- .../java/org/apposed/appose/FilePaths.java | 2 +- .../java/org/apposed/appose/GroovyWorker.java | 2 +- src/main/java/org/apposed/appose/NDArray.java | 2 +- src/main/java/org/apposed/appose/Service.java | 2 +- .../java/org/apposed/appose/SharedMemory.java | 2 +- .../java/org/apposed/appose/ShmFactory.java | 28 ++++++++ .../java/org/apposed/appose/TaskEvent.java | 2 +- src/main/java/org/apposed/appose/Types.java | 2 +- .../java/org/apposed/appose/shm/CLibrary.java | 35 ++++++---- .../java/org/apposed/appose/shm/LibRt.java | 35 ++++++---- .../org/apposed/appose/shm/MacosHelpers.java | 35 ++++++---- .../java/org/apposed/appose/shm/ShmLinux.java | 35 ++++++---- .../java/org/apposed/appose/shm/ShmMacOS.java | 35 ++++++---- .../java/org/apposed/appose/shm/ShmUtils.java | 28 ++++++++ .../org/apposed/appose/shm/ShmWindows.java | 37 +++++++---- .../java/org/apposed/appose/ApposeTest.java | 2 +- .../apposed/appose/NDArrayExampleGroovy.java | 28 ++++++++ .../apposed/appose/NDArrayExamplePython.java | 28 ++++++++ .../org/apposed/appose/SharedMemoryTest.java | 2 +- .../java/org/apposed/appose/TypesTest.java | 2 +- 25 files changed, 306 insertions(+), 112 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 3cf9f19..72ac216 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2023, Appose developers. +Copyright (c) 2023 - 2024, Appose developers. All rights reserved. Redistribution and use in source and binary forms, with or without modification, diff --git a/src/main/c/ShmCreate.c b/src/main/c/ShmCreate.c index 236e0c2..32fa208 100644 --- a/src/main/c/ShmCreate.c +++ b/src/main/c/ShmCreate.c @@ -1,3 +1,31 @@ +/*- + * #%L + * Appose: multi-language interprocess cooperation with shared memory. + * %% + * Copyright (C) 2023 - 2024 Appose developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ #include #include #include @@ -15,25 +43,25 @@ long get_shared_memory_size(int fd) { return (long)shm_stat.st_size; } -// Function to create a shared memory segment, modified to accept a long for size -int create_shared_memory(const char *name, long size) { - int fd = shm_open(name, O_CREAT | O_RDWR, 0666); - if (fd < 0) { - perror("shm_open"); - return -1; - } - long already_size = get_shared_memory_size(fd); - if (already_size > 0) { - return fd; - } - - if (ftruncate(fd, size) == -1) { - perror("ftruncate"); - close(fd); - return -1; - } - - return fd; +// Function to create a shared memory segment, modified to accept a long for size +int create_shared_memory(const char *name, long size) { + int fd = shm_open(name, O_CREAT | O_RDWR, 0666); + if (fd < 0) { + perror("shm_open"); + return -1; + } + long already_size = get_shared_memory_size(fd); + if (already_size > 0) { + return fd; + } + + if (ftruncate(fd, size) == -1) { + perror("ftruncate"); + close(fd); + return -1; + } + + return fd; } // Function to unlink a shared memory segment diff --git a/src/main/java/org/apposed/appose/Appose.java b/src/main/java/org/apposed/appose/Appose.java index ac94885..8bf7dd4 100644 --- a/src/main/java/org/apposed/appose/Appose.java +++ b/src/main/java/org/apposed/appose/Appose.java @@ -2,7 +2,7 @@ * #%L * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2023 Appose developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/main/java/org/apposed/appose/Builder.java b/src/main/java/org/apposed/appose/Builder.java index 18106f4..10cc756 100644 --- a/src/main/java/org/apposed/appose/Builder.java +++ b/src/main/java/org/apposed/appose/Builder.java @@ -2,7 +2,7 @@ * #%L * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2023 Appose developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/main/java/org/apposed/appose/Environment.java b/src/main/java/org/apposed/appose/Environment.java index 9fec5f0..1fe7204 100644 --- a/src/main/java/org/apposed/appose/Environment.java +++ b/src/main/java/org/apposed/appose/Environment.java @@ -2,7 +2,7 @@ * #%L * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2023 Appose developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/main/java/org/apposed/appose/FilePaths.java b/src/main/java/org/apposed/appose/FilePaths.java index e347119..1a3d2ee 100644 --- a/src/main/java/org/apposed/appose/FilePaths.java +++ b/src/main/java/org/apposed/appose/FilePaths.java @@ -2,7 +2,7 @@ * #%L * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2023 Appose developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/main/java/org/apposed/appose/GroovyWorker.java b/src/main/java/org/apposed/appose/GroovyWorker.java index 32c1625..7c64d92 100644 --- a/src/main/java/org/apposed/appose/GroovyWorker.java +++ b/src/main/java/org/apposed/appose/GroovyWorker.java @@ -2,7 +2,7 @@ * #%L * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2023 Appose developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/main/java/org/apposed/appose/NDArray.java b/src/main/java/org/apposed/appose/NDArray.java index 2f3e3c2..6257de7 100644 --- a/src/main/java/org/apposed/appose/NDArray.java +++ b/src/main/java/org/apposed/appose/NDArray.java @@ -2,7 +2,7 @@ * #%L * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2023 Appose developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/main/java/org/apposed/appose/Service.java b/src/main/java/org/apposed/appose/Service.java index a128e27..1d3ea3d 100644 --- a/src/main/java/org/apposed/appose/Service.java +++ b/src/main/java/org/apposed/appose/Service.java @@ -2,7 +2,7 @@ * #%L * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2023 Appose developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/main/java/org/apposed/appose/SharedMemory.java b/src/main/java/org/apposed/appose/SharedMemory.java index 47dcf7b..3bbba9b 100644 --- a/src/main/java/org/apposed/appose/SharedMemory.java +++ b/src/main/java/org/apposed/appose/SharedMemory.java @@ -2,7 +2,7 @@ * #%L * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2023 Appose developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/main/java/org/apposed/appose/ShmFactory.java b/src/main/java/org/apposed/appose/ShmFactory.java index 2b54f2d..6fabda1 100644 --- a/src/main/java/org/apposed/appose/ShmFactory.java +++ b/src/main/java/org/apposed/appose/ShmFactory.java @@ -1,3 +1,31 @@ +/*- + * #%L + * Appose: multi-language interprocess cooperation with shared memory. + * %% + * Copyright (C) 2023 - 2024 Appose developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ package org.apposed.appose; public interface ShmFactory { diff --git a/src/main/java/org/apposed/appose/TaskEvent.java b/src/main/java/org/apposed/appose/TaskEvent.java index 9b8002d..0e5ff78 100644 --- a/src/main/java/org/apposed/appose/TaskEvent.java +++ b/src/main/java/org/apposed/appose/TaskEvent.java @@ -2,7 +2,7 @@ * #%L * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2023 Appose developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/main/java/org/apposed/appose/Types.java b/src/main/java/org/apposed/appose/Types.java index f89bae4..e2b842e 100644 --- a/src/main/java/org/apposed/appose/Types.java +++ b/src/main/java/org/apposed/appose/Types.java @@ -2,7 +2,7 @@ * #%L * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2023 Appose developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/main/java/org/apposed/appose/shm/CLibrary.java b/src/main/java/org/apposed/appose/shm/CLibrary.java index f3a0214..caca1e3 100644 --- a/src/main/java/org/apposed/appose/shm/CLibrary.java +++ b/src/main/java/org/apposed/appose/shm/CLibrary.java @@ -1,20 +1,29 @@ /*- * #%L - * Use deep learning frameworks from Java in an agnostic and isolated way. + * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. * #L% */ package org.apposed.appose.shm; diff --git a/src/main/java/org/apposed/appose/shm/LibRt.java b/src/main/java/org/apposed/appose/shm/LibRt.java index 81ed482..28fd0f0 100644 --- a/src/main/java/org/apposed/appose/shm/LibRt.java +++ b/src/main/java/org/apposed/appose/shm/LibRt.java @@ -1,20 +1,29 @@ /*- * #%L - * Use deep learning frameworks from Java in an agnostic and isolated way. + * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. * #L% */ package org.apposed.appose.shm; diff --git a/src/main/java/org/apposed/appose/shm/MacosHelpers.java b/src/main/java/org/apposed/appose/shm/MacosHelpers.java index bd3fe0d..0fa281b 100644 --- a/src/main/java/org/apposed/appose/shm/MacosHelpers.java +++ b/src/main/java/org/apposed/appose/shm/MacosHelpers.java @@ -1,20 +1,29 @@ /*- * #%L - * Use deep learning frameworks from Java in an agnostic and isolated way. + * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. * #L% */ diff --git a/src/main/java/org/apposed/appose/shm/ShmLinux.java b/src/main/java/org/apposed/appose/shm/ShmLinux.java index 087b1f4..893dd0d 100644 --- a/src/main/java/org/apposed/appose/shm/ShmLinux.java +++ b/src/main/java/org/apposed/appose/shm/ShmLinux.java @@ -1,20 +1,29 @@ /*- * #%L - * Use deep learning frameworks from Java in an agnostic and isolated way. + * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. * #L% */ diff --git a/src/main/java/org/apposed/appose/shm/ShmMacOS.java b/src/main/java/org/apposed/appose/shm/ShmMacOS.java index 7837b33..a3a9daa 100644 --- a/src/main/java/org/apposed/appose/shm/ShmMacOS.java +++ b/src/main/java/org/apposed/appose/shm/ShmMacOS.java @@ -1,20 +1,29 @@ /*- * #%L - * Use deep learning frameworks from Java in an agnostic and isolated way. + * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. * #L% */ diff --git a/src/main/java/org/apposed/appose/shm/ShmUtils.java b/src/main/java/org/apposed/appose/shm/ShmUtils.java index 3476fc3..afd6bbf 100644 --- a/src/main/java/org/apposed/appose/shm/ShmUtils.java +++ b/src/main/java/org/apposed/appose/shm/ShmUtils.java @@ -1,3 +1,31 @@ +/*- + * #%L + * Appose: multi-language interprocess cooperation with shared memory. + * %% + * Copyright (C) 2023 - 2024 Appose developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ package org.apposed.appose.shm; import java.util.Random; diff --git a/src/main/java/org/apposed/appose/shm/ShmWindows.java b/src/main/java/org/apposed/appose/shm/ShmWindows.java index 9c3e470..d64ebdd 100644 --- a/src/main/java/org/apposed/appose/shm/ShmWindows.java +++ b/src/main/java/org/apposed/appose/shm/ShmWindows.java @@ -1,20 +1,29 @@ /*- * #%L - * Use deep learning frameworks from Java in an agnostic and isolated way. + * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. * #L% */ @@ -262,4 +271,4 @@ public String toString() { '}'; } } -} \ No newline at end of file +} diff --git a/src/test/java/org/apposed/appose/ApposeTest.java b/src/test/java/org/apposed/appose/ApposeTest.java index 894e733..3b82c55 100644 --- a/src/test/java/org/apposed/appose/ApposeTest.java +++ b/src/test/java/org/apposed/appose/ApposeTest.java @@ -2,7 +2,7 @@ * #%L * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2023 Appose developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/test/java/org/apposed/appose/NDArrayExampleGroovy.java b/src/test/java/org/apposed/appose/NDArrayExampleGroovy.java index 0948605..b3bc045 100644 --- a/src/test/java/org/apposed/appose/NDArrayExampleGroovy.java +++ b/src/test/java/org/apposed/appose/NDArrayExampleGroovy.java @@ -1,3 +1,31 @@ +/*- + * #%L + * Appose: multi-language interprocess cooperation with shared memory. + * %% + * Copyright (C) 2023 - 2024 Appose developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ package org.apposed.appose; import java.nio.FloatBuffer; diff --git a/src/test/java/org/apposed/appose/NDArrayExamplePython.java b/src/test/java/org/apposed/appose/NDArrayExamplePython.java index 4512377..e37582c 100644 --- a/src/test/java/org/apposed/appose/NDArrayExamplePython.java +++ b/src/test/java/org/apposed/appose/NDArrayExamplePython.java @@ -1,3 +1,31 @@ +/*- + * #%L + * Appose: multi-language interprocess cooperation with shared memory. + * %% + * Copyright (C) 2023 - 2024 Appose developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ package org.apposed.appose; import java.nio.FloatBuffer; diff --git a/src/test/java/org/apposed/appose/SharedMemoryTest.java b/src/test/java/org/apposed/appose/SharedMemoryTest.java index be8b13c..59b4e6a 100644 --- a/src/test/java/org/apposed/appose/SharedMemoryTest.java +++ b/src/test/java/org/apposed/appose/SharedMemoryTest.java @@ -2,7 +2,7 @@ * #%L * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2023 Appose developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/src/test/java/org/apposed/appose/TypesTest.java b/src/test/java/org/apposed/appose/TypesTest.java index 14badbb..f90607e 100644 --- a/src/test/java/org/apposed/appose/TypesTest.java +++ b/src/test/java/org/apposed/appose/TypesTest.java @@ -2,7 +2,7 @@ * #%L * Appose: multi-language interprocess cooperation with shared memory. * %% - * Copyright (C) 2023 Appose developers. + * Copyright (C) 2023 - 2024 Appose developers. * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: From 7f2f57b23511a78ead18723c1e232f7b951b7fc0 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 15 Jul 2024 16:15:33 -0500 Subject: [PATCH 15/16] Test NDArray<->JSON en/de-coding --- .../java/org/apposed/appose/TypesTest.java | 50 ++++++++++++++++--- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/apposed/appose/TypesTest.java b/src/test/java/org/apposed/appose/TypesTest.java index f90607e..06e7e39 100644 --- a/src/test/java/org/apposed/appose/TypesTest.java +++ b/src/test/java/org/apposed/appose/TypesTest.java @@ -40,6 +40,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; /** * Tests {@link Types}. @@ -61,8 +62,18 @@ public class TypesTest { "AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz" + "~!@#$%^&*()\"," + "\"numbers\":[1,1,2,3,5,8]," + - "\"words\":[\"quick\",\"brown\",\"fox\"]" + - "}"; + "\"words\":[\"quick\",\"brown\",\"fox\"]," + + "\"ndArray\":{" + + "\"appose_type\":\"ndarray\"," + + "\"dtype\":\"float32\"," + + "\"shape\":[2,20,25]," + + "\"shm\":{" + + "\"appose_type\":\"shm\"," + + "\"name\":\"SHM_NAME\"," + + "\"size\":4000" + + "}" + + "}" + + "}"; private static final String STRING = "-=[]\\;',./_+{}|:\"<>?" + "AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz" + @@ -93,16 +104,31 @@ public void testEncode() { data.put("aString", STRING); data.put("numbers", NUMBERS); data.put("words", WORDS); - String json = Types.encode(data); - assertNotNull(json); - assertEquals(JSON, json); + NDArray.DType dtype = NDArray.DType.FLOAT32; + NDArray.Shape shape = new NDArray.Shape(NDArray.Shape.Order.C_ORDER, 2, 20, 25); + try (NDArray ndArray = new NDArray(dtype, shape)) { + data.put("ndArray", ndArray); + String json = Types.encode(data); + assertNotNull(json); + String expected = JSON.replaceAll("SHM_NAME", ndArray.shm().name()); + assertEquals(expected, json); + } } @Test public void testDecode() { - Map data = Types.decode(JSON); + Map data; + String shmName; + + // Create name shared memory segment and decode JSON block. + try (SharedMemory shm = SharedMemory.create(null, 4000)) { + shmName = shm.name(); + data = Types.decode(JSON.replaceAll("SHM_NAME", shmName)); + } + + // Validate results. assertNotNull(data); - assertEquals(18, data.size()); + assertEquals(19, data.size()); assertEquals(123, data.get("posByte")); // NB: decodes back to int assertEquals(-98, data.get("negByte")); // NB: decodes back to int assertEquals(9.876543210123456, bd(data.get("posDouble")).doubleValue()); @@ -124,6 +150,16 @@ public void testDecode() { .collect(Collectors.toList()); assertEquals(numbersList, data.get("numbers")); assertEquals(Arrays.asList(WORDS), data.get("words")); + try (NDArray ndArray = (NDArray) data.get("ndArray")) { + assertSame(NDArray.DType.FLOAT32, ndArray.dType()); + assertEquals(NDArray.Shape.Order.C_ORDER, ndArray.shape().order()); + assertEquals(3, ndArray.shape().length()); + assertEquals(2, ndArray.shape().get(0)); + assertEquals(20, ndArray.shape().get(1)); + assertEquals(25, ndArray.shape().get(2)); + assertEquals(shmName, ndArray.shm().name()); + assertEquals(4000, ndArray.shm().size()); + } } private BigDecimal bd(Object posDouble) { From 5aa52d4505a93bdb1e1998f78fc9f3a2cc0ac9c9 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 17 Jul 2024 21:06:48 -0500 Subject: [PATCH 16/16] Split common Shm logic to base class And decouple the close and unlink functions. There was a lot of duplicate code between ShmLinux, ShmMacOS, and ShmWindows, which is now in the internal ShmBase class. This made it easier to have consistent logic around the close vs. unlink behavior across platforms, with less copy/pasted code. --- src/main/java/org/apposed/appose/NDArray.java | 2 +- .../java/org/apposed/appose/SharedMemory.java | 22 ++- .../java/org/apposed/appose/ShmFactory.java | 13 +- .../java/org/apposed/appose/shm/ShmBase.java | 130 ++++++++++++++ .../java/org/apposed/appose/shm/ShmLinux.java | 111 +++--------- .../java/org/apposed/appose/shm/ShmMacOS.java | 118 ++++--------- .../org/apposed/appose/shm/ShmWindows.java | 167 ++++++------------ 7 files changed, 277 insertions(+), 286 deletions(-) create mode 100644 src/main/java/org/apposed/appose/shm/ShmBase.java diff --git a/src/main/java/org/apposed/appose/NDArray.java b/src/main/java/org/apposed/appose/NDArray.java index 6257de7..62e19b4 100644 --- a/src/main/java/org/apposed/appose/NDArray.java +++ b/src/main/java/org/apposed/appose/NDArray.java @@ -33,7 +33,7 @@ import java.util.Arrays; /** - * Represents a multi-dimensional array similar to NumPy ndarray. + * Represents a multidimensional array similar to a NumPy ndarray. *

    * The array contains elements of a {@link DType data type}, arranged in a * particular {@link Shape}, and flattened into {@link SharedMemory}, diff --git a/src/main/java/org/apposed/appose/SharedMemory.java b/src/main/java/org/apposed/appose/SharedMemory.java index 3bbba9b..073d4d5 100644 --- a/src/main/java/org/apposed/appose/SharedMemory.java +++ b/src/main/java/org/apposed/appose/SharedMemory.java @@ -106,7 +106,7 @@ static SharedMemory createOrAttach(String name, boolean create, int size) { * * @return The length in bytes of the shared memory. */ - long size(); + int size(); /** * JNA pointer to the shared memory segment. @@ -115,19 +115,29 @@ static SharedMemory createOrAttach(String name, boolean create, int size) { */ Pointer pointer(); + /** + * Sets whether the {@link #unlink()} method should be invoked to destroy + * the shared memory block when the {@link #close()} method is called. + *

    + * By default, shared memory objects constructed with {@link #create} will + * behave this way, whereas shared memory objects constructed with + * {@link #attach} will not. But this method allows to override the behavior. + *

    + */ + void unlinkOnClose(boolean unlinkOnClose); + /** * Requests that the underlying shared memory block be destroyed. * In order to ensure proper cleanup of resources, unlink should be * called once (and only once) across all processes which have access * to the shared memory block. */ - default void unlink() { - throw new UnsupportedOperationException(); - } + void unlink(); /** - * Closes access to the shared memory from this instance but does - * not destroy the shared memory block. + * Closes access to the shared memory from this instance, + * but does not necessarily destroy the shared memory block. + * See also {@link #unlinkOnClose(boolean)}. */ @Override void close(); diff --git a/src/main/java/org/apposed/appose/ShmFactory.java b/src/main/java/org/apposed/appose/ShmFactory.java index 6fabda1..717d436 100644 --- a/src/main/java/org/apposed/appose/ShmFactory.java +++ b/src/main/java/org/apposed/appose/ShmFactory.java @@ -28,6 +28,17 @@ */ package org.apposed.appose; +/** + * Interface for platform-specific creation of {@link SharedMemory} instances. + *

    + * Each platform (e.g. Linux, macOS, Windows) provides its own implementation + * of this interface, which knows how to manufacture {@link SharedMemory} blocks + * for that platform. These implementations are declared as implementations in + * {@code META-INF/services/org.apposed.appose.ShmFactory}, so that Java's + * {@code ServiceLoader} can discover them in an extensible way, and then use + * the one best suited for the platform at hand. + *

    + */ public interface ShmFactory { -SharedMemory create(String name, boolean create, int size); + SharedMemory create(String name, boolean create, int size); } diff --git a/src/main/java/org/apposed/appose/shm/ShmBase.java b/src/main/java/org/apposed/appose/shm/ShmBase.java new file mode 100644 index 0000000..83b6349 --- /dev/null +++ b/src/main/java/org/apposed/appose/shm/ShmBase.java @@ -0,0 +1,130 @@ +/*- + * #%L + * Appose: multi-language interprocess cooperation with shared memory. + * %% + * Copyright (C) 2023 - 2024 Appose developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.apposed.appose.shm; + +import com.sun.jna.Pointer; +import org.apposed.appose.SharedMemory; + +/** + * Base class for platform-specific shared memory implementations. + * + * @author Carlos Garcia Lopez de Haro + * @author Tobias Pietzsch + * @author Curtis Rueden + */ +abstract class ShmBase implements SharedMemory { + + /** Struct containing shm details, including name, size, pointer(s), and handle. */ + protected final ShmInfo info; + + /** Whether the memory block has been closed. */ + private boolean closed; + + /** Whether the memory block has been unlinked. */ + private boolean unlinked; + + protected ShmBase(final ShmInfo info) { + this.info = info; + } + + protected abstract void doClose(); + protected abstract void doUnlink(); + + @Override + public String name() { + return info.name; + } + + @Override + public int size() { + return info.size; + } + + @Override + public Pointer pointer() { + return info.pointer; + } + + @Override + public void unlinkOnClose(boolean unlinkOnClose) { + info.unlinkOnClose = unlinkOnClose; + } + + @Override + public synchronized void unlink() { + if (unlinked) throw new IllegalStateException("Shared memory '" + info.name + "' is already unlinked."); + doClose(); + doUnlink(); + unlinked = true; + } + + @Override + public synchronized void close() { + if (closed) return; + doClose(); + if (info.unlinkOnClose) doUnlink(); + closed = true; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("{"); + sb.append("name='").append(name()).append("', size=").append(size()); + if (info.pointer != null) sb.append(", pointer=").append(info.pointer); + if (info.writePointer != null) sb.append(", writePointer=").append(info.writePointer); + if (info.handle != null) sb.append("handle=").append(info.handle); + sb.append(", closed=").append(closed); + sb.append(", unlinked=").append(unlinked); + sb.append("}"); + return sb.toString(); + } + + /** Struct containing details about this shared memory block. */ + protected static class ShmInfo { + + /** Unique name that identifies the shared memory segment. */ + String name; + + /** Size in bytes. */ + int size; + + /** Pointer referencing the shared memory. */ + Pointer pointer; + + Pointer writePointer; + + /** File handle for the shared memory block's (pseudo-)file. */ + HANDLE handle; + + /** Whether to destroy the shared memory block when {@link #close()} is called. */ + boolean unlinkOnClose; + } +} diff --git a/src/main/java/org/apposed/appose/shm/ShmLinux.java b/src/main/java/org/apposed/appose/shm/ShmLinux.java index 893dd0d..85f021d 100644 --- a/src/main/java/org/apposed/appose/shm/ShmLinux.java +++ b/src/main/java/org/apposed/appose/shm/ShmLinux.java @@ -47,12 +47,10 @@ /** * Linux-specific shared memory implementation. - *

    - * TODO separate unlink and close - *

    * * @author Carlos Garcia Lopez de Haro * @author Tobias Pietzsch + * @author Curtis Rueden */ public class ShmLinux implements ShmFactory { @@ -62,35 +60,32 @@ public SharedMemory create(final String name, final boolean create, final int si return new SharedMemoryLinux(name, create, size); } - private static class SharedMemoryLinux implements SharedMemory { + private static class SharedMemoryLinux extends ShmBase { - /** - * File descriptor - */ - private final int fd; - - /** - * Size in bytes - */ - private final int size; + // name without leading slash + private SharedMemoryLinux(final String name, final boolean create, final int size) { + super(prepareShm(name, create, size)); + } - /** - * Pointer referencing the shared memory - */ - private final Pointer pointer; + @Override + protected void doUnlink() { + LibRtOrC.shm_unlink(name()); + } - /** - * Unique name that identifies the shared memory segment. - */ - private final String name; + @Override + protected void doClose() { + // Unmap the shared memory + if (pointer() != Pointer.NULL && LibRtOrC.munmap(pointer(), size()) == -1) { + throw new RuntimeException("munmap failed. Errno: " + Native.getLastError()); + } - /** - * Whether the memory block has been closed and unlinked - */ - private boolean unlinked = false; + // Close the file descriptor + if (LibRtOrC.close(info.handle) == -1) { + throw new RuntimeException("close failed. Errno: " + Native.getLastError()); + } + } - // name without leading slash - private SharedMemoryLinux(final String name, final boolean create, final int size) { + private static ShmInfo prepareShm(String name, boolean create, int size) { String shm_name; long prevSize; if (name == null) { @@ -123,49 +118,13 @@ private SharedMemoryLinux(final String name, final boolean create, final int siz throw new RuntimeException("mmap failed, errno: " + Native.getLastError()); } - this.size = shm_size; - this.name = withoutLeadingSlash(shm_name); - this.fd = shmFd; - this.pointer = pointer; - } - - @Override - public String name() { - return name; - } - - @Override - public Pointer pointer() { - return pointer; - } - - @Override - public long size() { - return size; - } - - /** - * Unmap and close the shared memory. Necessary to eliminate the shared memory block - */ - @Override - public synchronized void close() { - if (unlinked) { - return; - } - - // Unmap the shared memory - if (this.pointer != Pointer.NULL && LibRtOrC.munmap(this.pointer, size) == -1) { - throw new RuntimeException("munmap failed. Errno: " + Native.getLastError()); - } - - // Close the file descriptor - if (LibRtOrC.close(this.fd) == -1) { - throw new RuntimeException("close failed. Errno: " + Native.getLastError()); - } - - // Unlink the shared memory object - LibRtOrC.shm_unlink(this.name); - unlinked = true; + ShmInfo info = new ShmInfo<>(); + info.size = shm_size; + info.name = withoutLeadingSlash(shm_name); + info.pointer = pointer; + info.handle = shmFd; + info.unlinkOnClose = create; + return info; } /** @@ -202,18 +161,6 @@ private static long getSHMSize(final int shmFd) { return size; } - @Override - public String toString() { - return "ShmLinux{" + - "fd=" + fd + - ", size=" + size + - ", pointer=" + pointer + - ", name='" + name + '\'' + - ", unlinked=" + unlinked + - '}'; - } - - private static class LibRtOrC { /** diff --git a/src/main/java/org/apposed/appose/shm/ShmMacOS.java b/src/main/java/org/apposed/appose/shm/ShmMacOS.java index a3a9daa..bb4d405 100644 --- a/src/main/java/org/apposed/appose/shm/ShmMacOS.java +++ b/src/main/java/org/apposed/appose/shm/ShmMacOS.java @@ -45,12 +45,10 @@ /** * MacOS-specific shared memory implementation. - *

    - * TODO separate unlink and close - *

    * * @author Carlos Garcia Lopez de Haro * @author Tobias Pietzsch + * @author Curtis Rueden */ public class ShmMacOS implements ShmFactory { @@ -60,8 +58,30 @@ public SharedMemory create(final String name, final boolean create, final int si return new SharedMemoryMacOS(name, create, size); } - private static class SharedMemoryMacOS implements SharedMemory { + private static class SharedMemoryMacOS extends ShmBase { private SharedMemoryMacOS(final String name, final boolean create, final int size) { + super(prepareShm(name, create, size)); + } + + @Override + protected void doUnlink() { + CLibrary.INSTANCE.shm_unlink(name()); + } + + @Override + protected void doClose() { + // Unmap the shared memory + if (pointer() != Pointer.NULL && CLibrary.INSTANCE.munmap(pointer(), size()) == -1) { + throw new RuntimeException("munmap failed. Errno: " + Native.getLastError()); + } + + // Close the file descriptor + if (CLibrary.INSTANCE.close(info.handle) == -1) { + throw new RuntimeException("close failed. Errno: " + Native.getLastError()); + } + } + + private static ShmInfo prepareShm(String name, boolean create, int size) { String shm_name; long prevSize; if (name == null) { @@ -75,7 +95,7 @@ private SharedMemoryMacOS(final String name, final boolean create, final int siz } ShmUtils.checkSize(shm_name, prevSize, size); - // shmFd = INSTANCE.shm_open(this.memoryName, O_RDWR, 0666); + // shmFd = INSTANCE.shm_open(this.memoryName, O_RDWR, 0666); final int shmFd = MacosHelpers.INSTANCE.create_shared_memory(shm_name, size); if (shmFd < 0) { throw new RuntimeException("shm_open failed, errno: " + Native.getLastError()); @@ -89,78 +109,15 @@ private SharedMemoryMacOS(final String name, final boolean create, final int siz throw new RuntimeException("mmap failed, errno: " + Native.getLastError()); } - this.size = shm_size; - this.name = withoutLeadingSlash(shm_name); - this.fd = shmFd; - this.pointer = pointer; + ShmInfo info = new ShmInfo<>(); + info.size = shm_size; + info.name = withoutLeadingSlash(shm_name); + info.pointer = pointer; + info.handle = shmFd; + info.unlinkOnClose = create; + return info; } - /** - * File descriptor - */ - private final int fd; - - /** - * Size in bytes - */ - private final int size; - - /** - * Pointer referencing the shared memory - */ - private final Pointer pointer; - - /** - * Unique name that identifies the shared memory segment. - */ - private final String name; - - /** - * Whether the memory block has been closed and unlinked - */ - private boolean unlinked = false; - - @Override - public String name() { - return name; - } - - @Override - public Pointer pointer() { - return pointer; - } - - @Override - public long size() { - return size; - } - - /** - * Unmap and close the shared memory. Necessary to eliminate the shared memory block - */ - @Override - public synchronized void close() { - if (unlinked) { - return; - } - - // Unmap the shared memory - if (this.pointer != Pointer.NULL && CLibrary.INSTANCE.munmap(this.pointer, size) == -1) { - throw new RuntimeException("munmap failed. Errno: " + Native.getLastError()); - } - - // Close the file descriptor - if (CLibrary.INSTANCE.close(this.fd) == -1) { - throw new RuntimeException("close failed. Errno: " + Native.getLastError()); - } - - // Unlink the shared memory object - CLibrary.INSTANCE.shm_unlink(this.name); - unlinked = true; - } - - // name without leading slash - /** * Try to open {@code name} and get its size in bytes. * @@ -194,16 +151,5 @@ private static long getSHMSize(final int shmFd) { } return size; } - - @Override - public String toString() { - return "ShmMacOS{" + - "fd=" + fd + - ", size=" + size + - ", pointer=" + pointer + - ", name='" + name + '\'' + - ", unlinked=" + unlinked + - '}'; - } } } diff --git a/src/main/java/org/apposed/appose/shm/ShmWindows.java b/src/main/java/org/apposed/appose/shm/ShmWindows.java index d64ebdd..b52c91e 100644 --- a/src/main/java/org/apposed/appose/shm/ShmWindows.java +++ b/src/main/java/org/apposed/appose/shm/ShmWindows.java @@ -52,145 +52,96 @@ public class ShmWindows implements ShmFactory { // name is WITHOUT prefix etc + @Override public SharedMemory create(final String name, final boolean create, final int size) { if (ShmUtils.os != ShmUtils.OS.WINDOWS) return null; // wrong platform return new SharedMemoryWindows(name, create, size); } - private static class SharedMemoryWindows implements SharedMemory { - String shm_name; - long prevSize; + private static class SharedMemoryWindows extends ShmBase { + private SharedMemoryWindows(final String name, final boolean create, final int size) { - if(name ==null) + super(prepareShm(name, create, size)); + } + + @Override + protected void doClose() { + cleanup(info.pointer, info.writePointer, info.handle); + } + + @Override + protected void doUnlink() { + // Note: The shared memory object will be deleted when all processes + // have closed their handles to it. There's no direct "unlink" + // equivalent in Windows like there is in POSIX systems. + } - { + private static ShmInfo prepareShm(String name, boolean create, int size) { + String shm_name; + long prevSize; + if (name == null) { do { shm_name = ShmUtils.make_filename(SHM_SAFE_NAME_LENGTH, SHM_NAME_PREFIX_WIN); prevSize = getSHMSize(shm_name); } while (prevSize >= 0); - } else - - { + } else { shm_name = nameMangle_TODO(name); prevSize = getSHMSize(shm_name); } ShmUtils.checkSize(shm_name, prevSize, size); - final WinNT.HANDLE shm_hMapFile = Kernel32.INSTANCE.CreateFileMapping( - WinBase.INVALID_HANDLE_VALUE, - null, - WinNT.PAGE_READWRITE, - 0, - size, - shm_name + final WinNT.HANDLE hMapFile = Kernel32.INSTANCE.CreateFileMapping( + WinBase.INVALID_HANDLE_VALUE, + null, + WinNT.PAGE_READWRITE, + 0, + size, + shm_name ); - if(hMapFile ==null) - - { + if (hMapFile == null) { throw new RuntimeException("Error creating shared memory array. CreateFileMapping failed: " + Kernel32.INSTANCE.GetLastError()); } - final int shm_size = (int) getSHMSize(shm_hMapFile); + final int shm_size = (int) getSHMSize(hMapFile); // Map the shared memory Pointer pointer = Kernel32.INSTANCE.MapViewOfFile( - hMapFile, - WinNT.FILE_MAP_WRITE, - 0, - 0, - size + hMapFile, + WinNT.FILE_MAP_WRITE, + 0, + 0, + size ); - if(pointer ==null) - - { + if (isNull(pointer)) { Kernel32.INSTANCE.CloseHandle(hMapFile); throw new RuntimeException("Error creating shared memory array. " + Kernel32.INSTANCE.GetLastError()); } Pointer writePointer = Kernel32.INSTANCE.VirtualAllocEx(Kernel32.INSTANCE.GetCurrentProcess(), - pointer, new BaseTSD.SIZE_T(size), WinNT.MEM_COMMIT, WinNT.PAGE_READWRITE); - if(writePointer ==null) - - { - close(); + pointer, new BaseTSD.SIZE_T(size), WinNT.MEM_COMMIT, WinNT.PAGE_READWRITE); + if (isNull(writePointer)) { + cleanup(pointer, writePointer, hMapFile); throw new RuntimeException("Error committing to the shared memory pages. Errno: " + Kernel32.INSTANCE.GetLastError()); } - this.size =shm_size; - this.name = - - nameUnmangle_TODO(shm_name); - this.hMapFile =shm_hMapFile; - this.pointer =pointer; - this.writePointer =writePointer; - } - - /** - * reference to the file that covers the shared memory region - */ - private WinNT.HANDLE hMapFile; - - /** - * Size in bytes - */ - private final int size; - - /** - * Pointer referencing the shared memory - */ - private final Pointer pointer; - - private final Pointer writePointer; - - /** - * Unique name that identifies the shared memory segment. - */ - private final String name; - - /** - * Whether the memory block has been closed and unlinked - */ - private boolean unlinked = false; - - @Override - public String name() { - return name; - } - - @Override - public Pointer pointer() { - return pointer; - } - - @Override - public long size() { - return size; - } - - /** - * Unmap and close the shared memory. Necessary to eliminate the shared memory block - */ - @Override - public synchronized void close() { - if (unlinked) { - return; - } - if (writePointer != null) { - Kernel32.INSTANCE.UnmapViewOfFile(this.writePointer); - } - Kernel32.INSTANCE.UnmapViewOfFile(pointer); - Kernel32.INSTANCE.CloseHandle(hMapFile); - unlinked = true; + ShmInfo info = new ShmInfo<>(); + info.size = shm_size; + info.name = nameUnmangle_TODO(shm_name); + info.pointer = pointer; + info.writePointer = writePointer; + info.handle = hMapFile; + info.unlinkOnClose = create; + return info; } // TODO equivalent of removing slash - private String nameUnmangle_TODO (String memoryName){ + private static String nameUnmangle_TODO (String memoryName){ return memoryName; } // TODO equivalent of adding slash // Do we need the "Local\" prefix? - private String nameMangle_TODO (String memoryName){ + private static String nameMangle_TODO (String memoryName){ // if (!memoryName.startsWith("Local" + File.separator) && !memoryName.startsWith("Global" + File.separator)) // memoryName = "Local" + File.separator + memoryName; return memoryName; @@ -258,17 +209,13 @@ private static long getSHMSize(final WinNT.HANDLE hMapFile) { return size; } + } - @Override - public String toString() { - return "ShmWindows{" + - "hMapFile=" + hMapFile + - ", size=" + size + - ", pointer=" + pointer + - ", writePointer=" + writePointer + - ", name='" + name + '\'' + - ", unlinked=" + unlinked + - '}'; - } + private static void cleanup(Pointer pointer, Pointer writePointer, WinNT.HANDLE handle) { + if (!isNull(writePointer)) Kernel32.INSTANCE.UnmapViewOfFile(writePointer); + if (!isNull(pointer)) Kernel32.INSTANCE.UnmapViewOfFile(pointer); + if (handle != null) Kernel32.INSTANCE.CloseHandle(handle); } + + private static boolean isNull(Pointer p) { return p == null || p == Pointer.NULL; } }