From 4eecc130bdaf68a9f250230efcabe1f0a0598cf0 Mon Sep 17 00:00:00 2001 From: Jason Madden Date: Sun, 29 Mar 2015 10:54:42 -0500 Subject: [PATCH 1/5] Make build and install under PyPy. Replaces PyObject_Malloc/Free with their PyMem counterparts; PyPy does not support the PyObject variants. Arguably, based on https://docs.python.org/2.3/whatsnew/section-pymalloc.html the PyMem variants are a better fit for this use case anyway (PyObject_ being intended for "small" allocations). Disables the use of CP1250 under PyPy because it lacks the PyUnicode_Encode function. There are no new tests failures. (testConnectWithWrongDB fails with a 1044 error for me and not 1049 with both the original code under CPython and this code under PyPy). --- python/umysql.c | 11 ++++++++--- setup.py | 16 +++++++++------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/python/umysql.c b/python/umysql.c index 2564e97..c568ed3 100644 --- a/python/umysql.c +++ b/python/umysql.c @@ -769,11 +769,12 @@ PyObject *Connection_isConnected(Connection *self, PyObject *args) Py_RETURN_FALSE; } +#ifndef PYPY_VERSION PyObject *PyUnicode_EncodeCP1250Helper(const Py_UNICODE *data, Py_ssize_t length, const char *errors) { return PyUnicode_Encode (data, length, "cp1250", errors); } - +#endif PyObject *HandleError(Connection *self, const char *funcName) { @@ -878,8 +879,12 @@ PyObject *Connection_connect(Connection *self, PyObject *args) else if (strcmp (pstrCharset, "cp1250") == 0) { +#ifndef PYPY_VERSION self->charset = MCS_cp1250_general_ci; self->PFN_PyUnicode_Encode = PyUnicode_EncodeCP1250Helper; +#else + return PyErr_Format (PyExc_ValueError, "Unsupported character set '%s' specified", pstrCharset); +#endif } else if (strcmp (pstrCharset, "utf8mb4") == 0) @@ -1093,7 +1098,7 @@ PyObject *EscapeQueryArguments(Connection *self, PyObject *inQuery, PyObject *it { /* FIXME: Allocate a PyString and resize it just like the Python code does it */ - obuffer = (char *) PyObject_Malloc(cbOutQuery); + obuffer = (char *) PyMem_Malloc(cbOutQuery); heap = 1; } else @@ -1124,7 +1129,7 @@ PyObject *EscapeQueryArguments(Connection *self, PyObject *inQuery, PyObject *it if (*iptr != 's' && *iptr != '%') { Py_DECREF(iterator); - if (heap) PyObject_Free(obuffer); + if (heap) PyMem_Free(obuffer); return PyErr_Format (PyExc_ValueError, "Found character %c expected %%", *iptr); } diff --git a/setup.py b/setup.py index 81383bc..58b6918 100644 --- a/setup.py +++ b/setup.py @@ -68,6 +68,8 @@ Programming Language :: Python Topic :: Database Topic :: Software Development :: Libraries :: Python Modules +Programming Language :: Python :: Implementation :: CPython +Programming Language :: Python :: Implementation :: PyPy """.splitlines())) """ @@ -75,13 +77,13 @@ shutil.rmtree("./build") except(OSError): pass -""" - +""" + libs = [] if sys.platform != "win32": libs.append("stdc++") - + if sys.platform == "win32": libs.append("ws2_32") @@ -92,16 +94,16 @@ library_dirs = [], libraries=libs, define_macros=[('WIN32_LEAN_AND_MEAN', None)]) - + setup (name = 'umysql', - version = "2.61", + version = "2.62.dev0", description = "Ultra fast MySQL driver for Python", ext_modules = [module1], author="Jonas Tarnstrom", author_email="jonas.tarnstrom@esn.me", download_url="http://github.com/esnme/ultramysql", license="BSD License", - platforms=['any'], + platforms=['any'], url="http://www.esn.me", classifiers=CLASSIFIERS, - ) + ) From 9d14d3cdc566a1f92451d3ef919396a5a4297359 Mon Sep 17 00:00:00 2001 From: Jason Madden Date: Wed, 27 Jul 2016 12:26:34 -0500 Subject: [PATCH 2/5] Defrag the read buffer when it gets full. If we are reading many packets of different sizes from a fast database server, it's possible that the skip() method of the packet reader will never find the start and end pointers aligned and so it will never reset them. This can lead to throwing 'Socket receive buffer full' errors when we reach the end, even if there's plenty of unused space at the start of the buffer from previous packets. This checks for this situation and moves the live data and current pointers to the start of the buffer, thus making room (if possible). --- lib/Connection.cpp | 13 +++++++------ lib/PacketReader.cpp | 25 +++++++++++++++++++------ lib/PacketReader.h | 3 ++- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/lib/Connection.cpp b/lib/Connection.cpp index fcacef6..8d6f516 100644 --- a/lib/Connection.cpp +++ b/lib/Connection.cpp @@ -68,10 +68,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define snprintf _snprintf #endif -//#define PRINTMARK() fprintf(stderr, "%08x:%s:%s MARK(%d)\n", GetTickCount(), __FILE__, __FUNCTION__, __LINE__) -#define PRINTMARK() +//#define PRINTMARK() fprintf(stderr, "%08x:%s:%s MARK(%d)\n", GetTickCount(), __FILE__, __FUNCTION__, __LINE__) +#define PRINTMARK() -Connection::Connection (UMConnectionCAPI *_capi) +Connection::Connection (UMConnectionCAPI *_capi) : m_reader(MYSQL_RX_BUFFER_SIZE) , m_writer(MYSQL_TX_BUFFER_SIZE) { @@ -127,15 +127,16 @@ void Connection::scramble(const char *_scramble1, const char *_scramble2, UINT8 for (int index = 0; index < 20; index ++) { - _outToken[index] = final_hash[index] ^ stage1_hash[index]; + _outToken[index] = final_hash[index] ^ stage1_hash[index]; } } bool Connection::readSocket() { + m_reader.defrag(); size_t bytesToRecv = m_reader.getEndPtr() - m_reader.getWritePtr(); - assert (bytesToRecv <= m_reader.getEndPtr() - m_reader.getWritePtr()); + assert (bytesToRecv <= m_reader.getEndPtr() - m_reader.getWritePtr()); if (bytesToRecv == 0) { @@ -207,7 +208,7 @@ bool Connection::close(void) m_writer.finalize(0); if (!sendPacket()) - { + { m_capi.clearException(); } } diff --git a/lib/PacketReader.cpp b/lib/PacketReader.cpp index 4acc321..0e206b5 100644 --- a/lib/PacketReader.cpp +++ b/lib/PacketReader.cpp @@ -96,6 +96,22 @@ void PacketReader::skip() } } +void PacketReader::defrag(void) +{ + if (m_writeCursor == m_buffEnd && m_readCursor > m_buffStart) { + assert (m_packetEnd == NULL); + size_t bytesToCopy = m_writeCursor - m_readCursor; + //fprintf(stderr, "%s: Transferring %zu bytes to beginning\n", __FUNCTION__, bytesToCopy); + memmove(m_buffStart, m_readCursor, bytesToCopy); + assert (*m_buffStart == *m_readCursor); + assert (m_buffStart[1] == m_readCursor[1]); + m_readCursor = m_buffStart; + m_writeCursor = m_buffStart + bytesToCopy; + memset(m_writeCursor + 1, 0, m_buffEnd - m_writeCursor - 1); + } + +} + void PacketReader::push(size_t _cbData) { //fprintf (stderr, "%s: Pushing %u bytes\n", __FUNCTION__, _cbData); @@ -249,20 +265,20 @@ UINT8 *PacketReader::readLengthCodedBinary(size_t *_outLen) case 252: m_readCursor ++; *_outLen = (size_t) *((UINT16 *) m_readCursor); - m_readCursor += 2; + m_readCursor += 2; break; case 253: m_readCursor ++; *_outLen = (size_t) *((UINT32 *) m_readCursor); *_outLen &= 0xffffff; - m_readCursor += 3; + m_readCursor += 3; break; case 254: m_readCursor ++; *_outLen = (size_t) *((UINT64 *) m_readCursor); - m_readCursor += 8; + m_readCursor += 8; break; } @@ -326,6 +342,3 @@ UINT64 PacketReader::readLengthCodedInteger() return ret; } - - - diff --git a/lib/PacketReader.h b/lib/PacketReader.h index 557f4ad..735dbfa 100644 --- a/lib/PacketReader.h +++ b/lib/PacketReader.h @@ -76,6 +76,7 @@ class PacketReader PacketReader (size_t cbSize); ~PacketReader (void); void skip(); + void defrag(); void push(size_t _cbData); char *getWritePtr(); char *getStartPtr(); @@ -95,4 +96,4 @@ class PacketReader UINT8 *readLengthCodedBinary(size_t *_outLen); }; -#endif \ No newline at end of file +#endif From d86184e72fbfefb6f1fc54c5ab997f2e11e4506a Mon Sep 17 00:00:00 2001 From: Carlos Sanchez Date: Wed, 27 Jul 2016 14:49:47 -0500 Subject: [PATCH 3/5] add to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 94d4066..def8295 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ /build *.pyc +umysql.egg-info +umysql.so \ No newline at end of file From 16653f8e36a569ab061a555f46c1a6bb3fefbbaf Mon Sep 17 00:00:00 2001 From: Carlos Sanchez Date: Sat, 25 Mar 2017 17:26:43 -0500 Subject: [PATCH 4/5] git ignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index def8295..c6e8698 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ /build *.pyc umysql.egg-info -umysql.so \ No newline at end of file +umysql.so +/.project +/.pydevproject From 6e4bf3af30493a3543e45b691db6132675421306 Mon Sep 17 00:00:00 2001 From: Jason Madden Date: Mon, 8 Jul 2019 15:41:16 -0500 Subject: [PATCH 5/5] Docs-deprecate because of https://github.com/zodb/relstorage/issues/264 --- README | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README b/README index d36779a..9a655e8 100644 --- a/README +++ b/README @@ -1,10 +1,12 @@ +:: WARNING :: +This driver is no longer supported or recommended. See https://github.com/zodb/relstorage/issues/264 + :: Description :: -A fast MySQL driver written in pure C/C++ for Python. +A fast MySQL driver written in pure C/C++ for Python. Compatible with gevent through monkey patching :: Requirements :: -Requires Python (http://www.python.org) +Requires Python (http://www.python.org) :: Installation :: python setup.py build install -