From 3b7005b38863bbd0f778ca430da4bfa9127e354a Mon Sep 17 00:00:00 2001 From: Preston Landers Date: Sun, 4 Dec 2016 09:56:10 -0600 Subject: [PATCH 1/7] Temporarily avoid the use of `get_module_constant` which appears to be broken in Python 3.6 --- memcache.py | 2 +- setup.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/memcache.py b/memcache.py index 9823a2f..79f13de 100644 --- a/memcache.py +++ b/memcache.py @@ -95,7 +95,7 @@ def useOldServerHashFunction(): # Original author: Evan Martin of Danga Interactive __author__ = "Sean Reifschneider " -__version__ = "1.58" +__version__ = "1.58.2" __copyright__ = "Copyright (C) 2003 Danga Interactive" # http://en.wikipedia.org/wiki/Python_Software_Foundation_License __license__ = "Python Software Foundation License" diff --git a/setup.py b/setup.py index c1dd198..a93a2b2 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,12 @@ #!/usr/bin/env python -from setuptools.depends import get_module_constant +# from setuptools.depends import get_module_constant from setuptools import setup # noqa setup(name="python-memcached", - version=get_module_constant('memcache', '__version__'), + # version=get_module_constant('memcache', '__version__'), + version='1.58.2', description="Pure python memcached client", long_description=open("README.md").read(), author="Evan Martin", From 118553383f04568c70397979e2d55de773793c5d Mon Sep 17 00:00:00 2001 From: Preston Landers Date: Mon, 5 Dec 2016 10:42:33 -0600 Subject: [PATCH 2/7] Add `raw_bytes` client option to return the bytes instead of trying to decode to Unicode string. --- memcache.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/memcache.py b/memcache.py index 79f13de..d225b41 100644 --- a/memcache.py +++ b/memcache.py @@ -176,7 +176,8 @@ def __init__(self, servers, debug=0, pickleProtocol=0, pload=None, pid=None, server_max_key_length=None, server_max_value_length=None, dead_retry=_DEAD_RETRY, socket_timeout=_SOCKET_TIMEOUT, - cache_cas=False, flush_on_reconnect=0, check_keys=True): + cache_cas=False, flush_on_reconnect=0, check_keys=True, + raw_bytes=False): """Create a new Client object with the given list of servers. @param servers: C{servers} is passed to L{set_servers}. @@ -219,6 +220,8 @@ def __init__(self, servers, debug=0, pickleProtocol=0, @param check_keys: (default True) If True, the key is checked to ensure it is the correct length and composed of the right characters. + @param raw_bytes: (default False) if True, we return raw + bytes instead of trying to decode to Unicode string. """ super(Client, self).__init__() self.debug = debug @@ -230,6 +233,7 @@ def __init__(self, servers, debug=0, pickleProtocol=0, self.cache_cas = cache_cas self.reset_cas() self.do_check_key = check_keys + self.raw_bytes = raw_bytes # Allow users to modify pickling/unpickling behavior self.pickleProtocol = pickleProtocol @@ -1267,7 +1271,7 @@ def _recv_value(self, server, flags, rlen): if flags == 0: # Bare string - if six.PY3: + if not self.raw_bytes and six.PY3: val = buf.decode('utf8') else: val = buf From 74a7d90df3910e368431aa814552dc7c69f65975 Mon Sep 17 00:00:00 2001 From: Preston Landers Date: Wed, 7 Dec 2016 15:24:22 -0600 Subject: [PATCH 3/7] Fix the following resource warning for unclosed sockets on connections errors: ``` ResourceWarning: unclosed ``` --- memcache.py | 57 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/memcache.py b/memcache.py index d225b41..4cdfe75 100644 --- a/memcache.py +++ b/memcache.py @@ -69,6 +69,8 @@ def cmemcache_hash(key): return ( (((binascii.crc32(key) & 0xffffffff) >> 16) & 0x7fff) or 1) + + serverHashFunction = cmemcache_hash @@ -77,7 +79,9 @@ def useOldServerHashFunction(): global serverHashFunction serverHashFunction = binascii.crc32 + from io import BytesIO + if six.PY2: try: unicode @@ -92,7 +96,6 @@ def useOldServerHashFunction(): valid_key_chars_re = re.compile(b'[\x21-\x7e\x80-\xff]+$') - # Original author: Evan Martin of Danga Interactive __author__ = "Sean Reifschneider " __version__ = "1.58.2" @@ -250,7 +253,7 @@ def __init__(self, servers, debug=0, pickleProtocol=0, if self.server_max_value_length is None: self.server_max_value_length = SERVER_MAX_VALUE_LENGTH - # figure out the pickler style + # figure out the pickler style file = BytesIO() try: pickler = self.pickler(file, protocol=self.pickleProtocol) @@ -261,7 +264,7 @@ def __init__(self, servers, debug=0, pickleProtocol=0, def _encode_key(self, key): if isinstance(key, tuple): if isinstance(key[1], six.text_type): - return (key[0], key[1].encode('utf8')) + return key[0], key[1].encode('utf8') elif isinstance(key, six.text_type): return key.encode('utf8') return key @@ -347,7 +350,7 @@ def get_stats(self, stat_args=None): stats = line.decode('ascii').split(' ', 2) serverData[stats[1]] = stats[2] - return(data) + return data def get_slab_stats(self): data = [] @@ -370,7 +373,7 @@ def get_slab_stats(self): break item = line.split(' ', 2) if line.startswith('STAT active_slabs') or line.startswith('STAT total_malloced'): - serverData[item[1]]=item[2] + serverData[item[1]] = item[2] else: # 0 = STAT, 1 = ITEM, 2 = Value slab = item[1].split(':', 2) @@ -827,7 +830,7 @@ def _map_and_prefix_keys(self, key_iterable, key_prefix): bytes_orig_key = key server, key = self._get_server(key_prefix + key) - # alert when passed in key is None + # alert when passed in key is None if orig_key is None: self.check_key(orig_key, key_extra_len=key_extra_len) @@ -843,7 +846,7 @@ def _map_and_prefix_keys(self, key_iterable, key_prefix): server_keys[server].append(key) prefixed_to_orig_key[key] = orig_key - return (server_keys, prefixed_to_orig_key) + return server_keys, prefixed_to_orig_key def set_multi(self, mapping, time=0, key_prefix='', min_compress_len=0, noreply=False): @@ -947,7 +950,7 @@ def set_multi(self, mapping, time=0, key_prefix='', min_compress_len=0, for server in dead_servers: del server_keys[server] - # short-circuit if there are no servers, just return all keys + # short-circuit if there are no servers, just return all keys if not server_keys: return list(mapping.keys()) @@ -1013,12 +1016,12 @@ def _val_to_store_info(self, val, min_compress_len): flags |= Client._FLAG_COMPRESSED val = comp_val - # silently do not store if value length exceeds maximum + # silently do not store if value length exceeds maximum if (self.server_max_value_length != 0 and - len(val) > self.server_max_value_length): - return(0) + len(val) > self.server_max_value_length): + return 0 - return (flags, len(val), val) + return flags, len(val), val def _set(self, cmd, key, val, time, min_compress_len=0, noreply=False): key = self._encode_key(key) @@ -1037,7 +1040,7 @@ def _unsafe_set(): store_info = self._val_to_store_info(val, min_compress_len) if not store_info: - return(0) + return 0 flags, len_val, encoded_val = store_info if cmd == 'cas': @@ -1052,8 +1055,8 @@ def _unsafe_set(): server.send_cmd(fullcmd) if noreply: return True - return(server.expect(b"STORED", raise_exception=True) - == b"STORED") + return (server.expect(b"STORED", raise_exception=True) + == b"STORED") except socket.error as msg: if isinstance(msg, tuple): msg = msg[1] @@ -1239,9 +1242,9 @@ def _expect_cas_value(self, server, line=None, raise_exception=False): if line and line[:5] == b'VALUE': resp, rkey, flags, len, cas_id = line.split() - return (rkey, int(flags), int(len), int(cas_id)) + return rkey, int(flags), int(len), int(cas_id) else: - return (None, None, None, None) + return None, None, None, None def _expectvalue(self, server, line=None, raise_exception=False): if not line: @@ -1251,9 +1254,9 @@ def _expectvalue(self, server, line=None, raise_exception=False): resp, rkey, flags, len = line.split() flags = int(flags) rlen = int(len) - return (rkey, flags, rlen) + return rkey, flags, rlen else: - return (None, None, None) + return None, None, None def _recv_value(self, server, flags, rlen): rlen += 2 # include \r\n @@ -1318,14 +1321,14 @@ def check_key(self, key, key_extra_len=0): if key_extra_len is 0: raise Client.MemcachedKeyNoneError("Key is empty") - # key is empty but there is some other component to key + # key is empty but there is some other component to key return if not isinstance(key, six.binary_type): raise Client.MemcachedKeyTypeError("Key must be a binary string") if (self.server_max_key_length != 0 and - len(key) + key_extra_len > self.server_max_key_length): + len(key) + key_extra_len > self.server_max_key_length): raise Client.MemcachedKeyLengthError( "Key length is > %s" % self.server_max_key_length ) @@ -1335,7 +1338,6 @@ def check_key(self, key, key_extra_len=0): class _Host(object): - def __init__(self, host, debug=0, dead_retry=_DEAD_RETRY, socket_timeout=_SOCKET_TIMEOUT, flush_on_reconnect=0): self.dead_retry = dead_retry @@ -1347,7 +1349,7 @@ def __init__(self, host, debug=0, dead_retry=_DEAD_RETRY, else: self.weight = 1 - # parse the connection string + # parse the connection string m = re.match(r'^(?Punix):(?P.*)$', host) if not m: m = re.match(r'^(?Pinet6):' @@ -1414,13 +1416,21 @@ def _get_socket(self): try: s.connect(self.address) except socket.timeout as msg: + s.close() + # print("timeout! %s" % (msg,)) self.mark_dead("connect: %s" % msg) return None except socket.error as msg: + s.close() if isinstance(msg, tuple): msg = msg[1] + # print("error! %s" % (msg,)) self.mark_dead("connect: %s" % msg) return None + except Exception as e: + print("Other error: %s" % (e,)) + self.debuglog("Other error: %s" % (e,)) + raise self.socket = s self.buffer = b'' if self.flush_on_next_connect: @@ -1526,5 +1536,4 @@ def _doctest(): if results.failed: sys.exit(1) - # vim: ts=4 sw=4 et : From fe1ef1a0c4051fc1a9303943b88f87391f55d3d9 Mon Sep 17 00:00:00 2001 From: Preston Landers Date: Tue, 25 Jul 2017 10:48:57 -0500 Subject: [PATCH 4/7] Add missing --- .gitignore | 1 + memcache.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index d705d4e..a80745b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ /python_memcached.egg-info .tox .coverage +.idea diff --git a/memcache.py b/memcache.py index cec0a07..9e8fbef 100644 --- a/memcache.py +++ b/memcache.py @@ -1041,8 +1041,8 @@ def _unsafe_set(): server.send_cmd(fullcmd) if noreply: return True - returnserver.expect(b"STORED", raise_exception=True) - == b"STORED" + return (server.expect(b"STORED", raise_exception=True) + == b"STORED") except socket.error as msg: if isinstance(msg, tuple): msg = msg[1] From fbc10a2ea29fcde6a043cf4ed48731c3bc2848e6 Mon Sep 17 00:00:00 2001 From: Preston Landers Date: Tue, 25 Jul 2017 10:58:04 -0500 Subject: [PATCH 5/7] Revert some formatting changes to make a cleaner PR for `raw_bytes` option. --- ChangeLog | 2 ++ memcache.py | 27 ++++++++++++--------------- setup.py | 5 ++--- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/ChangeLog b/ChangeLog index b56ac0d..afc237f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ + * Added `raw_bytes` option to avoid decoding bytes to str + * Added testing for Python 3.5 (PR from Tim Graham) #110 * Fixed typos in docstrings (PR from Romuald Brunet, reviewed by Tim diff --git a/memcache.py b/memcache.py index 9e8fbef..1fe7000 100644 --- a/memcache.py +++ b/memcache.py @@ -66,9 +66,7 @@ def cmemcache_hash(key): - return ( - ((binascii.crc32(key) & 0xffffffff) - >> 16) & 0x7fff) or 1 + return (((binascii.crc32(key) & 0xffffffff) >> 16) & 0x7fff) or 1 serverHashFunction = cmemcache_hash @@ -248,7 +246,7 @@ def __init__(self, servers, debug=0, pickleProtocol=0, def _encode_key(self, key): if isinstance(key, tuple): if isinstance(key[1], six.text_type): - return key[0], key[1].encode('utf8') + return (key[0], key[1].encode('utf8')) elif isinstance(key, six.text_type): return key.encode('utf8') return key @@ -832,7 +830,7 @@ def _map_and_prefix_keys(self, key_iterable, key_prefix): server_keys[server].append(key) prefixed_to_orig_key[key] = orig_key - return server_keys, prefixed_to_orig_key + return (server_keys, prefixed_to_orig_key) def set_multi(self, mapping, time=0, key_prefix='', min_compress_len=0, noreply=False): @@ -1007,7 +1005,7 @@ def _val_to_store_info(self, val, min_compress_len): len(val) > self.server_max_value_length): return 0 - return flags, len(val), val + return (flags, len(val), val) def _set(self, cmd, key, val, time, min_compress_len=0, noreply=False): key = self._encode_key(key) @@ -1041,8 +1039,7 @@ def _unsafe_set(): server.send_cmd(fullcmd) if noreply: return True - return (server.expect(b"STORED", raise_exception=True) - == b"STORED") + return server.expect(b"STORED", raise_exception=True) == b"STORED" except socket.error as msg: if isinstance(msg, tuple): msg = msg[1] @@ -1228,9 +1225,9 @@ def _expect_cas_value(self, server, line=None, raise_exception=False): if line and line[:5] == b'VALUE': resp, rkey, flags, len, cas_id = line.split() - return rkey, int(flags), int(len), int(cas_id) + return (rkey, int(flags), int(len), int(cas_id)) else: - return None, None, None, None + return (None, None, None, None) def _expectvalue(self, server, line=None, raise_exception=False): if not line: @@ -1240,9 +1237,9 @@ def _expectvalue(self, server, line=None, raise_exception=False): resp, rkey, flags, len = line.split() flags = int(flags) rlen = int(len) - return rkey, flags, rlen + return (rkey, flags, rlen) else: - return None, None, None + return (None, None, None) def _recv_value(self, server, flags, rlen): rlen += 2 # include \r\n @@ -1324,6 +1321,7 @@ def check_key(self, key, key_extra_len=0): class _Host(object): + def __init__(self, host, debug=0, dead_retry=_DEAD_RETRY, socket_timeout=_SOCKET_TIMEOUT, flush_on_reconnect=0): self.dead_retry = dead_retry @@ -1403,18 +1401,16 @@ def _get_socket(self): s.connect(self.address) except socket.timeout as msg: s.close() - # print("timeout! %s" % (msg,)) self.mark_dead("connect: %s" % msg) return None except socket.error as msg: s.close() if isinstance(msg, tuple): msg = msg[1] - # print("error! %s" % (msg,)) self.mark_dead("connect: %s" % msg) return None except Exception as e: - print("Other error: %s" % (e,)) + # print("Other error: %s" % (e,)) self.debuglog("Other error: %s" % (e,)) raise self.socket = s @@ -1522,4 +1518,5 @@ def _doctest(): if results.failed: sys.exit(1) + # vim: ts=4 sw=4 et : diff --git a/setup.py b/setup.py index d2b0b1d..af05fcf 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,11 @@ #!/usr/bin/env python -# from setuptools.depends import get_module_constant +from setuptools.depends import get_module_constant from setuptools import setup # noqa setup(name="python-memcached", - # version=get_module_constant('memcache', '__version__'), - version='1.58.2', + version=get_module_constant('memcache', '__version__'), description="Pure python memcached client", long_description=open("README.md").read(), author="Evan Martin", From fb41c3efcf7d3eeff6e89cb11900f2f11d37a49c Mon Sep 17 00:00:00 2001 From: Preston Landers Date: Tue, 25 Jul 2017 11:05:08 -0500 Subject: [PATCH 6/7] Undo version change for clean PR --- memcache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/memcache.py b/memcache.py index 1fe7000..d06e92d 100644 --- a/memcache.py +++ b/memcache.py @@ -78,9 +78,10 @@ def useOldServerHashFunction(): valid_key_chars_re = re.compile(b'[\x21-\x7e\x80-\xff]+$') + # Original author: Evan Martin of Danga Interactive __author__ = "Sean Reifschneider " -__version__ = "1.58.2" +__version__ = "1.58" __copyright__ = "Copyright (C) 2003 Danga Interactive" # http://en.wikipedia.org/wiki/Python_Software_Foundation_License __license__ = "Python Software Foundation License" From 92937d7eea2e409d58f5947f4e48a91830aec5f5 Mon Sep 17 00:00:00 2001 From: Preston Landers Date: Tue, 25 Jul 2017 11:16:55 -0500 Subject: [PATCH 7/7] Possibly fix PyLint warnings in travis --- memcache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/memcache.py b/memcache.py index d06e92d..58ab804 100644 --- a/memcache.py +++ b/memcache.py @@ -1002,8 +1002,8 @@ def _val_to_store_info(self, val, min_compress_len): val = comp_val # silently do not store if value length exceeds maximum - if (self.server_max_value_length != 0 and - len(val) > self.server_max_value_length): + if ((self.server_max_value_length != 0 + and len(val) > self.server_max_value_length)): return 0 return (flags, len(val), val) @@ -1311,8 +1311,8 @@ def check_key(self, key, key_extra_len=0): if not isinstance(key, six.binary_type): raise Client.MemcachedKeyTypeError("Key must be a binary string") - if (self.server_max_key_length != 0 and - len(key) + key_extra_len > self.server_max_key_length): + if ((self.server_max_key_length != 0 + and len(key) + key_extra_len > self.server_max_key_length)): raise Client.MemcachedKeyLengthError( "Key length is > %s" % self.server_max_key_length )