From e269c0997124b5c9aae983a2c240711a03b4d392 Mon Sep 17 00:00:00 2001 From: furkanonder Date: Thu, 18 Apr 2024 22:23:55 +0300 Subject: [PATCH 01/59] Use custom loads & dumps instead of custom pickler & unpickler for Shelf --- Lib/shelve.py | 36 +++++++++++++++++---------- Lib/test/test_shelve.py | 54 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 13 deletions(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 50584716e9ea64..73f84b87d5f8ba 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -56,13 +56,18 @@ the persistent dictionary on disk, if feasible). """ -from pickle import DEFAULT_PROTOCOL, Pickler, Unpickler +from pickle import DEFAULT_PROTOCOL, Unpickler, dumps, loads from io import BytesIO import collections.abc __all__ = ["Shelf", "BsdDbShelf", "DbfilenameShelf", "open"] + +class ShelveError(Exception): + pass + + class _ClosedDict(collections.abc.MutableMapping): 'Marker for a closed dict. Access attempts raise a ValueError.' @@ -82,7 +87,7 @@ class Shelf(collections.abc.MutableMapping): """ def __init__(self, dict, protocol=None, writeback=False, - keyencoding="utf-8"): + keyencoding="utf-8", *, serializer=None, deserializer=None): self.dict = dict if protocol is None: protocol = DEFAULT_PROTOCOL @@ -91,6 +96,15 @@ def __init__(self, dict, protocol=None, writeback=False, self.cache = {} self.keyencoding = keyencoding + if serializer is None and deserializer is None: + self.serializer = dumps + self.deserializer = loads + elif (serializer is None) ^ (deserializer is None): + raise ShelveError("Serializer and deserializer must be defined together.") + else: + self.serializer = serializer + self.deserializer = deserializer + def __iter__(self): for k in self.dict.keys(): yield k.decode(self.keyencoding) @@ -110,8 +124,8 @@ def __getitem__(self, key): try: value = self.cache[key] except KeyError: - f = BytesIO(self.dict[key.encode(self.keyencoding)]) - value = Unpickler(f).load() + f = self.dict[key.encode(self.keyencoding)] + value = self.deserializer(f) if self.writeback: self.cache[key] = value return value @@ -119,10 +133,7 @@ def __getitem__(self, key): def __setitem__(self, key, value): if self.writeback: self.cache[key] = value - f = BytesIO() - p = Pickler(f, self._protocol) - p.dump(value) - self.dict[key.encode(self.keyencoding)] = f.getvalue() + self.dict[key.encode(self.keyencoding)] = self.serializer(value, self._protocol) def __delitem__(self, key): del self.dict[key.encode(self.keyencoding)] @@ -222,9 +233,9 @@ class DbfilenameShelf(Shelf): See the module's __doc__ string for an overview of the interface. """ - def __init__(self, filename, flag='c', protocol=None, writeback=False): + def __init__(self, filename, flag='c', protocol=None, writeback=False, serializer=None, deserializer=None): import dbm - Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback) + Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback, serializer=serializer, deserializer=deserializer) def clear(self): """Remove all items from the shelf.""" @@ -233,8 +244,7 @@ def clear(self): self.cache.clear() self.dict.clear() - -def open(filename, flag='c', protocol=None, writeback=False): +def open(filename, flag='c', protocol=None, writeback=False, *, serializer=None, deserializer=None): """Open a persistent dictionary for reading and writing. The filename parameter is the base filename for the underlying @@ -247,4 +257,4 @@ def open(filename, flag='c', protocol=None, writeback=False): See the module's __doc__ string for an overview of the interface. """ - return DbfilenameShelf(filename, flag, protocol, writeback) + return DbfilenameShelf(filename, flag, protocol, writeback, serializer, deserializer) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 08c6562f2a273e..3fdd9f36393416 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -3,6 +3,8 @@ import shelve import pickle import os +from io import BytesIO +from pydoc import locate from test.support import os_helper from collections.abc import MutableMapping @@ -165,6 +167,58 @@ def test_default_protocol(self): with shelve.Shelf({}) as s: self.assertEqual(s._protocol, pickle.DEFAULT_PROTOCOL) + def test_custom_serializer_and_deserializer(self): + def serializer(obj, protocol=None): + return bytes(f"{type(obj).__name__}", 'utf-8') + + def deserializer(data): + value = BytesIO(data).read() + return locate(value.decode("utf-8")) + + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: + num = 1 + s['number'] = num + self.assertEqual(s['number'], type(num)) + + with self.assertRaises(AssertionError): + def serializer(obj, protocol=None): + return bytes(f"{type(obj).__name__}", 'utf-8') + + def deserializer(data): + pass + + with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: + s['number'] = 100 + self.assertEqual(s['number'], 100) + + with self.assertRaises(dbm.sqlite3.error): + def serializer(obj, protocol=None): + pass + + def deserializer(data): + return BytesIO(data).read().decode("utf-8") + + with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: + s['number'] = 100 + self.assertEqual(s['number'], 100) + + def test_missing_custom_deserializer(self): + def serializer(obj, protocol=None): + pass + + with self.assertRaises(shelve.ShelveError): + shelve.Shelf({}, protocol=2, writeback=False, serializer=serializer) + + def test_missing_custom_serializer(self): + def deserializer(data): + pass + + with self.assertRaises(shelve.ShelveError): + shelve.Shelf({}, protocol=2, writeback=False, deserializer=deserializer) + class TestShelveBase: type2test = shelve.Shelf From 346514923a946725e8579149db0cee5bfa4983ad Mon Sep 17 00:00:00 2001 From: furkanonder Date: Thu, 18 Apr 2024 22:45:28 +0300 Subject: [PATCH 02/59] Allow custom loads & dumps instead of custom pickler & unpickler for BsdDbShelf --- Lib/shelve.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 73f84b87d5f8ba..3ee63404a687b1 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -197,7 +197,15 @@ class BsdDbShelf(Shelf): """ def __init__(self, dict, protocol=None, writeback=False, - keyencoding="utf-8"): + keyencoding="utf-8", *, serializer=None, deserializer=None): + if serializer is None and deserializer is None: + self.serializer = dumps + self.deserializer = loads + elif (serializer is None) ^ (deserializer is None): + raise ShelveError("Serializer and deserializer must be defined together.") + else: + self.serializer = serializer + self.deserializer = deserializer Shelf.__init__(self, dict, protocol, writeback, keyencoding) def set_location(self, key): From 44b9fa1e9735f646b6dbc2a08b606eb8ac97ea5c Mon Sep 17 00:00:00 2001 From: furkanonder Date: Thu, 18 Apr 2024 22:49:05 +0300 Subject: [PATCH 03/59] Update documentation for serializer and deserializred functions --- Doc/library/shelve.rst | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 95c54991887022..c8086478ac927b 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -17,7 +17,7 @@ This includes most class instances, recursive data types, and objects containing lots of shared sub-objects. The keys are ordinary strings. -.. function:: open(filename, flag='c', protocol=None, writeback=False) +.. function:: open(filename, flag='c', protocol=None, writeback=False, *, serializer=None, deserializer=None) Open a persistent dictionary. The filename specified is the base filename for the underlying database. As a side-effect, an extension may be added to the @@ -41,6 +41,14 @@ lots of shared sub-objects. The keys are ordinary strings. determine which accessed entries are mutable, nor which ones were actually mutated). + By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` + for serializing and deserializing. However *serializer* can be the function + that takes the :term:`bytes-like object` and returns the object. *deserializer* + can be the function that takes the object and returns :class:`bytes`. For example, + :keyword:`lambda`, which the :mod:`pickle` does not support, can be used in + :mod:`shelve` using the serializer and deserializer functions, which do support + the :keyword:`lambda`. + .. versionchanged:: 3.10 :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. @@ -48,6 +56,9 @@ lots of shared sub-objects. The keys are ordinary strings. .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. + .. versionchanged:: 3.13 + Accepts *serializer* and *deserializer*. + .. note:: Do not rely on the shelf being closed automatically; always call @@ -150,7 +161,7 @@ Restrictions protocol. -.. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8') +.. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8', serializer=None, deserializer=None) A subclass of :class:`Shelf` which exposes :meth:`!first`, :meth:`!next`, :meth:`!previous`, :meth:`!last` and :meth:`!set_location` methods. @@ -164,7 +175,7 @@ Restrictions interpretation as for the :class:`Shelf` class. -.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False) +.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False, serializer=None, deserializer=None) A subclass of :class:`Shelf` which accepts a *filename* instead of a dict-like object. The underlying file will be opened using :func:`dbm.open`. By From f2eed32ec7d3395ba0aae175e9def8f69d2121f4 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 21 Apr 2024 00:11:47 +0300 Subject: [PATCH 04/59] Update Doc/library/shelve.rst Co-authored-by: Pieter Eendebak --- Doc/library/shelve.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index c8086478ac927b..b25a5ae08e0ede 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -42,7 +42,7 @@ lots of shared sub-objects. The keys are ordinary strings. mutated). By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` - for serializing and deserializing. However *serializer* can be the function + for serializing and deserializing. However *deserializer* can be the function that takes the :term:`bytes-like object` and returns the object. *deserializer* can be the function that takes the object and returns :class:`bytes`. For example, :keyword:`lambda`, which the :mod:`pickle` does not support, can be used in From b3e57236dbc44043e435b18125e3027360c7a7f7 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 21 Apr 2024 00:11:54 +0300 Subject: [PATCH 05/59] Update Doc/library/shelve.rst Co-authored-by: Pieter Eendebak --- Doc/library/shelve.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index b25a5ae08e0ede..c99394adee1cc2 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -43,7 +43,7 @@ lots of shared sub-objects. The keys are ordinary strings. By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` for serializing and deserializing. However *deserializer* can be the function - that takes the :term:`bytes-like object` and returns the object. *deserializer* + that takes the :term:`bytes-like object` and returns the object. *serializer* can be the function that takes the object and returns :class:`bytes`. For example, :keyword:`lambda`, which the :mod:`pickle` does not support, can be used in :mod:`shelve` using the serializer and deserializer functions, which do support From 53d5557de5eaa7d8cf7c34d719424528701dc903 Mon Sep 17 00:00:00 2001 From: furkanonder Date: Mon, 22 Apr 2024 00:03:22 +0300 Subject: [PATCH 06/59] Update documentation for serializer and deserializer functions --- Doc/library/shelve.rst | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index c99394adee1cc2..f36e46d8c61afe 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -43,11 +43,11 @@ lots of shared sub-objects. The keys are ordinary strings. By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` for serializing and deserializing. However *deserializer* can be the function - that takes the :term:`bytes-like object` and returns the object. *serializer* - can be the function that takes the object and returns :class:`bytes`. For example, - :keyword:`lambda`, which the :mod:`pickle` does not support, can be used in - :mod:`shelve` using the serializer and deserializer functions, which do support - the :keyword:`lambda`. + that takes the :term:`bytes-like object` and the *protocol* parameter and returns + the object. *serializer* can be the function that takes the object and returns + :class:`bytes`. For example, :keyword:`lambda`, which the :mod:`pickle` does not + support, can be used in :mod:`shelve` using the serializer and deserializer + functions, which do support the :keyword:`lambda`. .. versionchanged:: 3.10 :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle @@ -57,7 +57,7 @@ lots of shared sub-objects. The keys are ordinary strings. Accepts :term:`path-like object` for filename. .. versionchanged:: 3.13 - Accepts *serializer* and *deserializer*. + Accepts *serializer* and *deserializer* as parameters. .. note:: @@ -128,7 +128,7 @@ Restrictions which can cause hard crashes when trying to read from the database. -.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8') +.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8', serializer=None, deserializer=None) A subclass of :class:`collections.abc.MutableMapping` which stores pickled values in the *dict* object. @@ -146,6 +146,11 @@ Restrictions The *keyencoding* parameter is the encoding used to encode keys before they are used with the underlying dict. + The *deserializer* parameter can be the function that takes the + :term:`bytes-like object` and the *protocol* parameter and returns the + object. *serializer* parameter can be the function that takes the object + and returns :class:`bytes`. + A :class:`Shelf` object can also be used as a context manager, in which case it will be automatically closed when the :keyword:`with` block ends. @@ -160,6 +165,8 @@ Restrictions :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. + .. versionchanged:: 3.13 + Accepts *serializer* and *deserializer* as parameters. .. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8', serializer=None, deserializer=None) @@ -171,8 +178,8 @@ Restrictions modules. The *dict* object passed to the constructor must support those methods. This is generally accomplished by calling one of :func:`!bsddb.hashopen`, :func:`!bsddb.btopen` or :func:`!bsddb.rnopen`. The - optional *protocol*, *writeback*, and *keyencoding* parameters have the same - interpretation as for the :class:`Shelf` class. + optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* + parameters have the same interpretation as for the :class:`Shelf` class. .. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False, serializer=None, deserializer=None) @@ -181,8 +188,8 @@ Restrictions object. The underlying file will be opened using :func:`dbm.open`. By default, the file will be created and opened for both read and write. The optional *flag* parameter has the same interpretation as for the :func:`.open` - function. The optional *protocol* and *writeback* parameters have the same - interpretation as for the :class:`Shelf` class. + function. The optional *protocol*, *writeback*, *serializer* and *deserializer* + parameters have the same interpretation as for the :class:`Shelf` class. .. _shelve-example: From 1e295bad4fa551cae44e1c2f09b13527066ff746 Mon Sep 17 00:00:00 2001 From: furkanonder Date: Mon, 22 Apr 2024 00:35:57 +0300 Subject: [PATCH 07/59] Fix lines according to PEP-8 --- Lib/shelve.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 3ee63404a687b1..5532d3103cc7e7 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -100,7 +100,8 @@ def __init__(self, dict, protocol=None, writeback=False, self.serializer = dumps self.deserializer = loads elif (serializer is None) ^ (deserializer is None): - raise ShelveError("Serializer and deserializer must be defined together.") + raise ShelveError("Serializer and deserializer must be" + "defined together.") else: self.serializer = serializer self.deserializer = deserializer @@ -133,7 +134,8 @@ def __getitem__(self, key): def __setitem__(self, key, value): if self.writeback: self.cache[key] = value - self.dict[key.encode(self.keyencoding)] = self.serializer(value, self._protocol) + serialized_value = self.serializer(value, self._protocol) + self.dict[key.encode(self.keyencoding)] = serialized_value def __delitem__(self, key): del self.dict[key.encode(self.keyencoding)] @@ -202,7 +204,8 @@ def __init__(self, dict, protocol=None, writeback=False, self.serializer = dumps self.deserializer = loads elif (serializer is None) ^ (deserializer is None): - raise ShelveError("Serializer and deserializer must be defined together.") + raise ShelveError("Serializer and deserializer must be" + "defined together.") else: self.serializer = serializer self.deserializer = deserializer @@ -241,9 +244,11 @@ class DbfilenameShelf(Shelf): See the module's __doc__ string for an overview of the interface. """ - def __init__(self, filename, flag='c', protocol=None, writeback=False, serializer=None, deserializer=None): + def __init__(self, filename, flag='c', protocol=None, writeback=False, + serializer=None, deserializer=None): import dbm - Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback, serializer=serializer, deserializer=deserializer) + Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback, + serializer=serializer, deserializer=deserializer) def clear(self): """Remove all items from the shelf.""" @@ -252,7 +257,8 @@ def clear(self): self.cache.clear() self.dict.clear() -def open(filename, flag='c', protocol=None, writeback=False, *, serializer=None, deserializer=None): +def open(filename, flag='c', protocol=None, writeback=False, *, + serializer=None, deserializer=None): """Open a persistent dictionary for reading and writing. The filename parameter is the base filename for the underlying @@ -265,4 +271,5 @@ def open(filename, flag='c', protocol=None, writeback=False, *, serializer=None, See the module's __doc__ string for an overview of the interface. """ - return DbfilenameShelf(filename, flag, protocol, writeback, serializer, deserializer) + return DbfilenameShelf(filename, flag, protocol, writeback, + serializer, deserializer) From 3cbabe94ae5d920b30c1f8b0a17b6fdb9b863378 Mon Sep 17 00:00:00 2001 From: furkanonder Date: Mon, 22 Apr 2024 00:43:07 +0300 Subject: [PATCH 08/59] Fix doc according to line 80 --- Doc/library/shelve.rst | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index f36e46d8c61afe..f138c632f5a3de 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -17,7 +17,8 @@ This includes most class instances, recursive data types, and objects containing lots of shared sub-objects. The keys are ordinary strings. -.. function:: open(filename, flag='c', protocol=None, writeback=False, *, serializer=None, deserializer=None) +.. function:: open(filename, flag='c', protocol=None, writeback=False, *, \ + serializer=None, deserializer=None) Open a persistent dictionary. The filename specified is the base filename for the underlying database. As a side-effect, an extension may be added to the @@ -43,11 +44,12 @@ lots of shared sub-objects. The keys are ordinary strings. By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` for serializing and deserializing. However *deserializer* can be the function - that takes the :term:`bytes-like object` and the *protocol* parameter and returns - the object. *serializer* can be the function that takes the object and returns - :class:`bytes`. For example, :keyword:`lambda`, which the :mod:`pickle` does not - support, can be used in :mod:`shelve` using the serializer and deserializer - functions, which do support the :keyword:`lambda`. + that takes the :term:`bytes-like object` and the *protocol* parameter and + returns the object. *serializer* can be the function that takes the object + and returns :class:`bytes`. For example, :keyword:`lambda`, which + the :mod:`pickle` does notsupport, can be used in :mod:`shelve` using + theserializer and deserializer functions, which do support the + :keyword:`lambda`. .. versionchanged:: 3.10 :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle @@ -128,7 +130,8 @@ Restrictions which can cause hard crashes when trying to read from the database. -.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8', serializer=None, deserializer=None) +.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8', \ + serializer=None, deserializer=None) A subclass of :class:`collections.abc.MutableMapping` which stores pickled values in the *dict* object. @@ -168,7 +171,8 @@ Restrictions .. versionchanged:: 3.13 Accepts *serializer* and *deserializer* as parameters. -.. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8', serializer=None, deserializer=None) +.. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ + keyencoding='utf-8', serializer=None, deserializer=None) A subclass of :class:`Shelf` which exposes :meth:`!first`, :meth:`!next`, :meth:`!previous`, :meth:`!last` and :meth:`!set_location` methods. @@ -182,14 +186,16 @@ Restrictions parameters have the same interpretation as for the :class:`Shelf` class. -.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False, serializer=None, deserializer=None) +.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False, \ + serializer=None, deserializer=None) A subclass of :class:`Shelf` which accepts a *filename* instead of a dict-like object. The underlying file will be opened using :func:`dbm.open`. By default, the file will be created and opened for both read and write. The - optional *flag* parameter has the same interpretation as for the :func:`.open` - function. The optional *protocol*, *writeback*, *serializer* and *deserializer* - parameters have the same interpretation as for the :class:`Shelf` class. + optional *flag* parameter has the same interpretation as for the + :func:`.open` function. The optional *protocol*, *writeback*, *serializer* + and *deserializer*parameters have the same interpretation as for the + :class:`Shelf` class. .. _shelve-example: From c6b43e23a0b211c4f8cb52fe88040892c1e6f586 Mon Sep 17 00:00:00 2001 From: furkanonder Date: Tue, 23 Apr 2024 12:14:04 +0300 Subject: [PATCH 09/59] Fix inline emphasis issue in docs --- Doc/library/shelve.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index f138c632f5a3de..46746dd0d31ab0 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -194,7 +194,7 @@ Restrictions default, the file will be created and opened for both read and write. The optional *flag* parameter has the same interpretation as for the :func:`.open` function. The optional *protocol*, *writeback*, *serializer* - and *deserializer*parameters have the same interpretation as for the + and *deserializer* parameters have the same interpretation as for the :class:`Shelf` class. From 4f79cf607efea150fab4aa3d29827b86c9d0e888 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 14 Jul 2024 01:31:46 +0300 Subject: [PATCH 10/59] Update the definition of the open function. Co-authored-by: Petr Viktorin --- Doc/library/shelve.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 46746dd0d31ab0..7553cde9fd933a 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -43,13 +43,12 @@ lots of shared sub-objects. The keys are ordinary strings. mutated). By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` - for serializing and deserializing. However *deserializer* can be the function - that takes the :term:`bytes-like object` and the *protocol* parameter and - returns the object. *serializer* can be the function that takes the object - and returns :class:`bytes`. For example, :keyword:`lambda`, which - the :mod:`pickle` does notsupport, can be used in :mod:`shelve` using - theserializer and deserializer functions, which do support the - :keyword:`lambda`. + for serializing and deserializing. This can be changed by supplying + *serializer* and *deserializer*, respectively. The *serializer* argument + should be a function that takes an object and returns its representation + as a :term:`bytes-like object`; *deserializer* should be a function that + takes :class:`bytes` and returns the corresponding object. + If one of these is given, the other must be given as well. .. versionchanged:: 3.10 :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle From 798fdb227a6b64575456d12d626c1b8f3cba034d Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 14 Jul 2024 03:37:36 +0300 Subject: [PATCH 11/59] Pass the serializer and serializer arguments of Shelf.__init__ of BsdDbShelf --- Lib/shelve.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 5532d3103cc7e7..0564df6d3108e0 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -200,16 +200,8 @@ class BsdDbShelf(Shelf): def __init__(self, dict, protocol=None, writeback=False, keyencoding="utf-8", *, serializer=None, deserializer=None): - if serializer is None and deserializer is None: - self.serializer = dumps - self.deserializer = loads - elif (serializer is None) ^ (deserializer is None): - raise ShelveError("Serializer and deserializer must be" - "defined together.") - else: - self.serializer = serializer - self.deserializer = deserializer - Shelf.__init__(self, dict, protocol, writeback, keyencoding) + Shelf.__init__(self, dict, protocol, writeback, keyencoding, + serializer=serializer, deserializer=deserializer) def set_location(self, key): (key, value) = self.dict.set_location(key) From bb1150d5e4ea12fe6ddbda67102b1a55a20d5e03 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 14 Jul 2024 03:38:29 +0300 Subject: [PATCH 12/59] Add unittests for BsdDbShelf --- Lib/test/test_shelve.py | 77 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 3fdd9f36393416..87fc85f54ed5e2 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -6,7 +6,7 @@ from io import BytesIO from pydoc import locate -from test.support import os_helper +from test.support import os_helper, import_helper from collections.abc import MutableMapping from test.test_dbm import dbm_iterator @@ -178,7 +178,9 @@ def deserializer(data): os.mkdir(self.dirname) self.addCleanup(os_helper.rmtree, self.dirname) - with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: + with shelve.open(self.fn, + serializer=serializer, + deserializer=deserializer) as s: num = 1 s['number'] = num self.assertEqual(s['number'], type(num)) @@ -190,7 +192,9 @@ def serializer(obj, protocol=None): def deserializer(data): pass - with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: + with shelve.open(self.fn, + serializer=serializer, + deserializer=deserializer) as s: s['number'] = 100 self.assertEqual(s['number'], 100) @@ -201,23 +205,84 @@ def serializer(obj, protocol=None): def deserializer(data): return BytesIO(data).read().decode("utf-8") - with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: + with shelve.open(self.fn, + serializer=serializer, + deserializer=deserializer) as s: s['number'] = 100 self.assertEqual(s['number'], 100) + def test_custom_serializer_and_deserializer_bsd_db_shelf(self): + berkeleydb = import_helper.import_module('berkeleydb') + + def serializer(obj, protocol=None): + return bytes(f"{type(obj).__name__}", 'utf-8') + + def deserializer(data): + value = BytesIO(data).read() + return locate(value.decode("utf-8")) + + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + with shelve.BsdDbShelf(berkeleydb.hashopen(self.fn), + serializer=serializer, + deserializer=deserializer) as s: + num = 1 + s['number'] = num + self.assertEqual(s['number'], type(num)) + + with self.assertRaises(AssertionError): + def serializer(obj, protocol=None): + return bytes(f"{type(obj).__name__}", 'utf-8') + + def deserializer(data): + pass + + with shelve.BsdDbShelf(berkeleydb.hashopen(self.fn), + serializer=serializer, + deserializer=deserializer) as s: + s['number'] = 100 + self.assertEqual(s['number'], 100) + + def serializer(obj, protocol=None): + pass + + def deserializer(data): + return BytesIO(data).read().decode("utf-8") + + with shelve.BsdDbShelf(berkeleydb.hashopen(self.fn), + serializer=serializer, + deserializer=deserializer) as s: + s['number'] = 100 + self.assertNotEqual(s['number'], 100) + self.assertEqual(s['number'], "") + def test_missing_custom_deserializer(self): def serializer(obj, protocol=None): pass with self.assertRaises(shelve.ShelveError): - shelve.Shelf({}, protocol=2, writeback=False, serializer=serializer) + shelve.Shelf({}, + protocol=2, writeback=False, serializer=serializer) + + with self.assertRaises(shelve.ShelveError): + shelve.BsdDbShelf({}, + protocol=2, + writeback=False, serializer=serializer) def test_missing_custom_serializer(self): def deserializer(data): pass with self.assertRaises(shelve.ShelveError): - shelve.Shelf({}, protocol=2, writeback=False, deserializer=deserializer) + shelve.Shelf({}, + protocol=2, + writeback=False, deserializer=deserializer) + + with self.assertRaises(shelve.ShelveError): + shelve.BsdDbShelf({}, + protocol=2, + writeback=False, deserializer=deserializer) class TestShelveBase: From 1159bb6072e7405bed98dfd28c94ba6dba7c1fd1 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 14 Jul 2024 04:44:05 +0300 Subject: [PATCH 13/59] Update BsdDbShelf's set_location, last and first functions --- Lib/shelve.py | 9 +++------ Lib/test/test_shelve.py | 20 +++++++++++++++++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 0564df6d3108e0..c51d1d0dedfed0 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -205,8 +205,7 @@ def __init__(self, dict, protocol=None, writeback=False, def set_location(self, key): (key, value) = self.dict.set_location(key) - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def next(self): (key, value) = next(self.dict) @@ -220,13 +219,11 @@ def previous(self): def first(self): (key, value) = self.dict.first() - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def last(self): (key, value) = self.dict.last() - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) class DbfilenameShelf(Shelf): diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 87fc85f54ed5e2..e9877a91a87e57 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -224,13 +224,27 @@ def deserializer(data): os.mkdir(self.dirname) self.addCleanup(os_helper.rmtree, self.dirname) - with shelve.BsdDbShelf(berkeleydb.hashopen(self.fn), + with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), serializer=serializer, deserializer=deserializer) as s: num = 1 s['number'] = num + num2 = 2 + s['number2'] = num2 self.assertEqual(s['number'], type(num)) + key, value = s.set_location(b'number') + self.assertEqual("number", key) + self.assertEqual(value, type(num)) + + key, value = s.first() + self.assertEqual("number", key) + self.assertEqual(s['number'], value) + + key, value = s.last() + self.assertEqual("number2", key) + self.assertEqual(s['number2'], value) + with self.assertRaises(AssertionError): def serializer(obj, protocol=None): return bytes(f"{type(obj).__name__}", 'utf-8') @@ -238,7 +252,7 @@ def serializer(obj, protocol=None): def deserializer(data): pass - with shelve.BsdDbShelf(berkeleydb.hashopen(self.fn), + with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), serializer=serializer, deserializer=deserializer) as s: s['number'] = 100 @@ -250,7 +264,7 @@ def serializer(obj, protocol=None): def deserializer(data): return BytesIO(data).read().decode("utf-8") - with shelve.BsdDbShelf(berkeleydb.hashopen(self.fn), + with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), serializer=serializer, deserializer=deserializer) as s: s['number'] = 100 From 4b4f1b65668d7de8c502b26a443f1c17f4392055 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 14 Jul 2024 04:54:23 +0300 Subject: [PATCH 14/59] Update BsdDbShelf's next and previous functions --- Lib/shelve.py | 6 ++---- Lib/test/test_shelve.py | 8 ++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index c51d1d0dedfed0..9e8f68a6216250 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -209,13 +209,11 @@ def set_location(self, key): def next(self): (key, value) = next(self.dict) - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def previous(self): (key, value) = self.dict.previous() - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def first(self): (key, value) = self.dict.first() diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index e9877a91a87e57..43acbbec41bd9b 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -233,10 +233,18 @@ def deserializer(data): s['number2'] = num2 self.assertEqual(s['number'], type(num)) + key, value = s.previous() + self.assertEqual("number2", key) + self.assertEqual(value, type(num)) + key, value = s.set_location(b'number') self.assertEqual("number", key) self.assertEqual(value, type(num)) + key, value = s.next() + self.assertEqual("number2", key) + self.assertEqual(value, type(num)) + key, value = s.first() self.assertEqual("number", key) self.assertEqual(s['number'], value) From 41448d3b47da311983e02f993073d6ab1a2417b7 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Tue, 16 Jul 2024 01:43:29 +0300 Subject: [PATCH 15/59] Refer to shelve.open function for the deserializer and serializer arguments on the shelf. Co-authored-by: Petr Viktorin --- Doc/library/shelve.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 3712cfbb514569..9bc79244141b46 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -148,10 +148,7 @@ Restrictions The *keyencoding* parameter is the encoding used to encode keys before they are used with the underlying dict. - The *deserializer* parameter can be the function that takes the - :term:`bytes-like object` and the *protocol* parameter and returns the - object. *serializer* parameter can be the function that takes the object - and returns :class:`bytes`. + The *deserializer* and *serializer* are as in :func:`~shelve.open`. A :class:`Shelf` object can also be used as a context manager, in which case it will be automatically closed when the :keyword:`with` block ends. From fbbe5ea7d86956dc22aa982e3a9213923a5449f9 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Tue, 16 Jul 2024 02:56:49 +0300 Subject: [PATCH 16/59] Refer to shelve.open function for the deserializer and serializer arguments on the BsdDbShelf and DbfilenameShelf --- Doc/library/shelve.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 9bc79244141b46..00a9a6779a121f 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -57,7 +57,7 @@ lots of shared sub-objects. The keys are ordinary strings. .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 Accepts *serializer* and *deserializer* as parameters. .. note:: @@ -164,7 +164,7 @@ Restrictions :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 Accepts *serializer* and *deserializer* as parameters. .. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ @@ -179,7 +179,7 @@ Restrictions methods. This is generally accomplished by calling one of :func:`!bsddb.hashopen`, :func:`!bsddb.btopen` or :func:`!bsddb.rnopen`. The optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* - parameters have the same interpretation as for the :class:`Shelf` class. + parameters have the same interpretation as for the :func:`~shelve.open`. .. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False, \ @@ -191,7 +191,7 @@ Restrictions optional *flag* parameter has the same interpretation as for the :func:`.open` function. The optional *protocol*, *writeback*, *serializer* and *deserializer* parameters have the same interpretation as for the - :class:`Shelf` class. + :func:`~shelve.open` .. _shelve-example: From 6823ef2c5f7c6425774cda4e84f36a73f905b3b3 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 00:01:08 +0000 Subject: [PATCH 17/59] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst diff --git a/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst b/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst new file mode 100644 index 00000000000000..63f8d34728e03f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst @@ -0,0 +1 @@ +The :mod:`shelve` now accepts custom serializer and deserializer functions. From 2affece27e8dc9ed58ff33f86e5fbd0a34955a98 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Tue, 16 Jul 2024 03:12:37 +0300 Subject: [PATCH 18/59] Update the versionchanged statements --- Doc/library/shelve.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 00a9a6779a121f..f996f691272a99 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -58,7 +58,8 @@ lots of shared sub-objects. The keys are ordinary strings. Accepts :term:`path-like object` for filename. .. versionchanged:: 3.14 - Accepts *serializer* and *deserializer* as parameters. + Accepts custom *serializer* and *deserializer* functions in place of + :func:`pickle.dumps` and :func:`pickle.loads`. .. note:: @@ -165,7 +166,8 @@ Restrictions protocol. .. versionchanged:: 3.14 - Accepts *serializer* and *deserializer* as parameters. + Accepts custom *serializer* and *deserializer* functions in place of + :func:`pickle.dumps` and :func:`pickle.loads`. .. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ keyencoding='utf-8', serializer=None, deserializer=None) From 6bfebee0a70419b0f98e6e8712e0b96327fa3e67 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Tue, 16 Jul 2024 03:28:35 +0300 Subject: [PATCH 19/59] change type of num2 --- Lib/test/test_shelve.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 43acbbec41bd9b..ca64793f17b333 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -229,13 +229,13 @@ def deserializer(data): deserializer=deserializer) as s: num = 1 s['number'] = num - num2 = 2 + num2 = 2.3 s['number2'] = num2 self.assertEqual(s['number'], type(num)) key, value = s.previous() self.assertEqual("number2", key) - self.assertEqual(value, type(num)) + self.assertEqual(value, type(num2)) key, value = s.set_location(b'number') self.assertEqual("number", key) @@ -243,7 +243,7 @@ def deserializer(data): key, value = s.next() self.assertEqual("number2", key) - self.assertEqual(value, type(num)) + self.assertEqual(value, type(num2)) key, value = s.first() self.assertEqual("number", key) From 82d58a76f86aff004ddeda03c0ea887323d878e5 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Thu, 18 Jul 2024 00:45:01 +0300 Subject: [PATCH 20/59] Add test_custom_incomplete_serializer_and_deserializer case --- Lib/test/test_shelve.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index ca64793f17b333..919ef1d9830c60 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -198,7 +198,12 @@ def deserializer(data): s['number'] = 100 self.assertEqual(s['number'], 100) - with self.assertRaises(dbm.sqlite3.error): + def test_custom_incomplete_serializer_and_deserializer(self): + dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + with self.assertRaises(dbm_sqlite3.error): def serializer(obj, protocol=None): pass From 5f97676dee2b6c85d7518a9a746ff9553cb28183 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Wed, 24 Jul 2024 23:53:09 +0300 Subject: [PATCH 21/59] Specify that the Shelf, DbfilenameShelf and BsdDbShelf class's takes only keywords in documentation --- Doc/library/shelve.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index f996f691272a99..dd007762a666e3 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -130,8 +130,8 @@ Restrictions which can cause hard crashes when trying to read from the database. -.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8', \ - serializer=None, deserializer=None) +.. class:: Shelf(dict, protocol=None, writeback=False, \ + keyencoding='utf-8', * serializer=None, deserializer=None) A subclass of :class:`collections.abc.MutableMapping` which stores pickled values in the *dict* object. @@ -170,7 +170,8 @@ Restrictions :func:`pickle.dumps` and :func:`pickle.loads`. .. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ - keyencoding='utf-8', serializer=None, deserializer=None) + keyencoding='utf-8', *, \, + serializer=None, deserializer=None) A subclass of :class:`Shelf` which exposes :meth:`!first`, :meth:`!next`, :meth:`!previous`, :meth:`!last` and :meth:`!set_location` methods. @@ -184,8 +185,9 @@ Restrictions parameters have the same interpretation as for the :func:`~shelve.open`. -.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False, \ - serializer=None, deserializer=None) +.. class:: DbfilenameShelf(filename, flag='c', protocol=None, \ + writeback=False, *, serializer=None, \ + deserializer=None) A subclass of :class:`Shelf` which accepts a *filename* instead of a dict-like object. The underlying file will be opened using :func:`dbm.open`. By From 048daee641e09e4ae7d93beab142c6efa2ab7537 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Wed, 24 Jul 2024 23:55:48 +0300 Subject: [PATCH 22/59] And and update the versionchanged's text --- Doc/library/shelve.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index dd007762a666e3..8c25460602d871 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -58,8 +58,7 @@ lots of shared sub-objects. The keys are ordinary strings. Accepts :term:`path-like object` for filename. .. versionchanged:: 3.14 - Accepts custom *serializer* and *deserializer* functions in place of - :func:`pickle.dumps` and :func:`pickle.loads`. + Added the *serializer* and *deserializer* parameters. .. note:: @@ -166,8 +165,7 @@ Restrictions protocol. .. versionchanged:: 3.14 - Accepts custom *serializer* and *deserializer* functions in place of - :func:`pickle.dumps` and :func:`pickle.loads`. + Added the *serializer* and *deserializer* parameters. .. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ keyencoding='utf-8', *, \, @@ -184,6 +182,8 @@ Restrictions optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* parameters have the same interpretation as for the :func:`~shelve.open`. + .. versionchanged:: 3.14 + Added the *serializer* and *deserializer* parameters. .. class:: DbfilenameShelf(filename, flag='c', protocol=None, \ writeback=False, *, serializer=None, \ @@ -197,6 +197,8 @@ Restrictions and *deserializer* parameters have the same interpretation as for the :func:`~shelve.open` + .. versionchanged:: 3.14 + Added the *serializer* and *deserializer* parameters. .. _shelve-example: From 00837d0f6c69a65b2e338bf4e2c1398550a52346 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Wed, 24 Jul 2024 23:58:45 +0300 Subject: [PATCH 23/59] Update the news entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- .../next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst b/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst index 63f8d34728e03f..735249b4dae224 100644 --- a/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst +++ b/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst @@ -1 +1,2 @@ -The :mod:`shelve` now accepts custom serializer and deserializer functions. +The :mod:`shelve` module now accepts custom serialization +and deserialization functions. From 12929637e18d7202447d2057265a1279909f71ec Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Wed, 24 Jul 2024 23:58:54 +0300 Subject: [PATCH 24/59] Update the versionchanged's text --- Doc/library/shelve.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 8c25460602d871..1dab989ae3b2b7 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -58,7 +58,8 @@ lots of shared sub-objects. The keys are ordinary strings. Accepts :term:`path-like object` for filename. .. versionchanged:: 3.14 - Added the *serializer* and *deserializer* parameters. + Accepts custom *serializer* and *deserializer* functions in place of + :func:`pickle.dumps` and :func:`pickle.loads`. .. note:: @@ -195,7 +196,7 @@ Restrictions optional *flag* parameter has the same interpretation as for the :func:`.open` function. The optional *protocol*, *writeback*, *serializer* and *deserializer* parameters have the same interpretation as for the - :func:`~shelve.open` + :func:`~shelve.open`. .. versionchanged:: 3.14 Added the *serializer* and *deserializer* parameters. From 97a6d7cc5b4bf7e48b119d60fc0444abec4305ac Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Thu, 25 Jul 2024 01:19:44 +0300 Subject: [PATCH 25/59] Add new testcases to other bytes objects --- Lib/test/test_shelve.py | 57 +++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 919ef1d9830c60..f268d5fc52c04a 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -1,3 +1,5 @@ +import array + import unittest import dbm import shelve @@ -168,22 +170,57 @@ def test_default_protocol(self): self.assertEqual(s._protocol, pickle.DEFAULT_PROTOCOL) def test_custom_serializer_and_deserializer(self): - def serializer(obj, protocol=None): - return bytes(f"{type(obj).__name__}", 'utf-8') + def serializer(obj, protocol): + if isinstance(obj, (bytes, bytearray, memoryview, int)): + return f"{type(obj).__name__}" + elif isinstance(obj, array.array): + return obj.tobytes() + else: + raise TypeError( + f"Unsupported type for serialization: {type(obj)}" + ) def deserializer(data): - value = BytesIO(data).read() - return locate(value.decode("utf-8")) + if isinstance(data, (bytes, bytearray, memoryview, int)): + value = BytesIO(data).read() + return value.decode("utf-8") + elif isinstance(data, array.array): + return array.array("b", data) + else: + raise TypeError( + f"Unsupported type for deserialization: {type(data)}" + ) os.mkdir(self.dirname) self.addCleanup(os_helper.rmtree, self.dirname) - with shelve.open(self.fn, - serializer=serializer, - deserializer=deserializer) as s: - num = 1 - s['number'] = num - self.assertEqual(s['number'], type(num)) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + with shelve.open( + self.fn, + protocol=proto, + serializer=serializer, + deserializer=deserializer, + ) as s: + num = 1 + bytes_data = b"Hello, world!" + bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") + array_data = array.array("i", [1, 2, 3, 4, 5]) + memoryview_data = memoryview(b"abcdefgh") + + s["number"] = num + s["bytes_data"] = bytes_data + s["bytearray_data"] = bytearray_data + s["array_data"] = array_data + s["memoryview_data"] = memoryview_data + + self.assertEqual(s["number"], "int") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) + self.assertEqual(s["memoryview_data"], "memoryview") with self.assertRaises(AssertionError): def serializer(obj, protocol=None): From 3becbc84ff8ebfcd8658934914d9bcffc642e51a Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Thu, 25 Jul 2024 01:27:38 +0300 Subject: [PATCH 26/59] Add new testcases to test custom serializer protocl --- Lib/test/test_shelve.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index f268d5fc52c04a..a0427ec401372e 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -172,7 +172,10 @@ def test_default_protocol(self): def test_custom_serializer_and_deserializer(self): def serializer(obj, protocol): if isinstance(obj, (bytes, bytearray, memoryview, int)): - return f"{type(obj).__name__}" + if protocol == 5: + return obj + else: + return f"{type(obj).__name__}" elif isinstance(obj, array.array): return obj.tobytes() else: @@ -214,13 +217,24 @@ def deserializer(data): s["array_data"] = array_data s["memoryview_data"] = memoryview_data - self.assertEqual(s["number"], "int") - self.assertEqual(s["bytes_data"], "bytes") - self.assertEqual(s["bytearray_data"], "bytearray") - self.assertEqual( - s["array_data"], array_data.tobytes().decode() - ) - self.assertEqual(s["memoryview_data"], "memoryview") + if proto == 5: + self.assertEqual(s["number"], str(num)) + self.assertEqual(s["bytes_data"], "Hello, world!") + self.assertEqual( + s["bytearray_data"], bytearray_data.decode() + ) + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) + self.assertEqual(s["memoryview_data"], "abcdefgh") + else: + self.assertEqual(s["number"], "int") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) + self.assertEqual(s["memoryview_data"], "memoryview") with self.assertRaises(AssertionError): def serializer(obj, protocol=None): From 3a5d6ed31c1537f504dbc810d643789b0338b7fb Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Thu, 25 Jul 2024 02:01:13 +0300 Subject: [PATCH 27/59] Add new testcases to other bytes objects --- Lib/test/test_shelve.py | 162 ++++++++++++++++++++++++++++++++-------- 1 file changed, 130 insertions(+), 32 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index a0427ec401372e..83343c07540784 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -6,7 +6,6 @@ import pickle import os from io import BytesIO -from pydoc import locate from test.support import os_helper, import_helper from collections.abc import MutableMapping @@ -268,46 +267,145 @@ def deserializer(data): self.assertEqual(s['number'], 100) def test_custom_serializer_and_deserializer_bsd_db_shelf(self): - berkeleydb = import_helper.import_module('berkeleydb') + berkeleydb = import_helper.import_module("berkeleydb") def serializer(obj, protocol=None): - return bytes(f"{type(obj).__name__}", 'utf-8') + if protocol == 5: + return bytes(f"{len(type(obj).__name__)}", encoding="utf-8") + else: + return bytes(f"{type(obj).__name__}", "utf-8") def deserializer(data): value = BytesIO(data).read() - return locate(value.decode("utf-8")) + return value.decode("utf-8") os.mkdir(self.dirname) self.addCleanup(os_helper.rmtree, self.dirname) - with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), - serializer=serializer, - deserializer=deserializer) as s: - num = 1 - s['number'] = num - num2 = 2.3 - s['number2'] = num2 - self.assertEqual(s['number'], type(num)) - - key, value = s.previous() - self.assertEqual("number2", key) - self.assertEqual(value, type(num2)) - - key, value = s.set_location(b'number') - self.assertEqual("number", key) - self.assertEqual(value, type(num)) - - key, value = s.next() - self.assertEqual("number2", key) - self.assertEqual(value, type(num2)) - - key, value = s.first() - self.assertEqual("number", key) - self.assertEqual(s['number'], value) - - key, value = s.last() - self.assertEqual("number2", key) - self.assertEqual(s['number2'], value) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=5): + with shelve.BsdDbShelf( + berkeleydb.btopen(self.fn), + protocol=proto, + serializer=serializer, + deserializer=deserializer, + ) as s: + num = 1 + bytes_data = b"Hello, world!" + bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") + array_data = array.array("i", [1, 2, 3, 4, 5]) + memoryview_data = memoryview(b"abcdefgh") + + s["number"] = num + s["bytes_data"] = bytes_data + s["bytearray_data"] = bytearray_data + s["array_data"] = array_data + s["memoryview_data"] = memoryview_data + + if proto == 5: + self.assertEqual( + s["number"], f"{len(type(num).__name__)}" + ) + self.assertEqual( + s["bytes_data"], + f"{len(type(bytes_data).__name__)}", + ) + self.assertEqual( + s["bytearray_data"], + f"{len(type(bytearray_data).__name__)}", + ) + self.assertEqual( + s["array_data"], + f"{len(type(array_data).__name__)}", + ) + self.assertEqual( + s["memoryview_data"], + f"{len(type(memoryview_data).__name__)}", + ) + + key, value = s.set_location(b"number") + self.assertEqual("number", key) + self.assertEqual(value, f"{len(type(num).__name__)}") + + key, value = s.previous() + self.assertEqual("memoryview_data", key) + self.assertEqual( + value, f"{len(type(memoryview_data).__name__)}" + ) + + key, value = s.previous() + self.assertEqual("bytes_data", key) + self.assertEqual( + value, f"{len(type(bytes_data).__name__)}" + ) + + key, value = s.previous() + self.assertEqual("bytearray_data", key) + self.assertEqual( + value, f"{len(type(bytearray_data).__name__)}" + ) + + key, value = s.previous() + self.assertEqual("array_data", key) + self.assertEqual( + value, f"{len(type(array_data).__name__)}" + ) + + key, value = s.next() + self.assertEqual("bytearray_data", key) + self.assertEqual( + value, f"{len(type(bytearray_data).__name__)}" + ) + + key, value = s.first() + self.assertEqual("array_data", key) + self.assertEqual( + value, f"{len(type(array_data).__name__)}" + ) + + key, value = s.last() + self.assertEqual("number", key) + self.assertEqual( + s["number"], f"{len(type(num).__name__)}" + ) + else: + key, value = s.set_location(b"number") + self.assertEqual("number", key) + self.assertEqual(value, "int") + + key, value = s.previous() + self.assertEqual("memoryview_data", key) + self.assertEqual(value, "memoryview") + + key, value = s.previous() + self.assertEqual("bytes_data", key) + self.assertEqual(value, "bytes") + + key, value = s.previous() + self.assertEqual("bytearray_data", key) + self.assertEqual(value, "bytearray") + + key, value = s.previous() + self.assertEqual("array_data", key) + self.assertEqual(value, "array") + + key, value = s.next() + self.assertEqual("bytearray_data", key) + self.assertEqual(value, "bytearray") + + key, value = s.first() + self.assertEqual("array_data", key) + self.assertEqual(value, "array") + + key, value = s.last() + self.assertEqual("number", key) + self.assertEqual(s["number"], value) + + self.assertEqual(s["number"], "int") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual(s["array_data"], "array") + self.assertEqual(s["memoryview_data"], "memoryview") with self.assertRaises(AssertionError): def serializer(obj, protocol=None): From fb74832a5be03c30c1b890db39773469bf41dca4 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Thu, 25 Jul 2024 02:10:20 +0300 Subject: [PATCH 28/59] Delete comma from document --- Doc/library/shelve.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 1dab989ae3b2b7..c4929f1eac82a0 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -169,7 +169,7 @@ Restrictions Added the *serializer* and *deserializer* parameters. .. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ - keyencoding='utf-8', *, \, + keyencoding='utf-8', *, \ serializer=None, deserializer=None) A subclass of :class:`Shelf` which exposes :meth:`!first`, :meth:`!next`, From d670c95fc50fc83b64a55b8cb25aa7692df22d45 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Thu, 25 Jul 2024 23:17:37 +0300 Subject: [PATCH 29/59] Update the description of open function --- Doc/library/shelve.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index c4929f1eac82a0..628226c735403c 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -45,10 +45,12 @@ lots of shared sub-objects. The keys are ordinary strings. By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` for serializing and deserializing. This can be changed by supplying *serializer* and *deserializer*, respectively. The *serializer* argument - should be a function that takes an object and returns its representation - as a :term:`bytes-like object`; *deserializer* should be a function that - takes :class:`bytes` and returns the corresponding object. - If one of these is given, the other must be given as well. + should be a function that takes an object and the *protocol* argument passed + to the open function and returns its representation as a + :term:`bytes-like object`; *protocol* argument that may be ignored by the + function. *deserializer* should be a function that takes :class:`bytes` and + returns the corresponding object. If one of these is given, the other must + be given as well. Otherwise :mod:`shelve` will raise a :exc:`ShelveError`. .. versionchanged:: 3.10 :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle From f2e22eb7dafb9703484d8838e0a8753a2912598b Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 15:59:26 +0300 Subject: [PATCH 30/59] sort the imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 83343c07540784..8a6c65dd8b9098 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -7,7 +7,7 @@ import os from io import BytesIO -from test.support import os_helper, import_helper +from test.support import import_helper, os_helper from collections.abc import MutableMapping from test.test_dbm import dbm_iterator From e00a52ff7205b78f39302641129e28d5452501a5 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 16:01:13 +0300 Subject: [PATCH 31/59] add white space MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 8a6c65dd8b9098..3aede496d8d680 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -1,5 +1,5 @@ -import array +import array import unittest import dbm import shelve From 3af3f9774650723b4c995f7de62d99d77ab04cc7 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 16:01:53 +0300 Subject: [PATCH 32/59] Don't use f-string in type(obj).__name__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 3aede496d8d680..260bd828229b91 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -174,7 +174,7 @@ def serializer(obj, protocol): if protocol == 5: return obj else: - return f"{type(obj).__name__}" + return type(obj).__name__ elif isinstance(obj, array.array): return obj.tobytes() else: From 9d232e5f5324d1126d0412a626bd869a18f68280 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 16:02:04 +0300 Subject: [PATCH 33/59] Don't use f-string in type(obj).__name__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 260bd828229b91..dd254d9a698b6c 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -237,7 +237,7 @@ def deserializer(data): with self.assertRaises(AssertionError): def serializer(obj, protocol=None): - return bytes(f"{type(obj).__name__}", 'utf-8') + return bytes(type(obj).__name__, 'utf-8') def deserializer(data): pass From 26fc95958e09ff58221dd26159c3e68bf78f08eb Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 16:02:27 +0300 Subject: [PATCH 34/59] Don't use f-string in type(obj).__name__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index dd254d9a698b6c..8551d40ae03230 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -409,7 +409,7 @@ def deserializer(data): with self.assertRaises(AssertionError): def serializer(obj, protocol=None): - return bytes(f"{type(obj).__name__}", 'utf-8') + return bytes(type(obj).__name__, 'utf-8') def deserializer(data): pass From ab005aa269ebfd730a51eeeeb6a889473d29e2d6 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 16:03:36 +0300 Subject: [PATCH 35/59] Set shelve class argument only serializer and deserializer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 9e8f68a6216250..9343cae0e47c52 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -231,7 +231,7 @@ class DbfilenameShelf(Shelf): See the module's __doc__ string for an overview of the interface. """ - def __init__(self, filename, flag='c', protocol=None, writeback=False, + def __init__(self, filename, flag='c', protocol=None, writeback=False, *, serializer=None, deserializer=None): import dbm Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback, From 60523091fa7bdbe75ffeb927c8eacc4b39859add Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 16:04:39 +0300 Subject: [PATCH 36/59] Update shelveError message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/shelve.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 9343cae0e47c52..dec1c586b13410 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -100,8 +100,8 @@ def __init__(self, dict, protocol=None, writeback=False, self.serializer = dumps self.deserializer = loads elif (serializer is None) ^ (deserializer is None): - raise ShelveError("Serializer and deserializer must be" - "defined together.") + raise ShelveError("serializer and deserializer must be " + "defined together") else: self.serializer = serializer self.deserializer = deserializer From 87b66d5419d2f4985cd594da10c3272806b83152 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 16:05:20 +0300 Subject: [PATCH 37/59] pass serializer and deserializer as keyword argument to DbfilenameShelf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index dec1c586b13410..2dcea5efe7877f 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -259,4 +259,4 @@ def open(filename, flag='c', protocol=None, writeback=False, *, """ return DbfilenameShelf(filename, flag, protocol, writeback, - serializer, deserializer) + serializer=serializer, deserializer=deserializer) From 0c2f2551321ce805f51ddb6e9a5d12f66783db8f Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 19:13:52 +0300 Subject: [PATCH 38/59] Remove unused import --- Lib/shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index 2dcea5efe7877f..aeb4dcb76e02f7 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -56,7 +56,7 @@ the persistent dictionary on disk, if feasible). """ -from pickle import DEFAULT_PROTOCOL, Unpickler, dumps, loads +from pickle import DEFAULT_PROTOCOL, dumps, loads from io import BytesIO import collections.abc From 3db0c8e1fd4e735c0a7e414922b681035837f45c Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sat, 27 Jul 2024 19:14:47 +0300 Subject: [PATCH 39/59] Update shelve testcases --- Lib/test/test_shelve.py | 78 ++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 8551d40ae03230..eea1d5f46a8c49 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -170,7 +170,7 @@ def test_default_protocol(self): def test_custom_serializer_and_deserializer(self): def serializer(obj, protocol): - if isinstance(obj, (bytes, bytearray, memoryview, int)): + if isinstance(obj, (bytes, bytearray, memoryview, str)): if protocol == 5: return obj else: @@ -183,7 +183,7 @@ def serializer(obj, protocol): ) def deserializer(data): - if isinstance(data, (bytes, bytearray, memoryview, int)): + if isinstance(data, (bytes, bytearray, memoryview, str)): value = BytesIO(data).read() return value.decode("utf-8") elif isinstance(data, array.array): @@ -204,20 +204,20 @@ def deserializer(data): serializer=serializer, deserializer=deserializer, ) as s: - num = 1 + bar = "bar" bytes_data = b"Hello, world!" bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") array_data = array.array("i", [1, 2, 3, 4, 5]) memoryview_data = memoryview(b"abcdefgh") - s["number"] = num + s["foo"] = bar s["bytes_data"] = bytes_data s["bytearray_data"] = bytearray_data s["array_data"] = array_data s["memoryview_data"] = memoryview_data if proto == 5: - self.assertEqual(s["number"], str(num)) + self.assertEqual(s["foo"], str(bar)) self.assertEqual(s["bytes_data"], "Hello, world!") self.assertEqual( s["bytearray_data"], bytearray_data.decode() @@ -227,7 +227,7 @@ def deserializer(data): ) self.assertEqual(s["memoryview_data"], "abcdefgh") else: - self.assertEqual(s["number"], "int") + self.assertEqual(s["foo"], "str") self.assertEqual(s["bytes_data"], "bytes") self.assertEqual(s["bytearray_data"], "bytearray") self.assertEqual( @@ -245,8 +245,8 @@ def deserializer(data): with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: - s['number'] = 100 - self.assertEqual(s['number'], 100) + s["foo"] = "bar" + self.assertEqual(s["foo"], "bar") def test_custom_incomplete_serializer_and_deserializer(self): dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") @@ -263,8 +263,8 @@ def deserializer(data): with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: - s['number'] = 100 - self.assertEqual(s['number'], 100) + s["foo"] = "bar" + self.assertEqual(s["foo"], "bar") def test_custom_serializer_and_deserializer_bsd_db_shelf(self): berkeleydb = import_helper.import_module("berkeleydb") @@ -290,13 +290,13 @@ def deserializer(data): serializer=serializer, deserializer=deserializer, ) as s: - num = 1 + bar = "bar" bytes_data = b"Hello, world!" bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") array_data = array.array("i", [1, 2, 3, 4, 5]) memoryview_data = memoryview(b"abcdefgh") - s["number"] = num + s["foo"] = "bar" s["bytes_data"] = bytes_data s["bytearray_data"] = bytearray_data s["array_data"] = array_data @@ -304,7 +304,7 @@ def deserializer(data): if proto == 5: self.assertEqual( - s["number"], f"{len(type(num).__name__)}" + s["foo"], f"{len(type(bar).__name__)}" ) self.assertEqual( s["bytes_data"], @@ -323,15 +323,9 @@ def deserializer(data): f"{len(type(memoryview_data).__name__)}", ) - key, value = s.set_location(b"number") - self.assertEqual("number", key) - self.assertEqual(value, f"{len(type(num).__name__)}") - - key, value = s.previous() - self.assertEqual("memoryview_data", key) - self.assertEqual( - value, f"{len(type(memoryview_data).__name__)}" - ) + key, value = s.set_location(b"foo") + self.assertEqual("foo", key) + self.assertEqual(value, f"{len(type(bar).__name__)}") key, value = s.previous() self.assertEqual("bytes_data", key) @@ -357,6 +351,12 @@ def deserializer(data): value, f"{len(type(bytearray_data).__name__)}" ) + key, value = s.next() + self.assertEqual("bytes_data", key) + self.assertEqual( + value, f"{len(type(bytes_data).__name__)}" + ) + key, value = s.first() self.assertEqual("array_data", key) self.assertEqual( @@ -364,18 +364,14 @@ def deserializer(data): ) key, value = s.last() - self.assertEqual("number", key) + self.assertEqual("memoryview_data", key) self.assertEqual( - s["number"], f"{len(type(num).__name__)}" + s["memoryview_data"], f"{len(type(memoryview_data).__name__)}" ) else: - key, value = s.set_location(b"number") - self.assertEqual("number", key) - self.assertEqual(value, "int") - - key, value = s.previous() - self.assertEqual("memoryview_data", key) - self.assertEqual(value, "memoryview") + key, value = s.set_location(b"foo") + self.assertEqual("foo", key) + self.assertEqual(value, "str") key, value = s.previous() self.assertEqual("bytes_data", key) @@ -393,15 +389,19 @@ def deserializer(data): self.assertEqual("bytearray_data", key) self.assertEqual(value, "bytearray") + key, value = s.next() + self.assertEqual("bytes_data", key) + self.assertEqual(value, "bytes") + key, value = s.first() self.assertEqual("array_data", key) self.assertEqual(value, "array") key, value = s.last() - self.assertEqual("number", key) - self.assertEqual(s["number"], value) + self.assertEqual("memoryview_data", key) + self.assertEqual(s["memoryview_data"], value) - self.assertEqual(s["number"], "int") + self.assertEqual(s["foo"], "str") self.assertEqual(s["bytes_data"], "bytes") self.assertEqual(s["bytearray_data"], "bytearray") self.assertEqual(s["array_data"], "array") @@ -417,8 +417,8 @@ def deserializer(data): with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), serializer=serializer, deserializer=deserializer) as s: - s['number'] = 100 - self.assertEqual(s['number'], 100) + s["foo"] = "bar" + self.assertEqual(s["foo"], "bar") def serializer(obj, protocol=None): pass @@ -429,9 +429,9 @@ def deserializer(data): with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), serializer=serializer, deserializer=deserializer) as s: - s['number'] = 100 - self.assertNotEqual(s['number'], 100) - self.assertEqual(s['number'], "") + s["foo"] = "bar" + self.assertNotEqual(s["foo"], "bar") + self.assertEqual(s["foo"], "") def test_missing_custom_deserializer(self): def serializer(obj, protocol=None): From 5c39d94b282b71d2f5cf45942d1500580134f645 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 01:30:42 +0300 Subject: [PATCH 40/59] Remove memoryview testcases --- Lib/test/test_shelve.py | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index eea1d5f46a8c49..af8109962ab82e 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -170,7 +170,7 @@ def test_default_protocol(self): def test_custom_serializer_and_deserializer(self): def serializer(obj, protocol): - if isinstance(obj, (bytes, bytearray, memoryview, str)): + if isinstance(obj, (bytes, bytearray, str)): if protocol == 5: return obj else: @@ -183,7 +183,7 @@ def serializer(obj, protocol): ) def deserializer(data): - if isinstance(data, (bytes, bytearray, memoryview, str)): + if isinstance(data, (bytes, bytearray, str)): value = BytesIO(data).read() return value.decode("utf-8") elif isinstance(data, array.array): @@ -208,13 +208,11 @@ def deserializer(data): bytes_data = b"Hello, world!" bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") array_data = array.array("i", [1, 2, 3, 4, 5]) - memoryview_data = memoryview(b"abcdefgh") s["foo"] = bar s["bytes_data"] = bytes_data s["bytearray_data"] = bytearray_data s["array_data"] = array_data - s["memoryview_data"] = memoryview_data if proto == 5: self.assertEqual(s["foo"], str(bar)) @@ -225,7 +223,6 @@ def deserializer(data): self.assertEqual( s["array_data"], array_data.tobytes().decode() ) - self.assertEqual(s["memoryview_data"], "abcdefgh") else: self.assertEqual(s["foo"], "str") self.assertEqual(s["bytes_data"], "bytes") @@ -233,7 +230,6 @@ def deserializer(data): self.assertEqual( s["array_data"], array_data.tobytes().decode() ) - self.assertEqual(s["memoryview_data"], "memoryview") with self.assertRaises(AssertionError): def serializer(obj, protocol=None): @@ -294,13 +290,11 @@ def deserializer(data): bytes_data = b"Hello, world!" bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") array_data = array.array("i", [1, 2, 3, 4, 5]) - memoryview_data = memoryview(b"abcdefgh") s["foo"] = "bar" s["bytes_data"] = bytes_data s["bytearray_data"] = bytearray_data s["array_data"] = array_data - s["memoryview_data"] = memoryview_data if proto == 5: self.assertEqual( @@ -318,10 +312,6 @@ def deserializer(data): s["array_data"], f"{len(type(array_data).__name__)}", ) - self.assertEqual( - s["memoryview_data"], - f"{len(type(memoryview_data).__name__)}", - ) key, value = s.set_location(b"foo") self.assertEqual("foo", key) @@ -362,12 +352,6 @@ def deserializer(data): self.assertEqual( value, f"{len(type(array_data).__name__)}" ) - - key, value = s.last() - self.assertEqual("memoryview_data", key) - self.assertEqual( - s["memoryview_data"], f"{len(type(memoryview_data).__name__)}" - ) else: key, value = s.set_location(b"foo") self.assertEqual("foo", key) @@ -397,15 +381,10 @@ def deserializer(data): self.assertEqual("array_data", key) self.assertEqual(value, "array") - key, value = s.last() - self.assertEqual("memoryview_data", key) - self.assertEqual(s["memoryview_data"], value) - self.assertEqual(s["foo"], "str") self.assertEqual(s["bytes_data"], "bytes") self.assertEqual(s["bytearray_data"], "bytearray") self.assertEqual(s["array_data"], "array") - self.assertEqual(s["memoryview_data"], "memoryview") with self.assertRaises(AssertionError): def serializer(obj, protocol=None): From 1ca1801712b40f06b0f47127195b98fd02f3e943 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 02:07:50 +0300 Subject: [PATCH 41/59] Add ShelveError to shelve's __all__ --- Lib/shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index aeb4dcb76e02f7..69f775ee99bc41 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -61,7 +61,7 @@ import collections.abc -__all__ = ["Shelf", "BsdDbShelf", "DbfilenameShelf", "open"] +__all__ = ["ShelveError", "Shelf", "BsdDbShelf", "DbfilenameShelf", "open"] class ShelveError(Exception): From 5a42de1145a27a4d7ff1f19cde115571cc41b194 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 02:08:28 +0300 Subject: [PATCH 42/59] Add ShelveError to shelve documentation --- Doc/library/shelve.rst | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 628226c735403c..da6b3147b9dc60 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -133,7 +133,7 @@ Restrictions .. class:: Shelf(dict, protocol=None, writeback=False, \ - keyencoding='utf-8', * serializer=None, deserializer=None) + keyencoding='utf-8', *, serializer=None, deserializer=None) A subclass of :class:`collections.abc.MutableMapping` which stores pickled values in the *dict* object. @@ -151,7 +151,8 @@ Restrictions The *keyencoding* parameter is the encoding used to encode keys before they are used with the underlying dict. - The *deserializer* and *serializer* are as in :func:`~shelve.open`. + The *serializer* and *deserializer* parameters have the same interpretation + as in :func:`~shelve.open`. A :class:`Shelf` object can also be used as a context manager, in which case it will be automatically closed when the :keyword:`with` block ends. @@ -183,7 +184,7 @@ Restrictions methods. This is generally accomplished by calling one of :func:`!bsddb.hashopen`, :func:`!bsddb.btopen` or :func:`!bsddb.rnopen`. The optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* - parameters have the same interpretation as for the :func:`~shelve.open`. + parameters have the same interpretation as in :func:`~shelve.open`. .. versionchanged:: 3.14 Added the *serializer* and *deserializer* parameters. @@ -197,7 +198,7 @@ Restrictions default, the file will be created and opened for both read and write. The optional *flag* parameter has the same interpretation as for the :func:`.open` function. The optional *protocol*, *writeback*, *serializer* - and *deserializer* parameters have the same interpretation as for the + and *deserializer* parameters have the same interpretation as in the :func:`~shelve.open`. .. versionchanged:: 3.14 @@ -242,6 +243,19 @@ object):: d.close() # close it +Exceptions +---------- + +.. exception:: ShelveError + + Exception raised when one of the arguments *deserializer* and *serializer* + is missing in the :func:`~shelve.open`, :class:`Shelf`, :class:`BsdDbShelf` + and :class:`DbfilenameShelf` + + The *deserializer* and *serializer* arguments must be given together. + + .. versionadded:: 3.14 + .. seealso:: Module :mod:`dbm` From b0a5ee385977b3284fa1d36201019bc5b8348fcc Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 11:39:06 +0300 Subject: [PATCH 43/59] Add blank lines after versionadded and versionchanged --- Doc/library/shelve.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index da6b3147b9dc60..7b4c62af3897c2 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -171,6 +171,7 @@ Restrictions .. versionchanged:: 3.14 Added the *serializer* and *deserializer* parameters. + .. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ keyencoding='utf-8', *, \ serializer=None, deserializer=None) @@ -189,6 +190,7 @@ Restrictions .. versionchanged:: 3.14 Added the *serializer* and *deserializer* parameters. + .. class:: DbfilenameShelf(filename, flag='c', protocol=None, \ writeback=False, *, serializer=None, \ deserializer=None) @@ -204,6 +206,7 @@ Restrictions .. versionchanged:: 3.14 Added the *serializer* and *deserializer* parameters. + .. _shelve-example: Example @@ -256,6 +259,7 @@ Exceptions .. versionadded:: 3.14 + .. seealso:: Module :mod:`dbm` From 4202ede4dcf7c3257e99cf8f7ed6dc5d2edf8398 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 11:40:35 +0300 Subject: [PATCH 44/59] Remove white space in test_shelve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_shelve.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index af8109962ab82e..cd63d1c90c9f4f 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -1,4 +1,3 @@ - import array import unittest import dbm From 2827eb483c1066f442caa653c0a0a3e73221fca2 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 12:46:45 +0300 Subject: [PATCH 45/59] Add test_custom_incomplete_serializer_and_deserializer_bsd_db_shelf --- Lib/test/test_shelve.py | 52 ++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index af8109962ab82e..2d3a750b4139c5 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -231,19 +231,6 @@ def deserializer(data): s["array_data"], array_data.tobytes().decode() ) - with self.assertRaises(AssertionError): - def serializer(obj, protocol=None): - return bytes(type(obj).__name__, 'utf-8') - - def deserializer(data): - pass - - with shelve.open(self.fn, - serializer=serializer, - deserializer=deserializer) as s: - s["foo"] = "bar" - self.assertEqual(s["foo"], "bar") - def test_custom_incomplete_serializer_and_deserializer(self): dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") os.mkdir(self.dirname) @@ -256,11 +243,21 @@ def serializer(obj, protocol=None): def deserializer(data): return BytesIO(data).read().decode("utf-8") - with shelve.open(self.fn, - serializer=serializer, + with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: s["foo"] = "bar" - self.assertEqual(s["foo"], "bar") + + def serializer(obj, protocol=None): + return bytes(type(obj).__name__, 'utf-8') + + def deserializer(data): + pass + + with shelve.open(self.fn, serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + self.assertNotEqual(s["foo"], "bar") + self.assertEqual(s["foo"], None) def test_custom_serializer_and_deserializer_bsd_db_shelf(self): berkeleydb = import_helper.import_module("berkeleydb") @@ -386,18 +383,21 @@ def deserializer(data): self.assertEqual(s["bytearray_data"], "bytearray") self.assertEqual(s["array_data"], "array") - with self.assertRaises(AssertionError): - def serializer(obj, protocol=None): - return bytes(type(obj).__name__, 'utf-8') + def test_custom_incomplete_serializer_and_deserializer_bsd_db_shelf(self): + berkeleydb = import_helper.import_module("berkeleydb") - def deserializer(data): - pass + def serializer(obj, protocol=None): + return bytes(type(obj).__name__, 'utf-8') - with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), - serializer=serializer, - deserializer=deserializer) as s: - s["foo"] = "bar" - self.assertEqual(s["foo"], "bar") + def deserializer(data): + pass + + with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), + serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + self.assertNotEqual(s["foo"], "bar") + self.assertEqual(s["foo"], None) def serializer(obj, protocol=None): pass From 54188bd2bbf0ec5ce4d72c582c477d5d70588b80 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 13:01:51 +0300 Subject: [PATCH 46/59] Update the serializer and deserializer functions --- Lib/test/test_shelve.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 4cb3c8448319cc..a9550d06da4955 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -4,7 +4,6 @@ import shelve import pickle import os -from io import BytesIO from test.support import import_helper, os_helper from collections.abc import MutableMapping @@ -183,8 +182,7 @@ def serializer(obj, protocol): def deserializer(data): if isinstance(data, (bytes, bytearray, str)): - value = BytesIO(data).read() - return value.decode("utf-8") + return data.decode("utf-8") elif isinstance(data, array.array): return array.array("b", data) else: @@ -240,14 +238,14 @@ def serializer(obj, protocol=None): pass def deserializer(data): - return BytesIO(data).read().decode("utf-8") + return data.decode("utf-8") with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: s["foo"] = "bar" def serializer(obj, protocol=None): - return bytes(type(obj).__name__, 'utf-8') + return type(obj).__name__.encode("utf-8") def deserializer(data): pass @@ -262,14 +260,13 @@ def test_custom_serializer_and_deserializer_bsd_db_shelf(self): berkeleydb = import_helper.import_module("berkeleydb") def serializer(obj, protocol=None): + data = obj.__class__.__name__ if protocol == 5: - return bytes(f"{len(type(obj).__name__)}", encoding="utf-8") - else: - return bytes(f"{type(obj).__name__}", "utf-8") + data = str(len(data)) + return data.encode("utf-8") def deserializer(data): - value = BytesIO(data).read() - return value.decode("utf-8") + return data.decode("utf-8") os.mkdir(self.dirname) self.addCleanup(os_helper.rmtree, self.dirname) @@ -384,9 +381,11 @@ def deserializer(data): def test_custom_incomplete_serializer_and_deserializer_bsd_db_shelf(self): berkeleydb = import_helper.import_module("berkeleydb") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) def serializer(obj, protocol=None): - return bytes(type(obj).__name__, 'utf-8') + return type(obj).__name__.encode("utf-8") def deserializer(data): pass @@ -402,7 +401,7 @@ def serializer(obj, protocol=None): pass def deserializer(data): - return BytesIO(data).read().decode("utf-8") + return data.decode("utf-8") with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), serializer=serializer, From 786a248848a08e23d840a01bc9af3b545a1094f8 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 13:06:02 +0300 Subject: [PATCH 47/59] Move os.mkdir and addCleanup functions beginning of the testcases --- Lib/test/test_shelve.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index a9550d06da4955..dc9ced404a8903 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -167,6 +167,9 @@ def test_default_protocol(self): self.assertEqual(s._protocol, pickle.DEFAULT_PROTOCOL) def test_custom_serializer_and_deserializer(self): + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + def serializer(obj, protocol): if isinstance(obj, (bytes, bytearray, str)): if protocol == 5: @@ -190,9 +193,6 @@ def deserializer(data): f"Unsupported type for deserialization: {type(data)}" ) - os.mkdir(self.dirname) - self.addCleanup(os_helper.rmtree, self.dirname) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto): with shelve.open( @@ -258,6 +258,8 @@ def deserializer(data): def test_custom_serializer_and_deserializer_bsd_db_shelf(self): berkeleydb = import_helper.import_module("berkeleydb") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) def serializer(obj, protocol=None): data = obj.__class__.__name__ @@ -268,9 +270,6 @@ def serializer(obj, protocol=None): def deserializer(data): return data.decode("utf-8") - os.mkdir(self.dirname) - self.addCleanup(os_helper.rmtree, self.dirname) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=5): with shelve.BsdDbShelf( From 20c245005035f3567d7d45df9b3eb885bedf0d7c Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 13:17:06 +0300 Subject: [PATCH 48/59] Use self.assertIsNone when checking None types --- Lib/test/test_shelve.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index dc9ced404a8903..90793b842ca034 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -254,7 +254,7 @@ def deserializer(data): deserializer=deserializer) as s: s["foo"] = "bar" self.assertNotEqual(s["foo"], "bar") - self.assertEqual(s["foo"], None) + self.assertIsNone(s["foo"]) def test_custom_serializer_and_deserializer_bsd_db_shelf(self): berkeleydb = import_helper.import_module("berkeleydb") @@ -394,7 +394,7 @@ def deserializer(data): deserializer=deserializer) as s: s["foo"] = "bar" self.assertNotEqual(s["foo"], "bar") - self.assertEqual(s["foo"], None) + self.assertIsNone(s["foo"]) def serializer(obj, protocol=None): pass From b3770ae34aa40fa74b5b82a85c007d0011e9acb2 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 29 Jul 2024 15:12:37 +0300 Subject: [PATCH 49/59] change the test order --- Lib/test/test_shelve.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 90793b842ca034..151f4108700ccb 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -283,7 +283,7 @@ def deserializer(data): bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") array_data = array.array("i", [1, 2, 3, 4, 5]) - s["foo"] = "bar" + s["foo"] = bar s["bytes_data"] = bytes_data s["bytearray_data"] = bytearray_data s["array_data"] = array_data @@ -393,8 +393,8 @@ def deserializer(data): serializer=serializer, deserializer=deserializer) as s: s["foo"] = "bar" - self.assertNotEqual(s["foo"], "bar") self.assertIsNone(s["foo"]) + self.assertNotEqual(s["foo"], "bar") def serializer(obj, protocol=None): pass @@ -406,8 +406,9 @@ def deserializer(data): serializer=serializer, deserializer=deserializer) as s: s["foo"] = "bar" - self.assertNotEqual(s["foo"], "bar") self.assertEqual(s["foo"], "") + self.assertNotEqual(s["foo"], "bar") + def test_missing_custom_deserializer(self): def serializer(obj, protocol=None): From 4d9599b672e5b8819ff45ab935ff58ae50093bb5 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 30 May 2025 15:28:24 +0300 Subject: [PATCH 50/59] Update shelve module version references from 3.14 to 3.15 --- Doc/library/shelve.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 7b4c62af3897c2..f7e8496d646d10 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -59,7 +59,7 @@ lots of shared sub-objects. The keys are ordinary strings. .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. - .. versionchanged:: 3.14 + .. versionchanged:: 3.15 Accepts custom *serializer* and *deserializer* functions in place of :func:`pickle.dumps` and :func:`pickle.loads`. @@ -168,7 +168,7 @@ Restrictions :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. - .. versionchanged:: 3.14 + .. versionchanged:: 3.15 Added the *serializer* and *deserializer* parameters. @@ -187,7 +187,7 @@ Restrictions optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* parameters have the same interpretation as in :func:`~shelve.open`. - .. versionchanged:: 3.14 + .. versionchanged:: 3.15 Added the *serializer* and *deserializer* parameters. @@ -203,7 +203,7 @@ Restrictions and *deserializer* parameters have the same interpretation as in the :func:`~shelve.open`. - .. versionchanged:: 3.14 + .. versionchanged:: 3.15 Added the *serializer* and *deserializer* parameters. @@ -257,7 +257,7 @@ Exceptions The *deserializer* and *serializer* arguments must be given together. - .. versionadded:: 3.14 + .. versionadded:: 3.15 .. seealso:: From 8b06918f0c08e8eb3c63ee98daa16f3f7248829a Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 30 May 2025 18:24:05 +0300 Subject: [PATCH 51/59] Change shelve module version references from 3.15 to next modified: Doc/library/shelve.rst --- Doc/library/shelve.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index f7e8496d646d10..9e7c8af0dbdc17 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -59,7 +59,7 @@ lots of shared sub-objects. The keys are ordinary strings. .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. - .. versionchanged:: 3.15 + .. versionchanged:: next Accepts custom *serializer* and *deserializer* functions in place of :func:`pickle.dumps` and :func:`pickle.loads`. @@ -168,7 +168,7 @@ Restrictions :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. - .. versionchanged:: 3.15 + .. versionchanged:: next Added the *serializer* and *deserializer* parameters. @@ -187,7 +187,7 @@ Restrictions optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* parameters have the same interpretation as in :func:`~shelve.open`. - .. versionchanged:: 3.15 + .. versionchanged:: next Added the *serializer* and *deserializer* parameters. @@ -200,10 +200,10 @@ Restrictions default, the file will be created and opened for both read and write. The optional *flag* parameter has the same interpretation as for the :func:`.open` function. The optional *protocol*, *writeback*, *serializer* - and *deserializer* parameters have the same interpretation as in the + and *deserializer* parameters have the same interpretation as in :func:`~shelve.open`. - .. versionchanged:: 3.15 + .. versionchanged:: next Added the *serializer* and *deserializer* parameters. From b0f0bbc44186f520ec9182af3a78fde78a5f9bc6 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 30 May 2025 18:26:24 +0300 Subject: [PATCH 52/59] Change shelve module version references from 3.15 to next --- Doc/library/shelve.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 9e7c8af0dbdc17..01f6c3308f7fff 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -257,7 +257,7 @@ Exceptions The *deserializer* and *serializer* arguments must be given together. - .. versionadded:: 3.15 + .. versionadded:: next .. seealso:: From d1bb227ca24054235030004513d43707a6e88f16 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 30 May 2025 18:36:53 +0300 Subject: [PATCH 53/59] refactor nested context managers for better readability --- Lib/test/test_shelve.py | 65 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 151f4108700ccb..0e5f695c9c9619 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -194,39 +194,38 @@ def deserializer(data): ) for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(proto=proto): - with shelve.open( - self.fn, - protocol=proto, - serializer=serializer, - deserializer=deserializer, - ) as s: - bar = "bar" - bytes_data = b"Hello, world!" - bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") - array_data = array.array("i", [1, 2, 3, 4, 5]) - - s["foo"] = bar - s["bytes_data"] = bytes_data - s["bytearray_data"] = bytearray_data - s["array_data"] = array_data - - if proto == 5: - self.assertEqual(s["foo"], str(bar)) - self.assertEqual(s["bytes_data"], "Hello, world!") - self.assertEqual( - s["bytearray_data"], bytearray_data.decode() - ) - self.assertEqual( - s["array_data"], array_data.tobytes().decode() - ) - else: - self.assertEqual(s["foo"], "str") - self.assertEqual(s["bytes_data"], "bytes") - self.assertEqual(s["bytearray_data"], "bytearray") - self.assertEqual( - s["array_data"], array_data.tobytes().decode() - ) + with self.subTest(proto=proto), shelve.open( + self.fn, + protocol=proto, + serializer=serializer, + deserializer=deserializer + ) as s: + bar = "bar" + bytes_data = b"Hello, world!" + bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") + array_data = array.array("i", [1, 2, 3, 4, 5]) + + s["foo"] = bar + s["bytes_data"] = bytes_data + s["bytearray_data"] = bytearray_data + s["array_data"] = array_data + + if proto == 5: + self.assertEqual(s["foo"], str(bar)) + self.assertEqual(s["bytes_data"], "Hello, world!") + self.assertEqual( + s["bytearray_data"], bytearray_data.decode() + ) + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) + else: + self.assertEqual(s["foo"], "str") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) def test_custom_incomplete_serializer_and_deserializer(self): dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") From 791743b85b46cf8238f528a1e11d28d3e6aeb769 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 30 May 2025 18:40:06 +0300 Subject: [PATCH 54/59] simplify assertRaises calls in test_missing_custom_deserializer & test_missing_custom_serializer --- Lib/test/test_shelve.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 0e5f695c9c9619..8c75b9839a3da9 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -413,28 +413,17 @@ def test_missing_custom_deserializer(self): def serializer(obj, protocol=None): pass - with self.assertRaises(shelve.ShelveError): - shelve.Shelf({}, - protocol=2, writeback=False, serializer=serializer) - - with self.assertRaises(shelve.ShelveError): - shelve.BsdDbShelf({}, - protocol=2, - writeback=False, serializer=serializer) + kwargs = dict(protocol=2, writeback=False, serializer=serializer) + self.assertRaises(shelve.ShelveError, shelve.Shelf, {}, **kwargs) + self.assertRaises(shelve.ShelveError, shelve.BsdDbShelf, {}, **kwargs) def test_missing_custom_serializer(self): def deserializer(data): pass - with self.assertRaises(shelve.ShelveError): - shelve.Shelf({}, - protocol=2, - writeback=False, deserializer=deserializer) - - with self.assertRaises(shelve.ShelveError): - shelve.BsdDbShelf({}, - protocol=2, - writeback=False, deserializer=deserializer) + kwargs = dict(protocol=2, writeback=False, deserializer=deserializer) + self.assertRaises(shelve.ShelveError, shelve.Shelf, {}, **kwargs) + self.assertRaises(shelve.ShelveError, shelve.BsdDbShelf, {}, **kwargs) class TestShelveBase: From 6b4be8b7d1bf957ccc37a3561356bd6f9fa6f3dc Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 30 May 2025 18:45:23 +0300 Subject: [PATCH 55/59] refactor nested context managers for better readability --- Lib/test/test_shelve.py | 211 ++++++++++++++++++++-------------------- 1 file changed, 105 insertions(+), 106 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 8c75b9839a3da9..8e00b9e103f3e5 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -270,112 +270,111 @@ def deserializer(data): return data.decode("utf-8") for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(proto=5): - with shelve.BsdDbShelf( - berkeleydb.btopen(self.fn), - protocol=proto, - serializer=serializer, - deserializer=deserializer, - ) as s: - bar = "bar" - bytes_data = b"Hello, world!" - bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") - array_data = array.array("i", [1, 2, 3, 4, 5]) - - s["foo"] = bar - s["bytes_data"] = bytes_data - s["bytearray_data"] = bytearray_data - s["array_data"] = array_data - - if proto == 5: - self.assertEqual( - s["foo"], f"{len(type(bar).__name__)}" - ) - self.assertEqual( - s["bytes_data"], - f"{len(type(bytes_data).__name__)}", - ) - self.assertEqual( - s["bytearray_data"], - f"{len(type(bytearray_data).__name__)}", - ) - self.assertEqual( - s["array_data"], - f"{len(type(array_data).__name__)}", - ) - - key, value = s.set_location(b"foo") - self.assertEqual("foo", key) - self.assertEqual(value, f"{len(type(bar).__name__)}") - - key, value = s.previous() - self.assertEqual("bytes_data", key) - self.assertEqual( - value, f"{len(type(bytes_data).__name__)}" - ) - - key, value = s.previous() - self.assertEqual("bytearray_data", key) - self.assertEqual( - value, f"{len(type(bytearray_data).__name__)}" - ) - - key, value = s.previous() - self.assertEqual("array_data", key) - self.assertEqual( - value, f"{len(type(array_data).__name__)}" - ) - - key, value = s.next() - self.assertEqual("bytearray_data", key) - self.assertEqual( - value, f"{len(type(bytearray_data).__name__)}" - ) - - key, value = s.next() - self.assertEqual("bytes_data", key) - self.assertEqual( - value, f"{len(type(bytes_data).__name__)}" - ) - - key, value = s.first() - self.assertEqual("array_data", key) - self.assertEqual( - value, f"{len(type(array_data).__name__)}" - ) - else: - key, value = s.set_location(b"foo") - self.assertEqual("foo", key) - self.assertEqual(value, "str") - - key, value = s.previous() - self.assertEqual("bytes_data", key) - self.assertEqual(value, "bytes") - - key, value = s.previous() - self.assertEqual("bytearray_data", key) - self.assertEqual(value, "bytearray") - - key, value = s.previous() - self.assertEqual("array_data", key) - self.assertEqual(value, "array") - - key, value = s.next() - self.assertEqual("bytearray_data", key) - self.assertEqual(value, "bytearray") - - key, value = s.next() - self.assertEqual("bytes_data", key) - self.assertEqual(value, "bytes") - - key, value = s.first() - self.assertEqual("array_data", key) - self.assertEqual(value, "array") - - self.assertEqual(s["foo"], "str") - self.assertEqual(s["bytes_data"], "bytes") - self.assertEqual(s["bytearray_data"], "bytearray") - self.assertEqual(s["array_data"], "array") + with self.subTest(proto=proto), shelve.BsdDbShelf( + berkeleydb.btopen(self.fn), + protocol=proto, + serializer=serializer, + deserializer=deserializer, + ) as s: + bar = "bar" + bytes_data = b"Hello, world!" + bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") + array_data = array.array("i", [1, 2, 3, 4, 5]) + + s["foo"] = bar + s["bytes_data"] = bytes_data + s["bytearray_data"] = bytearray_data + s["array_data"] = array_data + + if proto == 5: + self.assertEqual( + s["foo"], f"{len(type(bar).__name__)}" + ) + self.assertEqual( + s["bytes_data"], + f"{len(type(bytes_data).__name__)}", + ) + self.assertEqual( + s["bytearray_data"], + f"{len(type(bytearray_data).__name__)}", + ) + self.assertEqual( + s["array_data"], + f"{len(type(array_data).__name__)}", + ) + + key, value = s.set_location(b"foo") + self.assertEqual("foo", key) + self.assertEqual(value, f"{len(type(bar).__name__)}") + + key, value = s.previous() + self.assertEqual("bytes_data", key) + self.assertEqual( + value, f"{len(type(bytes_data).__name__)}" + ) + + key, value = s.previous() + self.assertEqual("bytearray_data", key) + self.assertEqual( + value, f"{len(type(bytearray_data).__name__)}" + ) + + key, value = s.previous() + self.assertEqual("array_data", key) + self.assertEqual( + value, f"{len(type(array_data).__name__)}" + ) + + key, value = s.next() + self.assertEqual("bytearray_data", key) + self.assertEqual( + value, f"{len(type(bytearray_data).__name__)}" + ) + + key, value = s.next() + self.assertEqual("bytes_data", key) + self.assertEqual( + value, f"{len(type(bytes_data).__name__)}" + ) + + key, value = s.first() + self.assertEqual("array_data", key) + self.assertEqual( + value, f"{len(type(array_data).__name__)}" + ) + else: + key, value = s.set_location(b"foo") + self.assertEqual("foo", key) + self.assertEqual(value, "str") + + key, value = s.previous() + self.assertEqual("bytes_data", key) + self.assertEqual(value, "bytes") + + key, value = s.previous() + self.assertEqual("bytearray_data", key) + self.assertEqual(value, "bytearray") + + key, value = s.previous() + self.assertEqual("array_data", key) + self.assertEqual(value, "array") + + key, value = s.next() + self.assertEqual("bytearray_data", key) + self.assertEqual(value, "bytearray") + + key, value = s.next() + self.assertEqual("bytes_data", key) + self.assertEqual(value, "bytes") + + key, value = s.first() + self.assertEqual("array_data", key) + self.assertEqual(value, "array") + + self.assertEqual(s["foo"], "str") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual(s["array_data"], "array") def test_custom_incomplete_serializer_and_deserializer_bsd_db_shelf(self): berkeleydb = import_helper.import_module("berkeleydb") From 34a32b9842ca2bd35b975ab81e91e404358b604f Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Fri, 30 May 2025 19:03:07 +0300 Subject: [PATCH 56/59] Add type_name_len helper and use shorter variable names to reduce line wrapping --- Lib/test/test_shelve.py | 146 +++++++++++++++++----------------------- 1 file changed, 63 insertions(+), 83 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 8e00b9e103f3e5..bb4caf3c89a17a 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -269,6 +269,9 @@ def serializer(obj, protocol=None): def deserializer(data): return data.decode("utf-8") + def type_name_len(obj): + return f"{(len(type(obj).__name__))}" + for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto), shelve.BsdDbShelf( berkeleydb.btopen(self.fn), @@ -277,99 +280,77 @@ def deserializer(data): deserializer=deserializer, ) as s: bar = "bar" - bytes_data = b"Hello, world!" - bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") - array_data = array.array("i", [1, 2, 3, 4, 5]) + bytes_obj = b"Hello, world!" + bytearray_obj = bytearray(b"\x00\x01\x02\x03\x04") + arr_obj = array.array("i", [1, 2, 3, 4, 5]) s["foo"] = bar - s["bytes_data"] = bytes_data - s["bytearray_data"] = bytearray_data - s["array_data"] = array_data + s["bytes_data"] = bytes_obj + s["bytearray_data"] = bytearray_obj + s["array_data"] = arr_obj if proto == 5: - self.assertEqual( - s["foo"], f"{len(type(bar).__name__)}" - ) - self.assertEqual( - s["bytes_data"], - f"{len(type(bytes_data).__name__)}", - ) - self.assertEqual( - s["bytearray_data"], - f"{len(type(bytearray_data).__name__)}", - ) - self.assertEqual( - s["array_data"], - f"{len(type(array_data).__name__)}", - ) - - key, value = s.set_location(b"foo") - self.assertEqual("foo", key) - self.assertEqual(value, f"{len(type(bar).__name__)}") - - key, value = s.previous() - self.assertEqual("bytes_data", key) - self.assertEqual( - value, f"{len(type(bytes_data).__name__)}" - ) - - key, value = s.previous() - self.assertEqual("bytearray_data", key) - self.assertEqual( - value, f"{len(type(bytearray_data).__name__)}" - ) - - key, value = s.previous() - self.assertEqual("array_data", key) - self.assertEqual( - value, f"{len(type(array_data).__name__)}" - ) - - key, value = s.next() - self.assertEqual("bytearray_data", key) - self.assertEqual( - value, f"{len(type(bytearray_data).__name__)}" - ) - - key, value = s.next() - self.assertEqual("bytes_data", key) - self.assertEqual( - value, f"{len(type(bytes_data).__name__)}" - ) - - key, value = s.first() - self.assertEqual("array_data", key) - self.assertEqual( - value, f"{len(type(array_data).__name__)}" - ) + self.assertEqual(s["foo"], type_name_len(bar)) + self.assertEqual(s["bytes_data"], type_name_len(bytes_obj)) + self.assertEqual(s["bytearray_data"], + type_name_len(bytearray_obj)) + self.assertEqual(s["array_data"], type_name_len(arr_obj)) + + k, v = s.set_location(b"foo") + self.assertEqual(k, "foo") + self.assertEqual(v, type_name_len(bar)) + + k, v = s.previous() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, type_name_len(bytes_obj)) + + k, v = s.previous() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, type_name_len(bytearray_obj)) + + k, v = s.previous() + self.assertEqual(k, "array_data") + self.assertEqual(v, type_name_len(arr_obj)) + + k, v = s.next() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, type_name_len(bytearray_obj)) + + k, v = s.next() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, type_name_len(bytes_obj)) + + k, v = s.first() + self.assertEqual(k, "array_data") + self.assertEqual(v, type_name_len(arr_obj)) else: - key, value = s.set_location(b"foo") - self.assertEqual("foo", key) - self.assertEqual(value, "str") + k, v = s.set_location(b"foo") + self.assertEqual(k, "foo") + self.assertEqual(v, "str") - key, value = s.previous() - self.assertEqual("bytes_data", key) - self.assertEqual(value, "bytes") + k, v = s.previous() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, "bytes") - key, value = s.previous() - self.assertEqual("bytearray_data", key) - self.assertEqual(value, "bytearray") + k, v = s.previous() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, "bytearray") - key, value = s.previous() - self.assertEqual("array_data", key) - self.assertEqual(value, "array") + k, v = s.previous() + self.assertEqual(k, "array_data") + self.assertEqual(v, "array") - key, value = s.next() - self.assertEqual("bytearray_data", key) - self.assertEqual(value, "bytearray") + k, v = s.next() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, "bytearray") - key, value = s.next() - self.assertEqual("bytes_data", key) - self.assertEqual(value, "bytes") + k, v = s.next() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, "bytes") - key, value = s.first() - self.assertEqual("array_data", key) - self.assertEqual(value, "array") + k, v = s.first() + self.assertEqual(k, "array_data") + self.assertEqual(v, "array") self.assertEqual(s["foo"], "str") self.assertEqual(s["bytes_data"], "bytes") @@ -407,7 +388,6 @@ def deserializer(data): self.assertEqual(s["foo"], "") self.assertNotEqual(s["foo"], "bar") - def test_missing_custom_deserializer(self): def serializer(obj, protocol=None): pass From 4b000cd99d9398a1f36989e2c01540e0ef37bfa9 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 2 Jun 2025 20:05:48 +0300 Subject: [PATCH 57/59] Improve the description of the open function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/shelve.rst | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 01f6c3308f7fff..84fe97cf7f9212 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -44,13 +44,18 @@ lots of shared sub-objects. The keys are ordinary strings. By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` for serializing and deserializing. This can be changed by supplying - *serializer* and *deserializer*, respectively. The *serializer* argument - should be a function that takes an object and the *protocol* argument passed - to the open function and returns its representation as a - :term:`bytes-like object`; *protocol* argument that may be ignored by the - function. *deserializer* should be a function that takes :class:`bytes` and - returns the corresponding object. If one of these is given, the other must - be given as well. Otherwise :mod:`shelve` will raise a :exc:`ShelveError`. + *serializer* and *deserializer*, respectively. + + The *serializer* argument must be a callable which takes an object ``obj`` + and the *protocol* as inputs and returns the representation ``obj`` as a + :term:`bytes-like object`; the *protocol* value may be ignored by the + serializer. + + The *deserializer* argument must be callable which takes a serialized object + given as a :class:`bytes` object and returns the corresponding object. + + A :exc:`ShelveError` is raised if *serializer* is given but *deserializer* + is not, or vice-versa. .. versionchanged:: 3.10 :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle From 2dcda2a1dc3badcd8b76acef9a43ef1cd462b20d Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 2 Jun 2025 20:07:20 +0300 Subject: [PATCH 58/59] Update the description of ShelveError MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/shelve.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 84fe97cf7f9212..fc99f4590cc480 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -258,7 +258,7 @@ Exceptions Exception raised when one of the arguments *deserializer* and *serializer* is missing in the :func:`~shelve.open`, :class:`Shelf`, :class:`BsdDbShelf` - and :class:`DbfilenameShelf` + and :class:`DbfilenameShelf`. The *deserializer* and *serializer* arguments must be given together. From 00bfb014052d375cece488bae287a83752eadecc Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 2 Jun 2025 20:14:25 +0300 Subject: [PATCH 59/59] Simplify conditional branches in serializer and deserializer functions --- Lib/test/test_shelve.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index bb4caf3c89a17a..64609ab9dd9a62 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -174,24 +174,19 @@ def serializer(obj, protocol): if isinstance(obj, (bytes, bytearray, str)): if protocol == 5: return obj - else: - return type(obj).__name__ + return type(obj).__name__ elif isinstance(obj, array.array): return obj.tobytes() - else: - raise TypeError( - f"Unsupported type for serialization: {type(obj)}" - ) + raise TypeError(f"Unsupported type for serialization: {type(obj)}") def deserializer(data): if isinstance(data, (bytes, bytearray, str)): return data.decode("utf-8") elif isinstance(data, array.array): return array.array("b", data) - else: - raise TypeError( - f"Unsupported type for deserialization: {type(data)}" - ) + raise TypeError( + f"Unsupported type for deserialization: {type(data)}" + ) for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto), shelve.open( pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy