From 8c19c811c8077d671a1b92442fd3756decd12db8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 22 Jan 2024 14:46:27 +0100 Subject: [PATCH 01/38] gh-100414: Add SQLite backend to dbm --- Doc/whatsnew/3.13.rst | 3 + Lib/dbm/__init__.py | 11 ++ Lib/dbm/sqlite3.py | 104 ++++++++++++++++++ ...-01-23-13-03-22.gh-issue-100414.5kTdU5.rst | 2 + 4 files changed, 120 insertions(+) create mode 100644 Lib/dbm/sqlite3.py create mode 100644 Misc/NEWS.d/next/Library/2024-01-23-13-03-22.gh-issue-100414.5kTdU5.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 40f0cd37fe9318..f1e663e415c0f4 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -196,6 +196,9 @@ dbm from the database. (Contributed by Donghee Na in :gh:`107122`.) +* Add new :mod:`dbm.sqlite3` backend. + (Contributed by Raymond Hettinger and Erlend E. Aasland in :gh:`100414`.) + doctest ------- diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py index 8055d3769f9dd0..3de4d30228ffdc 100644 --- a/Lib/dbm/__init__.py +++ b/Lib/dbm/__init__.py @@ -49,6 +49,13 @@ class error(Exception): except ImportError: ndbm = None +try: + import sqlite3 + _names.insert(0, 'dbm.sqlite3') + _has_sqlite3 = True +except ImportError: + _has_sqlite3 = False + def open(file, flag='r', mode=0o666): """Open or create database at path given by *file*. @@ -164,6 +171,10 @@ def whichdb(filename): if len(s) != 4: return "" + # Check for SQLite3 header string. + if s16 == b"SQLite format 3\0": + return "dbm.sqlite3" if _has_sqlite3 else "" + # Convert to 4-byte int in native byte order -- return "" if impossible try: (magic,) = struct.unpack("=l", s) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py new file mode 100644 index 00000000000000..096d4c3c327eb4 --- /dev/null +++ b/Lib/dbm/sqlite3.py @@ -0,0 +1,104 @@ +import os +import sqlite3 +from contextlib import suppress, closing +from collections.abc import MutableMapping + +BUILD_TABLE = """ + CREATE TABLE Dict ( + key TEXT NOT NULL, + value BLOB NOT NULL, + PRIMARY KEY (key) + ) +""" +GET_SIZE = "SELECT COUNT (key) FROM Dict" +LOOKUP_KEY = "SELECT value FROM Dict WHERE key = ? LIMIT 1" +STORE_KV = "REPLACE INTO Dict (key, value) VALUES (?, ?)" +DELETE_KEY = "DELETE FROM Dict WHERE key = ?" +ITER_KEYS = "SELECT key FROM Dict" + + +def _normalize_uri_path(path): + path = os.fsdecode(path) + path = path.replace("?", "%3f") + path = path.replace("#", "%23") + while "//" in path: + path = path.replace("//", "/") + return path + + +class _Database(MutableMapping): + + def __init__(self, path, flag): + match flag: + case "r": + flag = "ro" + case "w": + flag = "rw" + case "c": + flag = "rwc" + case "n": + flag = "rwc" + try: + os.remove(path) + except FileNotFoundError: + pass + case _: + raise ValueError("Flag must be one of 'r', 'w', 'c', or 'n', not", + repr(flag)) + + # We use the URI format when opening the database. + if not path or path == ":memory:": + uri = "file:?mode=memory" + else: + path = _normalize_uri_path(path) + uri = f"file:{path}?mode={flag}" + + self.cx = sqlite3.connect(uri, autocommit=True, uri=True) + self.cx.execute("PRAGMA journal_mode = wal") + with suppress(sqlite3.OperationalError): + self.cx.execute(BUILD_TABLE) + + def __len__(self): + return self.cx.execute(GET_SIZE).fetchone()[0] + + def __getitem__(self, key): + rows = [row for row in self.cx.execute(LOOKUP_KEY, (key,))] + if not rows: + raise KeyError(key) + assert len(rows) == 1 + row = rows[0] + return row[0] + + def __setitem__(self, key, value): + self.cx.execute(STORE_KV, (key, value)); + + def __delitem__(self, key): + if key not in self: + raise KeyError(key) + self.cx.execute(DELETE_KEY, (key,)); + + def __iter__(self): + with closing(self.cx.execute(ITER_KEYS)) as cu: + for row in cu: + yield row[0] + + def close(self): + self.cx.close() + self.cx = None + + def keys(self): + return list(super().keys()) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +class error(OSError, sqlite3.Error): + pass + + +def open(path, flag="r", mode=None): + return _Database(path, flag) diff --git a/Misc/NEWS.d/next/Library/2024-01-23-13-03-22.gh-issue-100414.5kTdU5.rst b/Misc/NEWS.d/next/Library/2024-01-23-13-03-22.gh-issue-100414.5kTdU5.rst new file mode 100644 index 00000000000000..ffcb926a8d546c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-23-13-03-22.gh-issue-100414.5kTdU5.rst @@ -0,0 +1,2 @@ +Add :mod:`dbm.sqlite3` as a backend to :mod:`dbm`. +Patch by Raymond Hettinger and Erlend E. Aasland. From 7f327722990cf5de922f52a46645636653963337 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 23 Jan 2024 13:42:12 +0100 Subject: [PATCH 02/38] Preliminary docs --- Doc/library/dbm.rst | 72 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index cb95c61322582f..bad81796450c18 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -8,8 +8,13 @@ -------------- -:mod:`dbm` is a generic interface to variants of the DBM database --- -:mod:`dbm.gnu` or :mod:`dbm.ndbm`. If none of these modules is installed, the +:mod:`dbm` is a generic interface to variants of the DBM database: + +* :mod:`dbm.sqlite3` +* :mod:`dbm.gnu` +* :mod:`dbm.ndbm` + +If none of these modules are installed, the slow-but-simple implementation in module :mod:`dbm.dumb` will be used. There is a `third party interface `_ to the Oracle Berkeley DB. @@ -129,6 +134,69 @@ then prints out the contents of the database:: The individual submodules are described in the following sections. +.. Substitutions for the flag docs; all submodules use the same text. + +.. |flag_r| replace:: + Open existing database for reading only (default) + +.. |flag_w| replace:: + Open existing database for reading and writing + +.. |flag_c| replace:: + Open database for reading and writing, creating it if it doesn't exist. + +.. |flag_n| replace:: + Always create a new, empty database, open for reading and writing. + +:mod:`dbm.sqlite3` --- SQLite backend for dbm +---------------------------------------------- + +.. module:: dbm.sqlite3 + :platform: All + :synopsis: SQLite backend for dbm + +.. versionadded:: 3.13 + +**Source code:** :source:`Lib/dbm/sqlite3.py` + +-------------- + +This module uses the standard library :mod:`sqlite3` to provide an +SQLite backend for the :mod:`dbm` module. +The files created by :mod:`dbm.sqlite3` can thus be opened by :mod:`sqlite3`, +or any other SQLite browser, including the SQLite CLI. + +.. function:: open(filename, flag="r", mode=None) + + Open an SQLite database. + The returned object, behaves like a :term:`mapping`, + but also implements a :meth:`!close` method + and supports a "closing" context manager via the :keyword:`with` keyword. + + Neither keys nor values are coerced to a specific type + before being stored in the database. + + :param filename: + The path to the database to be opened. + :type filename: :term:`path-like object` + + :param str flag: + + .. list-table:: + :header-rows: 1 + + * - ``'r'`` + - |flag_r| + * - ``'w'`` + - |flag_w| + * - ``'c'`` + - |flag_c| + * - ``'n'`` + - |flag_n| + + :param mode: + This parameter is ignored by the :mod:`!dbm.sqlite3` module. + :mod:`dbm.gnu` --- GNU's reinterpretation of dbm ------------------------------------------------ From 3f8a119395a198c83659796944d7e57bdd35341f Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 23 Jan 2024 13:44:22 +0100 Subject: [PATCH 03/38] Decode path a little bit earlier --- Lib/dbm/sqlite3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 096d4c3c327eb4..53d515f9e47903 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -18,7 +18,6 @@ def _normalize_uri_path(path): - path = os.fsdecode(path) path = path.replace("?", "%3f") path = path.replace("#", "%23") while "//" in path: @@ -47,6 +46,7 @@ def __init__(self, path, flag): repr(flag)) # We use the URI format when opening the database. + path = os.fsdecode(path) if not path or path == ":memory:": uri = "file:?mode=memory" else: From 38a9b413c11cff28325fc94b8b2b4f21a524e04b Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 23 Jan 2024 20:56:24 +0100 Subject: [PATCH 04/38] Substitutions are already in place --- Doc/library/dbm.rst | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index a1d6d8b33bd86e..5ab56cd3eed2a9 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -141,20 +141,6 @@ then prints out the contents of the database:: The individual submodules are described in the following sections. -.. Substitutions for the flag docs; all submodules use the same text. - -.. |flag_r| replace:: - Open existing database for reading only (default) - -.. |flag_w| replace:: - Open existing database for reading and writing - -.. |flag_c| replace:: - Open database for reading and writing, creating it if it doesn't exist. - -.. |flag_n| replace:: - Always create a new, empty database, open for reading and writing. - :mod:`dbm.sqlite3` --- SQLite backend for dbm ---------------------------------------------- From db0f1488a7f64ae89b4b7c5f48037724738220ef Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 23 Jan 2024 20:57:55 +0100 Subject: [PATCH 05/38] Use CSV-style table, as the rest of the .rst file does --- Doc/library/dbm.rst | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 5ab56cd3eed2a9..e9667ffec25a1f 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -175,17 +175,13 @@ or any other SQLite browser, including the SQLite CLI. :param str flag: - .. list-table:: - :header-rows: 1 - - * - ``'r'`` - - |flag_r| - * - ``'w'`` - - |flag_w| - * - ``'c'`` - - |flag_c| - * - ``'n'`` - - |flag_n| + .. csv-table:: + :header: "Value", "Meaning" + + ``'r'`` (default), |flag_r| + ``'w'``, |flag_w| + ``'c'``, |flag_c| + ``'n'``, |flag_n| :param mode: This parameter is ignored by the :mod:`!dbm.sqlite3` module. From e50d1e9d1b99cb9eecc9d0dcd3abeeedaaa8cb5f Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 23 Jan 2024 21:55:13 +0100 Subject: [PATCH 06/38] dbm.sqlite3.error is a subclass of OSError --- Lib/dbm/sqlite3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 53d515f9e47903..3b3de6b0c27f8f 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -96,7 +96,7 @@ def __exit__(self, *args): self.close() -class error(OSError, sqlite3.Error): +class error(OSError): pass From 0abd7dc313b61705b562423bd5561a2551afc525 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 23 Jan 2024 22:59:31 +0100 Subject: [PATCH 07/38] Don't raise DB API exceptions; test this --- Lib/dbm/sqlite3.py | 31 +++++++----- Lib/test/test_dbm_sqlite3.py | 92 ++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 11 deletions(-) create mode 100644 Lib/test/test_dbm_sqlite3.py diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 3b3de6b0c27f8f..db4bc90f2b1c10 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -17,6 +17,10 @@ ITER_KEYS = "SELECT key FROM Dict" +class error(OSError): + pass + + def _normalize_uri_path(path): path = path.replace("?", "%3f") path = path.replace("#", "%23") @@ -55,14 +59,23 @@ def __init__(self, path, flag): self.cx = sqlite3.connect(uri, autocommit=True, uri=True) self.cx.execute("PRAGMA journal_mode = wal") - with suppress(sqlite3.OperationalError): - self.cx.execute(BUILD_TABLE) + if flag == "rwc": + with suppress(sqlite3.OperationalError): + self.cx.execute(BUILD_TABLE) + + def _execute(self, *args, **kwargs): + try: + ret = self.cx.execute(*args, **kwargs) + except sqlite3.Error as exc: + raise error(str(exc)) + else: + return ret def __len__(self): - return self.cx.execute(GET_SIZE).fetchone()[0] + return self._execute(GET_SIZE).fetchone()[0] def __getitem__(self, key): - rows = [row for row in self.cx.execute(LOOKUP_KEY, (key,))] + rows = [row for row in self._execute(LOOKUP_KEY, (key,))] if not rows: raise KeyError(key) assert len(rows) == 1 @@ -70,15 +83,15 @@ def __getitem__(self, key): return row[0] def __setitem__(self, key, value): - self.cx.execute(STORE_KV, (key, value)); + self._execute(STORE_KV, (key, value)); def __delitem__(self, key): if key not in self: raise KeyError(key) - self.cx.execute(DELETE_KEY, (key,)); + self._execute(DELETE_KEY, (key,)); def __iter__(self): - with closing(self.cx.execute(ITER_KEYS)) as cu: + with closing(self._execute(ITER_KEYS)) as cu: for row in cu: yield row[0] @@ -96,9 +109,5 @@ def __exit__(self, *args): self.close() -class error(OSError): - pass - - def open(path, flag="r", mode=None): return _Database(path, flag) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py new file mode 100644 index 00000000000000..1d0b113efd191d --- /dev/null +++ b/Lib/test/test_dbm_sqlite3.py @@ -0,0 +1,92 @@ +import sqlite3 +import test.support +import unittest +from contextlib import closing +from functools import partial +from test.support import cpython_only, import_helper, os_helper + + +dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") + + +class _SQLiteDbmTests(unittest.TestCase): + + def setUp(self): + self.filename = os_helper.TESTFN + db = dbm_sqlite3.open(self.filename, "c") + db.close() + + def tearDown(self): + for suffix in "", "-wal", "-shm": + os_helper.unlink(self.filename + suffix) + + +class DataTypes(_SQLiteDbmTests): + + dataset = 10, 2.5, "string", b"bytes" + + def setUp(self): + super().setUp() + self.db = dbm_sqlite3.open(self.filename, "w") + + def tearDown(self): + super().tearDown() + self.db.close() + + def test_values(self): + for value in self.dataset: + with self.subTest(value=value): + self.db["key"] = value + self.assertEqual(self.db["key"], value) + + def test_keys(self): + for key in self.dataset: + with self.subTest(key=key): + self.db[key] = "value" + self.assertEqual(self.db[key], "value") + + +class CorruptDatabase(_SQLiteDbmTests): + """Verify that database exceptions are raised as dbm.sqlite3.error.""" + + def setUp(self): + super().setUp() + with closing(sqlite3.connect(self.filename)) as cx: + with cx: + cx.execute("DROP TABLE IF EXISTS Dict") + cx.execute("CREATE TABLE Dict (invalid_schema)") + + def check(self, flag, fn, should_succeed=False): + with closing(dbm_sqlite3.open(self.filename, flag)) as db: + with self.assertRaises(dbm_sqlite3.error): + fn(db) + + @staticmethod + def read(db): + return db["key"] + + @staticmethod + def write(db): + db["key"] = "value" + + def test_readonly(self): + self.check(flag="r", fn=self.read) + + def test_readwrite(self): + check = partial(self.check, flag="w") + check(fn=self.read) + check(fn=self.write) + + def test_create(self): + check = partial(self.check, flag="c") + check(fn=self.read) + check(fn=self.write) + + def test_new(self): + with closing(dbm_sqlite3.open(self.filename, "n")) as db: + db["foo"] = "write" + _ = db["foo"] + + +if __name__ == "__main__": + unittest.main() From 2b9716040c8d50e2497896f90ccb1f7da252fc51 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 23 Jan 2024 23:21:05 +0100 Subject: [PATCH 08/38] Don't panic on double close(); add more tests --- Lib/dbm/sqlite3.py | 5 +-- Lib/test/test_dbm_sqlite3.py | 69 +++++++++++++++++++++++++++++++----- 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index db4bc90f2b1c10..1d9779ab19d66e 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -96,8 +96,9 @@ def __iter__(self): yield row[0] def close(self): - self.cx.close() - self.cx = None + if self.cx: + self.cx.close() + self.cx = None def keys(self): return list(super().keys()) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 1d0b113efd191d..2a3387d5c8d8af 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -21,6 +21,34 @@ def tearDown(self): os_helper.unlink(self.filename + suffix) +class Misuse(_SQLiteDbmTests): + + def setUp(self): + super().setUp() + self.db = dbm_sqlite3.open(self.filename, "w") + + def tearDown(self): + super().tearDown() + self.db.close() + + def test_double_close(self): + self.db.close() + + def test_invalid_flag(self): + with self.assertRaises(ValueError): + dbm_sqlite3.open(self.filename, flag="invalid") + + def test_double_delete(self): + self.db["key"] = "value" + del self.db["key"] + with self.assertRaises(KeyError): + del self.db["key"] + + def test_invalid_key(self): + with self.assertRaises(KeyError): + self.db["key"] + + class DataTypes(_SQLiteDbmTests): dataset = 10, 2.5, "string", b"bytes" @@ -69,23 +97,46 @@ def read(db): def write(db): db["key"] = "value" - def test_readonly(self): - self.check(flag="r", fn=self.read) + @staticmethod + def iter(db): + next(iter(db)) - def test_readwrite(self): - check = partial(self.check, flag="w") - check(fn=self.read) - check(fn=self.write) + @staticmethod + def keys(db): + db.keys() - def test_create(self): - check = partial(self.check, flag="c") + @staticmethod + def del_(db): + del db["key"] + + @staticmethod + def len_(db): + len(db) + + def test_readonly(self): + check = partial(self.check, flag="r") check(fn=self.read) - check(fn=self.write) + check(fn=self.iter) + check(fn=self.keys) + check(fn=self.del_) + + def test_readwrite(self): + for flag in "w", "c": + with self.subTest(flag=flag): + check = partial(self.check, flag=flag) + check(fn=self.read) + check(fn=self.write) + check(fn=self.iter) + check(fn=self.keys) + check(fn=self.del_) + check(fn=self.len_) def test_new(self): with closing(dbm_sqlite3.open(self.filename, "n")) as db: db["foo"] = "write" _ = db["foo"] + next(iter(db)) + del db["foo"] if __name__ == "__main__": From 917bfbaf9215a44ca30795a52dc2d827623f3d75 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 23 Jan 2024 23:29:16 +0100 Subject: [PATCH 09/38] Corruption tests: check all basic operations for flags 'r', 'w', 'c' --- Lib/test/test_dbm_sqlite3.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 2a3387d5c8d8af..52af95c58fe3a3 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -113,15 +113,8 @@ def del_(db): def len_(db): len(db) - def test_readonly(self): - check = partial(self.check, flag="r") - check(fn=self.read) - check(fn=self.iter) - check(fn=self.keys) - check(fn=self.del_) - def test_readwrite(self): - for flag in "w", "c": + for flag in "r", "w", "c": with self.subTest(flag=flag): check = partial(self.check, flag=flag) check(fn=self.read) From 874fb81cecab9ec36545d7e58f31a0e3c726be81 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 23 Jan 2024 23:43:04 +0100 Subject: [PATCH 10/38] Add read-only specific test --- Lib/test/test_dbm_sqlite3.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 52af95c58fe3a3..ad94ca5a871dc3 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -21,6 +21,38 @@ def tearDown(self): os_helper.unlink(self.filename + suffix) +class ReadOnly(_SQLiteDbmTests): + + def setUp(self): + super().setUp() + with dbm_sqlite3.open(self.filename, "w") as db: + db["key1"] = "value1" + db["key2"] = "value2" + self.db = dbm_sqlite3.open(self.filename, "r") + + def tearDown(self): + self.db.close() + super().tearDown() + + def test_read(self): + self.assertEqual(self.db["key1"], "value1") + self.assertEqual(self.db["key2"], "value2") + + def test_write(self): + with self.assertRaises(dbm_sqlite3.error): + self.db["new"] = "value" + + def test_delete(self): + with self.assertRaises(dbm_sqlite3.error): + del self.db["key1"] + + def test_keys(self): + self.assertEqual(self.db.keys(), ["key1", "key2"]) + + def test_iter(self): + self.assertEqual([k for k in self.db], ["key1", "key2"]) + + class Misuse(_SQLiteDbmTests): def setUp(self): @@ -28,8 +60,8 @@ def setUp(self): self.db = dbm_sqlite3.open(self.filename, "w") def tearDown(self): - super().tearDown() self.db.close() + super().tearDown() def test_double_close(self): self.db.close() @@ -58,8 +90,8 @@ def setUp(self): self.db = dbm_sqlite3.open(self.filename, "w") def tearDown(self): - super().tearDown() self.db.close() + super().tearDown() def test_values(self): for value in self.dataset: From b655473dbd199006945c2aab8e754684ba7f1fec Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 24 Jan 2024 09:21:14 +0100 Subject: [PATCH 11/38] Address review: always import sqlite3; let ImportError deal with missing deps --- Lib/dbm/__init__.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py index 3de4d30228ffdc..c161f520b974cf 100644 --- a/Lib/dbm/__init__.py +++ b/Lib/dbm/__init__.py @@ -38,7 +38,7 @@ class error(Exception): pass -_names = ['dbm.gnu', 'dbm.ndbm', 'dbm.dumb'] +_names = ['dbm.sqlite3', 'dbm.gnu', 'dbm.ndbm', 'dbm.dumb'] _defaultmod = None _modules = {} @@ -49,13 +49,6 @@ class error(Exception): except ImportError: ndbm = None -try: - import sqlite3 - _names.insert(0, 'dbm.sqlite3') - _has_sqlite3 = True -except ImportError: - _has_sqlite3 = False - def open(file, flag='r', mode=0o666): """Open or create database at path given by *file*. @@ -173,7 +166,7 @@ def whichdb(filename): # Check for SQLite3 header string. if s16 == b"SQLite format 3\0": - return "dbm.sqlite3" if _has_sqlite3 else "" + return "dbm.sqlite3" # Convert to 4-byte int in native byte order -- return "" if impossible try: From 95eba9348c8939cc714881b2ac08648aaf0bfad6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 24 Jan 2024 09:41:46 +0100 Subject: [PATCH 12/38] Test whichdb --- Lib/test/test_dbm.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index e3924d8ec8b5c1..87ce7932de0eaa 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -6,6 +6,13 @@ from test.support import import_helper from test.support import os_helper + +try: + from dbm import sqlite3 as dbm_sqlite3 +except ImportError: + dbm_sqlite3 = None + + try: from dbm import ndbm except ImportError: @@ -213,6 +220,27 @@ def test_whichdb_ndbm(self): for path in fnames: self.assertIsNone(self.dbm.whichdb(path)) + @unittest.skipUnless(dbm_sqlite3, reason='Test requires dbm.sqlite3') + def test_whichdb_sqlite3(self): + # Test that databases created by dbm.sqlite3 are detected correctly. + with dbm_sqlite3.open(_fname, "c") as db: + db["key"] = "value" + self.assertEqual(self.dbm.whichdb(_fname), "dbm.sqlite3") + + @unittest.skipUnless(dbm_sqlite3, reason='Test requires dbm.sqlite3') + def test_whichdb_sqlite3_existing_db(self): + # Test that existing sqlite3 databases are detected correctly. + sqlite3 = import_helper.import_module("sqlite3") + try: + # Create an empty database. + with sqlite3.connect(_fname) as cx: + cx.execute("CREATE TABLE dummy(database)") + cx.commit() + finally: + cx.close() + self.assertEqual(self.dbm.whichdb(_fname), "dbm.sqlite3") + + def setUp(self): self.addCleanup(cleaunup_test_dir) setup_test_dir() From 54751ba95c683b0d5fc57eeaa7a5fe98e1f740f3 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 24 Jan 2024 09:44:49 +0100 Subject: [PATCH 13/38] test namespacing --- Lib/test/test_dbm_sqlite3.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index ad94ca5a871dc3..675234374cd8f9 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -34,22 +34,22 @@ def tearDown(self): self.db.close() super().tearDown() - def test_read(self): + def test_readonly_read(self): self.assertEqual(self.db["key1"], "value1") self.assertEqual(self.db["key2"], "value2") - def test_write(self): + def test_readonly_write(self): with self.assertRaises(dbm_sqlite3.error): self.db["new"] = "value" - def test_delete(self): + def test_readonly_delete(self): with self.assertRaises(dbm_sqlite3.error): del self.db["key1"] - def test_keys(self): + def test_readonly_keys(self): self.assertEqual(self.db.keys(), ["key1", "key2"]) - def test_iter(self): + def test_readonly_iter(self): self.assertEqual([k for k in self.db], ["key1", "key2"]) @@ -63,20 +63,20 @@ def tearDown(self): self.db.close() super().tearDown() - def test_double_close(self): + def test_misuse_double_close(self): self.db.close() - def test_invalid_flag(self): + def test_misuse_invalid_flag(self): with self.assertRaises(ValueError): dbm_sqlite3.open(self.filename, flag="invalid") - def test_double_delete(self): + def test_misuse_double_delete(self): self.db["key"] = "value" del self.db["key"] with self.assertRaises(KeyError): del self.db["key"] - def test_invalid_key(self): + def test_misuse_invalid_key(self): with self.assertRaises(KeyError): self.db["key"] @@ -93,13 +93,13 @@ def tearDown(self): self.db.close() super().tearDown() - def test_values(self): + def test_datatypes_values(self): for value in self.dataset: with self.subTest(value=value): self.db["key"] = value self.assertEqual(self.db["key"], value) - def test_keys(self): + def test_datatypes_keys(self): for key in self.dataset: with self.subTest(key=key): self.db[key] = "value" @@ -145,7 +145,7 @@ def del_(db): def len_(db): len(db) - def test_readwrite(self): + def test_corrupt_readwrite(self): for flag in "r", "w", "c": with self.subTest(flag=flag): check = partial(self.check, flag=flag) @@ -156,7 +156,7 @@ def test_readwrite(self): check(fn=self.del_) check(fn=self.len_) - def test_new(self): + def test_corrupt_force_new(self): with closing(dbm_sqlite3.open(self.filename, "n")) as db: db["foo"] = "write" _ = db["foo"] From 906748eb3412c75a97131cdeaddfc0efda32e775 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 24 Jan 2024 10:01:57 +0100 Subject: [PATCH 14/38] Test that 'c' works even if you already have a database; make sure BUILD_TABLE never can raise a DB API exception --- Lib/dbm/sqlite3.py | 5 ++--- Lib/test/test_dbm_sqlite3.py | 5 +++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 1d9779ab19d66e..a464d08f9922f5 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -4,7 +4,7 @@ from collections.abc import MutableMapping BUILD_TABLE = """ - CREATE TABLE Dict ( + CREATE TABLE IF NOT EXISTS Dict ( key TEXT NOT NULL, value BLOB NOT NULL, PRIMARY KEY (key) @@ -60,8 +60,7 @@ def __init__(self, path, flag): self.cx = sqlite3.connect(uri, autocommit=True, uri=True) self.cx.execute("PRAGMA journal_mode = wal") if flag == "rwc": - with suppress(sqlite3.OperationalError): - self.cx.execute(BUILD_TABLE) + self._execute(BUILD_TABLE) def _execute(self, *args, **kwargs): try: diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 675234374cd8f9..292340f40c68de 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -63,6 +63,11 @@ def tearDown(self): self.db.close() super().tearDown() + def test_misuse_double_create(self): + self.db["key"] = "value" + with dbm_sqlite3.open(self.filename, "c") as db: + self.assertEqual(db["key"], "value") + def test_misuse_double_close(self): self.db.close() From c89c5fb082844bc638734a530ee0f11a4635f833 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 24 Jan 2024 10:29:54 +0100 Subject: [PATCH 15/38] Remove unneeded comma --- Doc/library/dbm.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index e9667ffec25a1f..637a5e74d4be44 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -162,8 +162,8 @@ or any other SQLite browser, including the SQLite CLI. .. function:: open(filename, flag="r", mode=None) Open an SQLite database. - The returned object, behaves like a :term:`mapping`, - but also implements a :meth:`!close` method + The returned object behaves like a :term:`mapping`, + implements a :meth:`!close` method, and supports a "closing" context manager via the :keyword:`with` keyword. Neither keys nor values are coerced to a specific type From 34d8c7f7e9d385c93563e0fdd1f49b7191234703 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 24 Jan 2024 11:00:39 +0100 Subject: [PATCH 16/38] Close cursors explicitly for each query --- Lib/dbm/sqlite3.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index a464d08f9922f5..97e05c69f3c365 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -60,7 +60,8 @@ def __init__(self, path, flag): self.cx = sqlite3.connect(uri, autocommit=True, uri=True) self.cx.execute("PRAGMA journal_mode = wal") if flag == "rwc": - self._execute(BUILD_TABLE) + with closing(self._execute(BUILD_TABLE)): + pass def _execute(self, *args, **kwargs): try: @@ -71,23 +72,26 @@ def _execute(self, *args, **kwargs): return ret def __len__(self): - return self._execute(GET_SIZE).fetchone()[0] + with closing(self._execute(GET_SIZE)) as cu: + row = cu.fetchone() + return row[0] def __getitem__(self, key): - rows = [row for row in self._execute(LOOKUP_KEY, (key,))] - if not rows: + with closing(self._execute(LOOKUP_KEY, (key,))) as cu: + row = cu.fetchone() + if not row: raise KeyError(key) - assert len(rows) == 1 - row = rows[0] return row[0] def __setitem__(self, key, value): - self._execute(STORE_KV, (key, value)); + with closing(self._execute(STORE_KV, (key, value))): + pass def __delitem__(self, key): if key not in self: raise KeyError(key) - self._execute(DELETE_KEY, (key,)); + with closing(self._execute(DELETE_KEY, (key,))): + pass def __iter__(self): with closing(self._execute(ITER_KEYS)) as cu: From 608c22917dfffa04d2b212c4bc8c41d1451b1f0a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 24 Jan 2024 22:52:28 +0100 Subject: [PATCH 17/38] Address review from Donghee, Serhiy, and myself: - Amend dbm docstring - Add misuse guards - Let SQLite manage the rowid - Atomic delete - Hide connection member - Remove :memory: feature - Improve URI normalisation - Fix ValueError formatting --- Lib/dbm/__init__.py | 2 +- Lib/dbm/sqlite3.py | 81 +++++++++++++++++++++-------------- Lib/test/test_dbm_sqlite3.py | 82 +++++++++++++++++++++++++++++++++++- 3 files changed, 131 insertions(+), 34 deletions(-) diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py index c161f520b974cf..763135bec40ee9 100644 --- a/Lib/dbm/__init__.py +++ b/Lib/dbm/__init__.py @@ -5,7 +5,7 @@ import dbm d = dbm.open(file, 'w', 0o666) -The returned object is a dbm.gnu, dbm.ndbm or dbm.dumb object, dependent on the +The returned object is a dbm.sqlite3, dbm.gnu, dbm.ndbm or dbm.dumb object, dependent on the type of database being opened (determined by the whichdb function) in the case of an existing dbm. If the dbm does not exist and the create or new flag ('c' or 'n') was specified, the dbm type will be determined by the availability of diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 97e05c69f3c365..4e9846ea91282a 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -1,17 +1,18 @@ import os import sqlite3 +import sys +from pathlib import Path from contextlib import suppress, closing from collections.abc import MutableMapping BUILD_TABLE = """ CREATE TABLE IF NOT EXISTS Dict ( - key TEXT NOT NULL, - value BLOB NOT NULL, - PRIMARY KEY (key) + key TEXT UNIQUE NOT NULL, + value BLOB NOT NULL ) """ GET_SIZE = "SELECT COUNT (key) FROM Dict" -LOOKUP_KEY = "SELECT value FROM Dict WHERE key = ? LIMIT 1" +LOOKUP_KEY = "SELECT value FROM Dict WHERE key = ?" STORE_KV = "REPLACE INTO Dict (key, value) VALUES (?, ?)" DELETE_KEY = "DELETE FROM Dict WHERE key = ?" ITER_KEYS = "SELECT key FROM Dict" @@ -21,17 +22,29 @@ class error(OSError): pass +_ERR_CLOSED = "DBM object has already been closed" +_ERR_REINIT = "DBM object does not support reinitialization" + + def _normalize_uri_path(path): - path = path.replace("?", "%3f") - path = path.replace("#", "%23") - while "//" in path: - path = path.replace("//", "/") - return path + path = os.fsdecode(path) + for char in "%", "?", "#": + path = path.replace(char, f"%{ord(char):02x}") + path = Path(path) + if sys.platform == "win32": + if path.drive: + if not path.is_absolute(): + path = Path(path).absolute() + return Path("/", path).as_posix() + return path.as_posix() class _Database(MutableMapping): - def __init__(self, path, flag): + def __init__(self, path, /, flag): + if hasattr(self, "_cx"): + raise error(_ERR_REINIT) + match flag: case "r": flag = "ro" @@ -46,26 +59,28 @@ def __init__(self, path, flag): except FileNotFoundError: pass case _: - raise ValueError("Flag must be one of 'r', 'w', 'c', or 'n', not", - repr(flag)) + raise ValueError("Flag must be one of 'r', 'w', 'c', or 'n', " + f"not {flag!r}") # We use the URI format when opening the database. - path = os.fsdecode(path) - if not path or path == ":memory:": - uri = "file:?mode=memory" - else: - path = _normalize_uri_path(path) - uri = f"file:{path}?mode={flag}" + path = _normalize_uri_path(path) + uri = f"file:{path}?mode={flag}" + + self._cx = sqlite3.connect(uri, autocommit=True, uri=True) + + # This is an optimization only; it's ok if it fails. + with suppress(sqlite3.OperationalError): + self._cx.execute("PRAGMA journal_mode = wal") - self.cx = sqlite3.connect(uri, autocommit=True, uri=True) - self.cx.execute("PRAGMA journal_mode = wal") if flag == "rwc": with closing(self._execute(BUILD_TABLE)): pass def _execute(self, *args, **kwargs): + if not self._cx: + raise error(_ERR_CLOSED) try: - ret = self.cx.execute(*args, **kwargs) + ret = self._cx.execute(*args, **kwargs) except sqlite3.Error as exc: raise error(str(exc)) else: @@ -88,20 +103,22 @@ def __setitem__(self, key, value): pass def __delitem__(self, key): - if key not in self: - raise KeyError(key) - with closing(self._execute(DELETE_KEY, (key,))): - pass + with closing(self._execute(DELETE_KEY, (key,))) as cu: + if not cu.rowcount: + raise KeyError(key) def __iter__(self): - with closing(self._execute(ITER_KEYS)) as cu: - for row in cu: - yield row[0] + try: + with closing(self._execute(ITER_KEYS)) as cu: + for row in cu: + yield row[0] + except sqlite3.Error as exc: + raise error(str(exc)) def close(self): - if self.cx: - self.cx.close() - self.cx = None + if self._cx: + self._cx.close() + self._cx = None def keys(self): return list(super().keys()) @@ -113,5 +130,5 @@ def __exit__(self, *args): self.close() -def open(path, flag="r", mode=None): +def open(path, /, flag="r", mode=None): return _Database(path, flag) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 292340f40c68de..b5b9f0e9ba051b 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -53,6 +53,52 @@ def test_readonly_iter(self): self.assertEqual([k for k in self.db], ["key1", "key2"]) +class ReadWrite(_SQLiteDbmTests): + + def setUp(self): + super().setUp() + self.db = dbm_sqlite3.open(self.filename, "w") + + def tearDown(self): + self.db.close() + super().tearDown() + + def db_content(self): + with closing(sqlite3.connect(self.filename)) as cx: + keys = [r[0] for r in cx.execute("SELECT key FROM Dict")] + vals = [r[0] for r in cx.execute("SELECT value FROM Dict")] + return keys, vals + + def test_readwrite_unique_key(self): + self.db["key"] = "value" + self.db["key"] = "other" + keys, vals = self.db_content() + self.assertEqual(keys, ["key"]) + self.assertEqual(vals, ["other"]) + + def test_readwrite_delete(self): + self.db["key"] = "value" + self.db["new"] = "other" + + del self.db["new"] + keys, vals = self.db_content() + self.assertEqual(keys, ["key"]) + self.assertEqual(vals, ["value"]) + + del self.db["key"] + keys, vals = self.db_content() + self.assertEqual(keys, []) + self.assertEqual(vals, []) + + def test_readwrite_null_key(self): + with self.assertRaises(dbm_sqlite3.error): + self.db[None] = "value" + + def test_readwrite_null_value(self): + with self.assertRaises(dbm_sqlite3.error): + self.db["key"] = None + + class Misuse(_SQLiteDbmTests): def setUp(self): @@ -72,7 +118,8 @@ def test_misuse_double_close(self): self.db.close() def test_misuse_invalid_flag(self): - with self.assertRaises(ValueError): + regex = "must be.*'r'.*'w'.*'c'.*'n', not 'invalid'" + with self.assertRaisesRegex(ValueError, regex): dbm_sqlite3.open(self.filename, flag="invalid") def test_misuse_double_delete(self): @@ -85,6 +132,39 @@ def test_misuse_invalid_key(self): with self.assertRaises(KeyError): self.db["key"] + def test_misuse_iter_close1(self): + self.db["1"] = 1 + it = iter(self.db) + self.db.close() + with self.assertRaises(dbm_sqlite3.error): + next(it) + + def test_misuse_iter_close2(self): + self.db["1"] = 1 + self.db["2"] = 2 + it = iter(self.db) + next(it) + self.db.close() + with self.assertRaises(dbm_sqlite3.error): + next(it) + + def test_misuse_use_after_close(self): + self.db.close() + with self.assertRaises(dbm_sqlite3.error): + self.db["read"] + with self.assertRaises(dbm_sqlite3.error): + self.db["write"] = "value" + with self.assertRaises(dbm_sqlite3.error): + del self.db["del"] + with self.assertRaises(dbm_sqlite3.error): + len(self.db) + with self.assertRaises(dbm_sqlite3.error): + self.db.keys() + + def test_misuse_reinit(self): + with self.assertRaises(dbm_sqlite3.error): + self.db.__init__(":memory:", flag="rw") + class DataTypes(_SQLiteDbmTests): From 75f7c6ab619629f4c3efaf38ead218a77278cd6b Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 24 Jan 2024 23:48:30 +0100 Subject: [PATCH 18/38] Add URI tests, fix whitespace, catch errors during db creation --- Lib/dbm/sqlite3.py | 9 +++++--- Lib/test/test_dbm_sqlite3.py | 45 +++++++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 4e9846ea91282a..fe541738957524 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -66,7 +66,10 @@ def __init__(self, path, /, flag): path = _normalize_uri_path(path) uri = f"file:{path}?mode={flag}" - self._cx = sqlite3.connect(uri, autocommit=True, uri=True) + try: + self._cx = sqlite3.connect(uri, autocommit=True, uri=True) + except sqlite3.Error as exc: + raise error(str(exc)) # This is an optimization only; it's ok if it fails. with suppress(sqlite3.OperationalError): @@ -110,8 +113,8 @@ def __delitem__(self, key): def __iter__(self): try: with closing(self._execute(ITER_KEYS)) as cu: - for row in cu: - yield row[0] + for row in cu: + yield row[0] except sqlite3.Error as exc: raise error(str(exc)) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index b5b9f0e9ba051b..39c69ae00c81da 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -1,12 +1,15 @@ import sqlite3 +import sys import test.support import unittest from contextlib import closing from functools import partial +from pathlib import PureWindowsPath from test.support import cpython_only, import_helper, os_helper dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") +from dbm.sqlite3 import _normalize_uri_path class _SQLiteDbmTests(unittest.TestCase): @@ -21,6 +24,41 @@ def tearDown(self): os_helper.unlink(self.filename + suffix) +class URI(unittest.TestCase): + + def test_uri_substitutions(self): + dataset = ( + ("relative/b//c", "relative/b/c"), + ("/absolute/////b/c", "/absolute/b/c"), + ("PRE#MID##END", "PRE%23MID%23%23END"), + ("%#?%%#", "%25%23%3f%25%25%23"), + ) + for path, normalized in dataset: + with self.subTest(path=path, normalized=normalized): + self.assertEqual(_normalize_uri_path(path), normalized) + + @unittest.skip("WIP") + def test_uri_windows(self): + dataset = ( + # Relative subdir. + (r"2018\January.xlsx", + "2018/January.xlsx"), + # Relative CWD. + (r"..\Publications\TravelBrochure.pdf", + "../Publications/TravelBrochure.pdf"), + # Absolute with drive letter. + (r"C:\Projects\apilibrary\apilibrary.sln", + "/C:/Projects/apilibrary/apilibrary.sln"), + # Relative with drive letter. + (r"C:Projects\apilibrary\apilibrary.sln", + "/C:Projects/apilibrary/apilibrary.sln"), + ) + for path, normalized in dataset: + with self.subTest(path=path, normalized=normalized): + path = PureWindowsPath(path) + self.assertEqual(_normalize_uri_path(path), normalized) + + class ReadOnly(_SQLiteDbmTests): def setUp(self): @@ -163,7 +201,12 @@ def test_misuse_use_after_close(self): def test_misuse_reinit(self): with self.assertRaises(dbm_sqlite3.error): - self.db.__init__(":memory:", flag="rw") + self.db.__init__("new.db", flag="rw") + + def test_misuse_empty_filename(self): + for flag in "r", "w", "c", "n": + with self.assertRaises(dbm_sqlite3.error): + db = dbm_sqlite3.open("", flag="c") class DataTypes(_SQLiteDbmTests): From 3ac010c0896914b57d72d27f764a811943eb7b62 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 25 Jan 2024 00:03:41 +0100 Subject: [PATCH 19/38] Enable test on Windows --- Lib/dbm/sqlite3.py | 2 +- Lib/test/test_dbm_sqlite3.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index fe541738957524..b9c14b065540e7 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -35,7 +35,7 @@ def _normalize_uri_path(path): if path.drive: if not path.is_absolute(): path = Path(path).absolute() - return Path("/", path).as_posix() + return "/" + Path(path).as_posix() return path.as_posix() diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 39c69ae00c81da..b307f93fb3753f 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -37,7 +37,7 @@ def test_uri_substitutions(self): with self.subTest(path=path, normalized=normalized): self.assertEqual(_normalize_uri_path(path), normalized) - @unittest.skip("WIP") + @unittest.skipUnless(sys.platform == "win32", "requires Windows") def test_uri_windows(self): dataset = ( # Relative subdir. From 08c38481c10d8cbfcc9ee02f2133a3285eed216b Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 25 Jan 2024 00:57:25 +0100 Subject: [PATCH 20/38] Use as_uri() instead --- Lib/dbm/sqlite3.py | 18 +++++++----------- Lib/test/test_dbm_sqlite3.py | 17 ++++++++--------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index b9c14b065540e7..b3408b7276c5e8 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -26,17 +26,13 @@ class error(OSError): _ERR_REINIT = "DBM object does not support reinitialization" -def _normalize_uri_path(path): +def _normalize_uri(path): path = os.fsdecode(path) - for char in "%", "?", "#": - path = path.replace(char, f"%{ord(char):02x}") path = Path(path) - if sys.platform == "win32": - if path.drive: - if not path.is_absolute(): - path = Path(path).absolute() - return "/" + Path(path).as_posix() - return path.as_posix() + uri = path.absolute().as_uri() + while "//" in uri: + uri = uri.replace("//", "/") + return uri class _Database(MutableMapping): @@ -63,8 +59,8 @@ def __init__(self, path, /, flag): f"not {flag!r}") # We use the URI format when opening the database. - path = _normalize_uri_path(path) - uri = f"file:{path}?mode={flag}" + uri = _normalize_uri(path) + uri = f"{uri}?mode={flag}" try: self._cx = sqlite3.connect(uri, autocommit=True, uri=True) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index b307f93fb3753f..5cec499b28e8b4 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -4,12 +4,12 @@ import unittest from contextlib import closing from functools import partial -from pathlib import PureWindowsPath +from pathlib import Path from test.support import cpython_only, import_helper, os_helper dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") -from dbm.sqlite3 import _normalize_uri_path +from dbm.sqlite3 import _normalize_uri class _SQLiteDbmTests(unittest.TestCase): @@ -28,14 +28,14 @@ class URI(unittest.TestCase): def test_uri_substitutions(self): dataset = ( - ("relative/b//c", "relative/b/c"), - ("/absolute/////b/c", "/absolute/b/c"), - ("PRE#MID##END", "PRE%23MID%23%23END"), - ("%#?%%#", "%25%23%3f%25%25%23"), + ("/absolute/////b/c", "file:/absolute/b/c"), + ("/PRE#MID##END", "file:/PRE%23MID%23%23END"), + ("/%#?%%#", "file:/%25%23%3F%25%25%23"), + ("/localhost", "file:/localhost"), ) for path, normalized in dataset: with self.subTest(path=path, normalized=normalized): - self.assertEqual(_normalize_uri_path(path), normalized) + self.assertEqual(_normalize_uri(path), normalized) @unittest.skipUnless(sys.platform == "win32", "requires Windows") def test_uri_windows(self): @@ -55,8 +55,7 @@ def test_uri_windows(self): ) for path, normalized in dataset: with self.subTest(path=path, normalized=normalized): - path = PureWindowsPath(path) - self.assertEqual(_normalize_uri_path(path), normalized) + self.assertEqual(_normalize_uri(path), normalized) class ReadOnly(_SQLiteDbmTests): From 775ff91738fe4ef2a5f8aff998af9df7b58986ee Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 25 Jan 2024 01:27:39 +0100 Subject: [PATCH 21/38] Skip tests with relative paths for now --- Lib/test/test_dbm_sqlite3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 5cec499b28e8b4..772892d2dadfce 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -55,6 +55,8 @@ def test_uri_windows(self): ) for path, normalized in dataset: with self.subTest(path=path, normalized=normalized): + if not Path(path).is_absolute(): + self.skipTest(f"skipping relative path: {path!r}") self.assertEqual(_normalize_uri(path), normalized) From 4ba9607e54c3fe9ab6447aea9e1e3b105598af15 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 25 Jan 2024 01:28:12 +0100 Subject: [PATCH 22/38] Add prelim docstring for dbm.sqlite3.open --- Lib/dbm/sqlite3.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index b3408b7276c5e8..46862bdf645bbb 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -129,5 +129,17 @@ def __exit__(self, *args): self.close() -def open(path, /, flag="r", mode=None): - return _Database(path, flag) +def open(filename, /, flag="r", mode=None): + """Open a dbm.sqlite3 database and return the dbm object. + + The 'filename' parameter is the name of the database file. + + The optional 'flag' parameter can be one of ...: + 'r' (default): open an existing database for read only access + 'w': open an existing database for read/write access + 'c': create a database if it does not exist; open for read/write access + 'n': always create a new, empty database; open for read/write access + + The optional 'mode' parameter is ignored. + """ + return _Database(filename, flag) From dc0ba2685f8b8636eee24503e8a1e45399993b29 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 25 Jan 2024 01:46:11 +0100 Subject: [PATCH 23/38] Amend Windows URI expected results --- Lib/test/test_dbm_sqlite3.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 772892d2dadfce..c6cad1d879e247 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -42,16 +42,16 @@ def test_uri_windows(self): dataset = ( # Relative subdir. (r"2018\January.xlsx", - "2018/January.xlsx"), + "file:/2018/January.xlsx"), # Relative CWD. (r"..\Publications\TravelBrochure.pdf", - "../Publications/TravelBrochure.pdf"), + "file:/../Publications/TravelBrochure.pdf"), # Absolute with drive letter. (r"C:\Projects\apilibrary\apilibrary.sln", - "/C:/Projects/apilibrary/apilibrary.sln"), + "file:/C:/Projects/apilibrary/apilibrary.sln"), # Relative with drive letter. (r"C:Projects\apilibrary\apilibrary.sln", - "/C:Projects/apilibrary/apilibrary.sln"), + "file:/C:Projects/apilibrary/apilibrary.sln"), ) for path, normalized in dataset: with self.subTest(path=path, normalized=normalized): From 37f99abf4c1861a314260fa2efea7217ac9ae897 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 25 Jan 2024 08:55:36 +0100 Subject: [PATCH 24/38] For now, just remove the relative URI tests on Windows --- Lib/test/test_dbm_sqlite3.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index c6cad1d879e247..72c843c3607b33 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -40,23 +40,12 @@ def test_uri_substitutions(self): @unittest.skipUnless(sys.platform == "win32", "requires Windows") def test_uri_windows(self): dataset = ( - # Relative subdir. - (r"2018\January.xlsx", - "file:/2018/January.xlsx"), - # Relative CWD. - (r"..\Publications\TravelBrochure.pdf", - "file:/../Publications/TravelBrochure.pdf"), # Absolute with drive letter. (r"C:\Projects\apilibrary\apilibrary.sln", "file:/C:/Projects/apilibrary/apilibrary.sln"), - # Relative with drive letter. - (r"C:Projects\apilibrary\apilibrary.sln", - "file:/C:Projects/apilibrary/apilibrary.sln"), ) for path, normalized in dataset: with self.subTest(path=path, normalized=normalized): - if not Path(path).is_absolute(): - self.skipTest(f"skipping relative path: {path!r}") self.assertEqual(_normalize_uri(path), normalized) From 177b2005054c70de82fc1de9dcb6da06277ac12e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 25 Jan 2024 08:57:04 +0100 Subject: [PATCH 25/38] Document dbm.sqlite3.open signature --- Doc/library/dbm.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 637a5e74d4be44..6baef7a6c10876 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -159,7 +159,7 @@ SQLite backend for the :mod:`dbm` module. The files created by :mod:`dbm.sqlite3` can thus be opened by :mod:`sqlite3`, or any other SQLite browser, including the SQLite CLI. -.. function:: open(filename, flag="r", mode=None) +.. function:: open(filename, /, flag="r", mode=None) Open an SQLite database. The returned object behaves like a :term:`mapping`, From 1a661a5aca7fde00754e3a640cb84711afbdf114 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 25 Jan 2024 09:02:36 +0100 Subject: [PATCH 26/38] Update Lib/test/test_dbm.py Co-authored-by: Mariusz Felisiak --- Lib/test/test_dbm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index 87ce7932de0eaa..7d32cee4eee994 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -222,7 +222,7 @@ def test_whichdb_ndbm(self): @unittest.skipUnless(dbm_sqlite3, reason='Test requires dbm.sqlite3') def test_whichdb_sqlite3(self): - # Test that databases created by dbm.sqlite3 are detected correctly. + # Databases created by dbm.sqlite3 are detected correctly. with dbm_sqlite3.open(_fname, "c") as db: db["key"] = "value" self.assertEqual(self.dbm.whichdb(_fname), "dbm.sqlite3") From fae860337f46570098017153198e7681b5583939 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 25 Jan 2024 09:04:40 +0100 Subject: [PATCH 27/38] Update Lib/test/test_dbm.py Co-authored-by: Mariusz Felisiak --- Lib/test/test_dbm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index 7d32cee4eee994..4be7c5649da68a 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -229,7 +229,7 @@ def test_whichdb_sqlite3(self): @unittest.skipUnless(dbm_sqlite3, reason='Test requires dbm.sqlite3') def test_whichdb_sqlite3_existing_db(self): - # Test that existing sqlite3 databases are detected correctly. + # Existing sqlite3 databases are detected correctly. sqlite3 = import_helper.import_module("sqlite3") try: # Create an empty database. From d5fc39ce034c45d2d91ac77874be1bae1a25da65 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 25 Jan 2024 09:40:17 +0100 Subject: [PATCH 28/38] Only test URI substitution and normalisation; discard prefix --- Lib/test/test_dbm_sqlite3.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 72c843c3607b33..ab6ee147795308 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -28,25 +28,32 @@ class URI(unittest.TestCase): def test_uri_substitutions(self): dataset = ( - ("/absolute/////b/c", "file:/absolute/b/c"), - ("/PRE#MID##END", "file:/PRE%23MID%23%23END"), - ("/%#?%%#", "file:/%25%23%3F%25%25%23"), - ("/localhost", "file:/localhost"), + ("/absolute/////b/c", "/absolute/b/c"), + ("PRE#MID##END", "PRE%23MID%23%23END"), + ("%#?%%#", "%25%23%3F%25%25%23"), ) for path, normalized in dataset: with self.subTest(path=path, normalized=normalized): - self.assertEqual(_normalize_uri(path), normalized) + self.assertTrue(_normalize_uri(path).endswith(normalized)) @unittest.skipUnless(sys.platform == "win32", "requires Windows") def test_uri_windows(self): dataset = ( + # Relative subdir. + (r"2018\January.xlsx", + "2018/January.xlsx"), # Absolute with drive letter. (r"C:\Projects\apilibrary\apilibrary.sln", - "file:/C:/Projects/apilibrary/apilibrary.sln"), + "/C:/Projects/apilibrary/apilibrary.sln"), + # Relative with drive letter. + (r"C:Projects\apilibrary\apilibrary.sln", + "/C:Projects/apilibrary/apilibrary.sln"), ) for path, normalized in dataset: with self.subTest(path=path, normalized=normalized): - self.assertEqual(_normalize_uri(path), normalized) + if not Path(path).is_absolute(): + self.skipTest(f"skipping relative path: {path!r}") + self.assertTrue(_normalize_uri(path).endswith(normalized)) class ReadOnly(_SQLiteDbmTests): From a8db17fb1952e76f9ddf936d4cf650694dc6235e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 25 Jan 2024 11:44:05 +0100 Subject: [PATCH 29/38] Doc amendments --- Doc/library/dbm.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 6baef7a6c10876..633adf7cc936a5 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -30,8 +30,8 @@ the Oracle Berkeley DB. .. function:: whichdb(filename) This function attempts to guess which of the several simple database modules - available --- :mod:`dbm.gnu`, :mod:`dbm.ndbm` or :mod:`dbm.dumb` --- should - be used to open a given file. + available --- :mod:`dbm.sqlite3`, :mod:`dbm.gnu`, :mod:`dbm.ndbm`, + or :mod:`dbm.dumb` --- should be used to open a given file. Returns one of the following values: ``None`` if the file can't be opened because it's unreadable or doesn't exist; the empty string (``''``) if the @@ -142,7 +142,7 @@ then prints out the contents of the database:: The individual submodules are described in the following sections. :mod:`dbm.sqlite3` --- SQLite backend for dbm ----------------------------------------------- +--------------------------------------------- .. module:: dbm.sqlite3 :platform: All @@ -154,7 +154,7 @@ The individual submodules are described in the following sections. -------------- -This module uses the standard library :mod:`sqlite3` to provide an +This module uses the standard library :mod:`sqlite3` module to provide an SQLite backend for the :mod:`dbm` module. The files created by :mod:`dbm.sqlite3` can thus be opened by :mod:`sqlite3`, or any other SQLite browser, including the SQLite CLI. From bb49fab656542723f8c9ba1db2de0b79697eb8f9 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 26 Jan 2024 14:45:24 +0100 Subject: [PATCH 30/38] Mark up flags as list; it renders better --- Doc/library/dbm.rst | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index aa5f7481778805..9680a4fad8c7c2 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -179,14 +179,10 @@ or any other SQLite browser, including the SQLite CLI. :type filename: :term:`path-like object` :param str flag: - - .. csv-table:: - :header: "Value", "Meaning" - - ``'r'`` (default), |flag_r| - ``'w'``, |flag_w| - ``'c'``, |flag_c| - ``'n'``, |flag_n| + * ``'r'`` (default), |flag_r| + * ``'w'``, |flag_w| + * ``'c'``, |flag_c| + * ``'n'``, |flag_n| :param mode: This parameter is ignored by the :mod:`!dbm.sqlite3` module. From 26a2c69ac0b2a302918d9b0378ecb2bad54bea0a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 26 Jan 2024 15:24:38 +0100 Subject: [PATCH 31/38] Apply suggestions from code review Co-authored-by: Serhiy Storchaka --- Lib/dbm/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py index 763135bec40ee9..97c0bb1c9ca946 100644 --- a/Lib/dbm/__init__.py +++ b/Lib/dbm/__init__.py @@ -5,7 +5,7 @@ import dbm d = dbm.open(file, 'w', 0o666) -The returned object is a dbm.sqlite3, dbm.gnu, dbm.ndbm or dbm.dumb object, dependent on the +The returned object is a dbm.sqlite3, dbm.gnu, dbm.ndbm or dbm.dumb database object, dependent on the type of database being opened (determined by the whichdb function) in the case of an existing dbm. If the dbm does not exist and the create or new flag ('c' or 'n') was specified, the dbm type will be determined by the availability of @@ -38,7 +38,7 @@ class error(Exception): pass -_names = ['dbm.sqlite3', 'dbm.gnu', 'dbm.ndbm', 'dbm.dumb'] +_names = ['dbm.gnu', 'dbm.ndbm', 'dbm.sqlite3', 'dbm.dumb'] _defaultmod = None _modules = {} From ae53f95f904bef09c841b521db7b607cf193ea53 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 26 Jan 2024 15:32:02 +0100 Subject: [PATCH 32/38] Address review: use closing() in _execute() wrapper --- Lib/dbm/sqlite3.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 46862bdf645bbb..a7e14bcdf66c51 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -72,43 +72,39 @@ def __init__(self, path, /, flag): self._cx.execute("PRAGMA journal_mode = wal") if flag == "rwc": - with closing(self._execute(BUILD_TABLE)): - pass + self._execute(BUILD_TABLE) def _execute(self, *args, **kwargs): if not self._cx: raise error(_ERR_CLOSED) try: - ret = self._cx.execute(*args, **kwargs) + return closing(self._cx.execute(*args, **kwargs)) except sqlite3.Error as exc: raise error(str(exc)) - else: - return ret def __len__(self): - with closing(self._execute(GET_SIZE)) as cu: + with self._execute(GET_SIZE) as cu: row = cu.fetchone() return row[0] def __getitem__(self, key): - with closing(self._execute(LOOKUP_KEY, (key,))) as cu: + with self._execute(LOOKUP_KEY, (key,)) as cu: row = cu.fetchone() if not row: raise KeyError(key) return row[0] def __setitem__(self, key, value): - with closing(self._execute(STORE_KV, (key, value))): - pass + self._execute(STORE_KV, (key, value)) def __delitem__(self, key): - with closing(self._execute(DELETE_KEY, (key,))) as cu: + with self._execute(DELETE_KEY, (key,)) as cu: if not cu.rowcount: raise KeyError(key) def __iter__(self): try: - with closing(self._execute(ITER_KEYS)) as cu: + with self._execute(ITER_KEYS) as cu: for row in cu: yield row[0] except sqlite3.Error as exc: From 926ef1a9086c2baaaa306c19194e24a593c5cb2c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 26 Jan 2024 16:15:45 +0100 Subject: [PATCH 33/38] Update Doc/library/dbm.rst --- Doc/library/dbm.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 9680a4fad8c7c2..80c5c969acefb1 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -179,6 +179,7 @@ or any other SQLite browser, including the SQLite CLI. :type filename: :term:`path-like object` :param str flag: + * ``'r'`` (default), |flag_r| * ``'w'``, |flag_w| * ``'c'``, |flag_c| From c0878d9c77e2e5890deaf332952f3b3b150eeb21 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 30 Jan 2024 23:23:21 +0100 Subject: [PATCH 34/38] Add support for 'mode' param in dbm.sqlite3.open() --- Doc/library/dbm.rst | 13 +++++++------ Lib/dbm/sqlite3.py | 18 +++++++++--------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 80c5c969acefb1..37ced603a13f3a 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -164,7 +164,7 @@ SQLite backend for the :mod:`dbm` module. The files created by :mod:`dbm.sqlite3` can thus be opened by :mod:`sqlite3`, or any other SQLite browser, including the SQLite CLI. -.. function:: open(filename, /, flag="r", mode=None) +.. function:: open(filename, /, flag="r", mode=0o666) Open an SQLite database. The returned object behaves like a :term:`mapping`, @@ -180,13 +180,14 @@ or any other SQLite browser, including the SQLite CLI. :param str flag: - * ``'r'`` (default), |flag_r| - * ``'w'``, |flag_w| - * ``'c'``, |flag_c| - * ``'n'``, |flag_n| + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| :param mode: - This parameter is ignored by the :mod:`!dbm.sqlite3` module. + The Unix file access mode of the file (default: octal ``0o666``), + used only when the database has to be created. :mod:`dbm.gnu` --- GNU database manager diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index a7e14bcdf66c51..7265d3098cddf7 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -27,7 +27,6 @@ class error(OSError): def _normalize_uri(path): - path = os.fsdecode(path) path = Path(path) uri = path.absolute().as_uri() while "//" in uri: @@ -37,10 +36,11 @@ def _normalize_uri(path): class _Database(MutableMapping): - def __init__(self, path, /, flag): + def __init__(self, path, /, *, flag, mode): if hasattr(self, "_cx"): raise error(_ERR_REINIT) + path = os.fsdecode(path) match flag: case "r": flag = "ro" @@ -48,12 +48,11 @@ def __init__(self, path, /, flag): flag = "rw" case "c": flag = "rwc" + Path(path).touch(mode=mode, exist_ok=True) case "n": flag = "rwc" - try: - os.remove(path) - except FileNotFoundError: - pass + Path(path).unlink(missing_ok=True) + Path(path).touch(mode=mode) case _: raise ValueError("Flag must be one of 'r', 'w', 'c', or 'n', " f"not {flag!r}") @@ -125,7 +124,7 @@ def __exit__(self, *args): self.close() -def open(filename, /, flag="r", mode=None): +def open(filename, /, flag="r", mode=0o666): """Open a dbm.sqlite3 database and return the dbm object. The 'filename' parameter is the name of the database file. @@ -136,6 +135,7 @@ def open(filename, /, flag="r", mode=None): 'c': create a database if it does not exist; open for read/write access 'n': always create a new, empty database; open for read/write access - The optional 'mode' parameter is ignored. + The optional 'mode' parameter is the Unix file access mode of the database; + only used when creating a new database. Default: 0o666. """ - return _Database(filename, flag) + return _Database(filename, flag=flag, mode=mode) From d6d7c667be24465925ac5b1f8810872d32a22d7b Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 31 Jan 2024 12:14:03 +0100 Subject: [PATCH 35/38] Fix test_misuse_reinit() --- Lib/test/test_dbm_sqlite3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index ab6ee147795308..4cad84a54c6016 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -198,7 +198,7 @@ def test_misuse_use_after_close(self): def test_misuse_reinit(self): with self.assertRaises(dbm_sqlite3.error): - self.db.__init__("new.db", flag="rw") + self.db.__init__("new.db", flag="n", mode=0o666) def test_misuse_empty_filename(self): for flag in "r", "w", "c", "n": From e782fad38fa032cc2eb3535dd5a9c1f19488353e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 14 Feb 2024 11:27:30 +0100 Subject: [PATCH 36/38] Address Serhiy's offline remark: coerce keys/values to bytes --- Lib/dbm/sqlite3.py | 4 +- Lib/test/test_dbm_sqlite3.py | 75 ++++++++++++++++++++---------------- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 7265d3098cddf7..4cfda81cbffdb1 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -7,13 +7,13 @@ BUILD_TABLE = """ CREATE TABLE IF NOT EXISTS Dict ( - key TEXT UNIQUE NOT NULL, + key BLOB UNIQUE NOT NULL, value BLOB NOT NULL ) """ GET_SIZE = "SELECT COUNT (key) FROM Dict" LOOKUP_KEY = "SELECT value FROM Dict WHERE key = ?" -STORE_KV = "REPLACE INTO Dict (key, value) VALUES (?, ?)" +STORE_KV = "REPLACE INTO Dict (key, value) VALUES (CAST(? AS BLOB), CAST(? AS BLOB))" DELETE_KEY = "DELETE FROM Dict WHERE key = ?" ITER_KEYS = "SELECT key FROM Dict" diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 4cad84a54c6016..ca5d85daf9e668 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -61,8 +61,8 @@ class ReadOnly(_SQLiteDbmTests): def setUp(self): super().setUp() with dbm_sqlite3.open(self.filename, "w") as db: - db["key1"] = "value1" - db["key2"] = "value2" + db[b"key1"] = "value1" + db[b"key2"] = "value2" self.db = dbm_sqlite3.open(self.filename, "r") def tearDown(self): @@ -70,22 +70,22 @@ def tearDown(self): super().tearDown() def test_readonly_read(self): - self.assertEqual(self.db["key1"], "value1") - self.assertEqual(self.db["key2"], "value2") + self.assertEqual(self.db[b"key1"], b"value1") + self.assertEqual(self.db[b"key2"], b"value2") def test_readonly_write(self): with self.assertRaises(dbm_sqlite3.error): - self.db["new"] = "value" + self.db[b"new"] = "value" def test_readonly_delete(self): with self.assertRaises(dbm_sqlite3.error): - del self.db["key1"] + del self.db[b"key1"] def test_readonly_keys(self): - self.assertEqual(self.db.keys(), ["key1", "key2"]) + self.assertEqual(self.db.keys(), [b"key1", b"key2"]) def test_readonly_iter(self): - self.assertEqual([k for k in self.db], ["key1", "key2"]) + self.assertEqual([k for k in self.db], [b"key1", b"key2"]) class ReadWrite(_SQLiteDbmTests): @@ -108,19 +108,19 @@ def test_readwrite_unique_key(self): self.db["key"] = "value" self.db["key"] = "other" keys, vals = self.db_content() - self.assertEqual(keys, ["key"]) - self.assertEqual(vals, ["other"]) + self.assertEqual(keys, [b"key"]) + self.assertEqual(vals, [b"other"]) def test_readwrite_delete(self): self.db["key"] = "value" self.db["new"] = "other" - del self.db["new"] + del self.db[b"new"] keys, vals = self.db_content() - self.assertEqual(keys, ["key"]) - self.assertEqual(vals, ["value"]) + self.assertEqual(keys, [b"key"]) + self.assertEqual(vals, [b"value"]) - del self.db["key"] + del self.db[b"key"] keys, vals = self.db_content() self.assertEqual(keys, []) self.assertEqual(vals, []) @@ -131,7 +131,7 @@ def test_readwrite_null_key(self): def test_readwrite_null_value(self): with self.assertRaises(dbm_sqlite3.error): - self.db["key"] = None + self.db[b"key"] = None class Misuse(_SQLiteDbmTests): @@ -147,7 +147,7 @@ def tearDown(self): def test_misuse_double_create(self): self.db["key"] = "value" with dbm_sqlite3.open(self.filename, "c") as db: - self.assertEqual(db["key"], "value") + self.assertEqual(db[b"key"], b"value") def test_misuse_double_close(self): self.db.close() @@ -159,13 +159,13 @@ def test_misuse_invalid_flag(self): def test_misuse_double_delete(self): self.db["key"] = "value" - del self.db["key"] + del self.db[b"key"] with self.assertRaises(KeyError): - del self.db["key"] + del self.db[b"key"] def test_misuse_invalid_key(self): with self.assertRaises(KeyError): - self.db["key"] + self.db[b"key"] def test_misuse_iter_close1(self): self.db["1"] = 1 @@ -186,11 +186,11 @@ def test_misuse_iter_close2(self): def test_misuse_use_after_close(self): self.db.close() with self.assertRaises(dbm_sqlite3.error): - self.db["read"] + self.db[b"read"] with self.assertRaises(dbm_sqlite3.error): - self.db["write"] = "value" + self.db[b"write"] = "value" with self.assertRaises(dbm_sqlite3.error): - del self.db["del"] + del self.db[b"del"] with self.assertRaises(dbm_sqlite3.error): len(self.db) with self.assertRaises(dbm_sqlite3.error): @@ -208,7 +208,12 @@ def test_misuse_empty_filename(self): class DataTypes(_SQLiteDbmTests): - dataset = 10, 2.5, "string", b"bytes" + dataset = ( + # (raw, coerced) + (42, b"42"), + (3.14, b"3.14"), + ("string", b"string"), + ) def setUp(self): super().setUp() @@ -219,16 +224,20 @@ def tearDown(self): super().tearDown() def test_datatypes_values(self): - for value in self.dataset: - with self.subTest(value=value): - self.db["key"] = value - self.assertEqual(self.db["key"], value) + for raw, coerced in self.dataset: + with self.subTest(raw=raw, coerced=coerced): + self.db["key"] = raw + self.assertEqual(self.db[b"key"], coerced) def test_datatypes_keys(self): - for key in self.dataset: - with self.subTest(key=key): - self.db[key] = "value" - self.assertEqual(self.db[key], "value") + for raw, coerced in self.dataset: + with self.subTest(raw=raw, coerced=coerced): + self.db[raw] = "value" + self.assertEqual(self.db[coerced], b"value") + with self.assertRaises(KeyError): + self.db[raw] + with self.assertRaises(KeyError): + del self.db[raw] class CorruptDatabase(_SQLiteDbmTests): @@ -284,9 +293,9 @@ def test_corrupt_readwrite(self): def test_corrupt_force_new(self): with closing(dbm_sqlite3.open(self.filename, "n")) as db: db["foo"] = "write" - _ = db["foo"] + _ = db[b"foo"] next(iter(db)) - del db["foo"] + del db[b"foo"] if __name__ == "__main__": From b1b9a9bc791e90feeab667b747ff0ca5f3d36d08 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 14 Feb 2024 11:29:33 +0100 Subject: [PATCH 37/38] Align docs to e782fad38f --- Doc/library/dbm.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index edb8eafdfed4e6..0f9c825fec9385 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -174,9 +174,6 @@ or any other SQLite browser, including the SQLite CLI. implements a :meth:`!close` method, and supports a "closing" context manager via the :keyword:`with` keyword. - Neither keys nor values are coerced to a specific type - before being stored in the database. - :param filename: The path to the database to be opened. :type filename: :term:`path-like object` From 34930cb9fdc70eaa51548e4140d62c7444c85992 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 14 Feb 2024 11:50:11 +0100 Subject: [PATCH 38/38] Compat with other backends: silently coerce keys to bytes --- Lib/dbm/sqlite3.py | 4 ++-- Lib/test/test_dbm_sqlite3.py | 14 ++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 4cfda81cbffdb1..74c9d9b7e2f1d8 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -12,9 +12,9 @@ ) """ GET_SIZE = "SELECT COUNT (key) FROM Dict" -LOOKUP_KEY = "SELECT value FROM Dict WHERE key = ?" +LOOKUP_KEY = "SELECT value FROM Dict WHERE key = CAST(? AS BLOB)" STORE_KV = "REPLACE INTO Dict (key, value) VALUES (CAST(? AS BLOB), CAST(? AS BLOB))" -DELETE_KEY = "DELETE FROM Dict WHERE key = ?" +DELETE_KEY = "DELETE FROM Dict WHERE key = CAST(? AS BLOB)" ITER_KEYS = "SELECT key FROM Dict" diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index ca5d85daf9e668..7bc2a030352835 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -213,6 +213,7 @@ class DataTypes(_SQLiteDbmTests): (42, b"42"), (3.14, b"3.14"), ("string", b"string"), + (b"bytes", b"bytes"), ) def setUp(self): @@ -234,10 +235,15 @@ def test_datatypes_keys(self): with self.subTest(raw=raw, coerced=coerced): self.db[raw] = "value" self.assertEqual(self.db[coerced], b"value") - with self.assertRaises(KeyError): - self.db[raw] - with self.assertRaises(KeyError): - del self.db[raw] + # Raw keys are silently coerced to bytes. + self.assertEqual(self.db[raw], b"value") + del self.db[raw] + + def test_datatypes_replace_coerced(self): + self.db["10"] = "value" + self.db[b"10"] = "value" + self.db[10] = "value" + self.assertEqual(self.db.keys(), [b"10"]) class CorruptDatabase(_SQLiteDbmTests): pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy