From d05c9abd19e4221d72efb4eeb40759e7abb68555 Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Thu, 14 Mar 2024 16:07:28 -0500 Subject: [PATCH 1/3] PersistentDict: compress pickled data --- pytools/persistent_dict.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pytools/persistent_dict.py b/pytools/persistent_dict.py index ed1e8031..8e09dd1a 100644 --- a/pytools/persistent_dict.py +++ b/pytools/persistent_dict.py @@ -582,14 +582,20 @@ def fetch(self, key, _stacklevel=0): @staticmethod def _read(path): + import lzma from pickle import load - with open(path, "rb") as inf: - return load(inf) + try: + with lzma.open(path, "rb") as inf: + return load(inf) + except lzma.LZMAError: + with open(path, "rb") as inf: + return load(inf) @staticmethod def _write(path, value): + import lzma from pickle import HIGHEST_PROTOCOL, dump - with open(path, "wb") as outf: + with lzma.open(path, "wb") as outf: dump(value, outf, protocol=HIGHEST_PROTOCOL) def _item_dir(self, hexdigest_key): From 32093892cef0162d31f1265c28697b8a035b8203 Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Mon, 18 Mar 2024 20:57:54 -0500 Subject: [PATCH 2/3] flatten dir structure --- pytools/persistent_dict.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pytools/persistent_dict.py b/pytools/persistent_dict.py index 8e09dd1a..78f0ad22 100644 --- a/pytools/persistent_dict.py +++ b/pytools/persistent_dict.py @@ -606,9 +606,8 @@ def _item_dir(self, hexdigest_key): # This doesn't solve that problem, but it makes it much less likely return join(self.container_dir, - hexdigest_key[:3], - hexdigest_key[3:6], - hexdigest_key[6:]) + hexdigest_key[:1], + hexdigest_key[1:]) def _key_file(self, hexdigest_key): from os.path import join From 5308f332b49c6c2aa27641f1e30c365f8c6a6458 Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Tue, 19 Mar 2024 11:38:30 -0500 Subject: [PATCH 3/3] bump version --- pytools/persistent_dict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytools/persistent_dict.py b/pytools/persistent_dict.py index 78f0ad22..887c886d 100644 --- a/pytools/persistent_dict.py +++ b/pytools/persistent_dict.py @@ -558,7 +558,7 @@ def __init__(self, identifier, key_builder=None, container_dir=None): container_dir = join( appdirs.user_cache_dir("pytools", "pytools"), - "pdict-v4-{}-py{}".format( + "pdict-v5-{}-py{}".format( identifier, ".".join(str(i) for i in sys.version_info)))