Skip to content

Commit b3c5abb

Browse files
tacerustacerus
committed
dictdiffer: deep diff of dicts in lists
So far, lists would always be compared as a whole - in case of lists containing dicts as elements, this would cause lots of output even if only parts of the contained dicts changed. Allow matching of lists with consistently keyed dicts. We do have listdiffer, but custom logic was implemented as importing listdiffer in dictdiffer would cause a circular import. Signed-off-by: Georg Pfuetzenreuter <georg.pfuetzenreuter@suse.com>
1 parent d9cecbd commit b3c5abb

2 files changed

Lines changed: 227 additions & 14 deletions

File tree

‎salt/utils/dictdiffer.py‎

Lines changed: 83 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,9 @@ def deep_diff(old, new, ignore=None):
7979
return res
8080

8181

82-
def recursive_diff(past_dict, current_dict, ignore_missing_keys=True):
82+
def recursive_diff(
83+
past_dict, current_dict, ignore_missing_keys=True, list_dict_matchers=[]
84+
):
8385
"""
8486
Returns a RecursiveDictDiffer object that computes the recursive diffs
8587
between two dictionaries
@@ -95,8 +97,19 @@ def recursive_diff(past_dict, current_dict, ignore_missing_keys=True):
9597
current_dict, but exist in the past_dict. If true, the diff will
9698
not contain the missing keys.
9799
Default is True.
100+
101+
list_dict_matchers
102+
List of keys to consider for deep comparison of dicts inside a list.
103+
If not specified or if not all of the dicts contained in a list are
104+
matchable with one of these keys, changes to dicts in such a list will
105+
return the two differing lists as a whole instead of only the differing
106+
dict elements.
107+
Empty by default, meaning lists of dicts will not be diffed deeply.
108+
98109
"""
99-
return RecursiveDictDiffer(past_dict, current_dict, ignore_missing_keys)
110+
return RecursiveDictDiffer(
111+
past_dict, current_dict, ignore_missing_keys, list_dict_matchers
112+
)
100113

101114

102115
class RecursiveDictDiffer(DictDiffer):
@@ -142,7 +155,9 @@ class RecursiveDictDiffer(DictDiffer):
142155

143156
NONE_VALUE = "<_null_>"
144157

145-
def __init__(self, past_dict, current_dict, ignore_missing_keys):
158+
def __init__(
159+
self, past_dict, current_dict, ignore_missing_keys, list_dict_matchers=[]
160+
):
146161
"""
147162
past_dict
148163
Past dictionary.
@@ -154,37 +169,94 @@ def __init__(self, past_dict, current_dict, ignore_missing_keys):
154169
Flag specifying whether to ignore keys that no longer exist in the
155170
current_dict, but exist in the past_dict. If true, the diff will
156171
not contain the missing keys.
172+
173+
list_dict_matchers
174+
List of keys to consider for deep comparison of dicts inside a list.
175+
If not specified or if not all of the dicts contained in a list are
176+
matchable with one of these keys, changes to dicts in such a list will
177+
return the two differing lists as a whole instead of only the differing
178+
dict elements.
157179
"""
158180
super().__init__(current_dict, past_dict)
159181
self._diffs = self._get_diffs(
160-
self.current_dict, self.past_dict, ignore_missing_keys
182+
self.current_dict, self.past_dict, ignore_missing_keys, list_dict_matchers
161183
)
162184
# Ignores unet values when assessing the changes
163185
self.ignore_unset_values = True
164186

165187
@classmethod
166-
def _get_diffs(cls, dict1, dict2, ignore_missing_keys):
188+
def _get_diffs(cls, dict1, dict2, ignore_missing_keys, list_dict_matchers):
167189
"""
168190
Returns a dict with the differences between dict1 and dict2
169191
170192
Notes:
171-
Keys that only exist in dict2 are not included in the diff if
172-
ignore_missing_keys is True, otherwise they are
173-
Simple compares are done on lists
193+
- Keys that only exist in dict2 are not included in the diff if
194+
ignore_missing_keys is True, otherwise they are
195+
- Simple compares are done on lists, unless the list contains dicts
196+
and all contained dicts are matchable with a key listed in list_dict_matchers
174197
"""
175198
ret_dict = {}
199+
176200
for p in dict1:
177201
if p not in dict2:
178202
ret_dict.update({p: {"new": dict1[p], "old": cls.NONE_VALUE}})
179203
elif dict1[p] != dict2[p]:
180204
if isinstance(dict1[p], dict) and isinstance(dict2[p], dict):
181205
sub_diff_dict = cls._get_diffs(
182-
dict1[p], dict2[p], ignore_missing_keys
206+
dict1[p], dict2[p], ignore_missing_keys, list_dict_matchers
183207
)
184208
if sub_diff_dict:
185209
ret_dict.update({p: sub_diff_dict})
186-
else:
187-
ret_dict.update({p: {"new": dict1[p], "old": dict2[p]}})
210+
211+
continue
212+
213+
elif (
214+
list_dict_matchers
215+
and isinstance(dict1[p], list)
216+
and isinstance(dict2[p], list)
217+
and isinstance(dict1[p][0], dict)
218+
and isinstance(dict2[p][0], dict)
219+
):
220+
221+
match_key = None
222+
match_results = {}
223+
224+
for d1 in dict1[p]:
225+
if match_key is None:
226+
for matcher in list_dict_matchers:
227+
if matcher in d1:
228+
match_key = matcher
229+
230+
for d2 in dict2[p]:
231+
if match_key not in d2:
232+
match_key = None
233+
break
234+
235+
if d1[match_key] == d2[match_key]:
236+
sub_diff_dict = cls._get_diffs(
237+
d1, d2, ignore_missing_keys, list_dict_matchers
238+
)
239+
if sub_diff_dict:
240+
match_results.update(
241+
{p: {d1[match_key]: sub_diff_dict}}
242+
)
243+
244+
break
245+
else:
246+
if match_key is not None:
247+
continue
248+
249+
if match_key is None:
250+
break
251+
252+
if match_results:
253+
ret_dict.update(match_results)
254+
255+
if match_key is not None:
256+
continue
257+
258+
ret_dict.update({p: {"new": dict1[p], "old": dict2[p]}})
259+
188260
if not ignore_missing_keys:
189261
for p in dict2:
190262
if p not in dict1:

‎tests/pytests/unit/utils/test_dictdiffer.py‎

Lines changed: 144 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,19 @@
88

99
@pytest.fixture
1010
def differ(request):
11-
old, new, *ignore_missing = request.param
11+
old, new, *extra = request.param
12+
1213
try:
13-
ignore_missing = bool(ignore_missing.pop(0))
14+
ignore_missing = bool(extra.pop(0))
1415
except IndexError:
1516
ignore_missing = False
16-
return dictdiffer.RecursiveDictDiffer(old, new, ignore_missing)
17+
18+
try:
19+
list_dict_matchers = list(extra.pop(0))
20+
except IndexError:
21+
list_dict_matchers = []
22+
23+
return dictdiffer.RecursiveDictDiffer(old, new, ignore_missing, list_dict_matchers)
1724

1825

1926
@pytest.mark.parametrize("separator", [None, ":"])
@@ -219,6 +226,140 @@ def test_unchanged(differ, expected, separator):
219226
{"b": {"old": {"c": "c"}, "new": NONE}},
220227
),
221228
(({"a": "a", "b": {"c": "c"}}, {"a": "a"}, IGNORE_MISSING), {}),
229+
(
230+
# list of dicts with single matcher key, expect deep diff
231+
(
232+
{
233+
"a": {
234+
"b": [
235+
{"name": "sub1", "foo": "bar"},
236+
{"name": "sub2", "x": "x"},
237+
]
238+
}
239+
},
240+
{
241+
"a": {
242+
"b": [
243+
{"name": "sub1", "foo": "baz"},
244+
{"name": "sub2", "x": "x"},
245+
]
246+
}
247+
},
248+
False,
249+
["name"],
250+
),
251+
{"a": {"b": {"sub1": {"foo": {"old": "bar", "new": "baz"}}}}},
252+
),
253+
(
254+
# list of dicts with multiple matcher keys, expect deep diff
255+
(
256+
{
257+
"a": {
258+
"b": [{"name": "sub1", "foo": "bar"}],
259+
"c": [{"type": "sub2", "x": "y"}],
260+
}
261+
},
262+
{
263+
"a": {
264+
"b": [{"name": "sub1", "foo": "baz"}],
265+
"c": [{"type": "sub2", "x": "z"}],
266+
}
267+
},
268+
False,
269+
["name", "type"],
270+
),
271+
{
272+
"a": {
273+
"b": {"sub1": {"foo": {"old": "bar", "new": "baz"}}},
274+
"c": {"sub2": {"x": {"old": "y", "new": "z"}}},
275+
}
276+
},
277+
),
278+
(
279+
# identical list of dicts, with matcher key, expect no changes
280+
(
281+
{
282+
"a": {
283+
"b": [
284+
{"name": "sub1", "foo": "bar"},
285+
{"name": "sub2", "x": "x"},
286+
]
287+
}
288+
},
289+
{
290+
"a": {
291+
"b": [
292+
{"name": "sub1", "foo": "bar"},
293+
{"name": "sub2", "x": "x"},
294+
]
295+
}
296+
},
297+
False,
298+
["name"],
299+
),
300+
{},
301+
),
302+
(
303+
# identical list of dicts, without matcher key, expect no changes
304+
(
305+
{
306+
"a": {
307+
"b": [
308+
{"name": "sub1", "foo": "bar"},
309+
{"name": "sub2", "x": "x"},
310+
]
311+
}
312+
},
313+
{
314+
"a": {
315+
"b": [
316+
{"name": "sub1", "foo": "bar"},
317+
{"name": "sub2", "x": "x"},
318+
]
319+
}
320+
},
321+
False,
322+
[],
323+
),
324+
{},
325+
),
326+
(
327+
# matcher key "name" does not exist in second list of dicts, expect fallback to simple list diff
328+
(
329+
{
330+
"a": {
331+
"b": [
332+
{"name": "sub1", "foo": "bar"},
333+
{"name": "sub2", "x": "y"},
334+
]
335+
}
336+
},
337+
{
338+
"a": {
339+
"b": [
340+
{"noname": "sub1", "foo": "baz"},
341+
{"noname": "sub2", "x": "x"},
342+
]
343+
}
344+
},
345+
False,
346+
["name"],
347+
),
348+
{
349+
"a": {
350+
"b": {
351+
"old": [
352+
{"name": "sub1", "foo": "bar"},
353+
{"name": "sub2", "x": "y"},
354+
],
355+
"new": [
356+
{"noname": "sub1", "foo": "baz"},
357+
{"noname": "sub2", "x": "x"},
358+
],
359+
}
360+
}
361+
},
362+
),
222363
],
223364
indirect=["differ"],
224365
)

0 commit comments

Comments
 (0)