jay_diff: fix two correctness bugs + add real-data round-trip tests
Bug 1 (crash): jay_merge_delete iterated _new.items() while _old (which
aliases _new sub-dicts due to jay_diff_rev's side-effect writes) was being
mutated. Fixed by snapshotting with list(_new.items()) before iteration.
Bug 2 (wrong round-trip): When new has an empty dict {} at a location where
old had a non-empty dict, jay_merge_delete incorrectly pruned the now-empty
parent key. Root cause: jay_diff_del stored the full old sub-dict in the
delete delta for both "key absent from new" and "key present but empty in
new", making them indistinguishable at merge time.
Fix: jay_diff_del is now a standalone recursive function that stores None
for dict keys entirely absent from new (sentinel for "delete entire key")
vs a sub-dict of specific sub-keys for partial deletions. jay_merge_delete
no longer prunes empty dicts; None (or any non-dict value) in the delta is
the explicit signal to remove the parent key.
Also adds:
- tests/data/ — copy of 25 real JSON log files for integration testing
- tests/test_jay_diff_real_data.py — 2028 parametrised round-trip tests
over 675 real vehicle snapshots; exposed and confirmed both bugs above
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.6
+34
-28
@@ -25,8 +25,16 @@ def jay_diff_add(_old, _new):
|
||||
|
||||
|
||||
def jay_diff_del(_old, _new):
|
||||
__new = deepcopy(_new)
|
||||
delta = jay_diff_rev(__new, _old)
|
||||
delta = {}
|
||||
for k in _old:
|
||||
if k in _new:
|
||||
if isinstance(_old[k], dict) and isinstance(_new[k], dict):
|
||||
sub = jay_diff_del(_old[k], _new[k])
|
||||
if sub:
|
||||
delta[k] = sub
|
||||
else:
|
||||
# key absent from _new: None signals "delete entire key" for dicts
|
||||
delta[k] = None if isinstance(_old[k], dict) else _old[k]
|
||||
return delta
|
||||
|
||||
|
||||
@@ -100,15 +108,13 @@ def jay_merge_add(_old, _new):
|
||||
|
||||
|
||||
def jay_merge_delete(_old, _new):
|
||||
for k, v in _new.items():
|
||||
for k, v in list(_new.items()):
|
||||
if isinstance(v, dict) and v:
|
||||
jay_merge_delete(_old[k], _new[k])
|
||||
if len(_old[k].keys()) == 0:
|
||||
del _old[k]
|
||||
if k in _old and isinstance(_old[k], dict):
|
||||
jay_merge_delete(_old[k], v)
|
||||
else:
|
||||
if k in _old:
|
||||
del _old[k]
|
||||
|
||||
return _old
|
||||
|
||||
|
||||
@@ -171,7 +177,7 @@ def _legacy_assertions() -> None:
|
||||
assert result == {}
|
||||
|
||||
result = jay_diff_del({'a': 'hamburger', 'b': 'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a': 'hamburger', 'b': 'fries'})
|
||||
assert result == {'c': {'e': 'coke', 'f': 'coffee'}}
|
||||
assert result == {'c': None}
|
||||
|
||||
result = jay_diff_del({'a': 'hamburger', 'b': 'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'c': {'e': 'coke'}})
|
||||
assert result == {'a':'hamburger', 'b':'fries', 'c': {'f': 'coffee'}}
|
||||
@@ -275,7 +281,7 @@ def _legacy_assertions() -> None:
|
||||
assert result == {'update': {'c': 'coke'}}
|
||||
|
||||
result = jay_diff_full({'a': 'hamburger', 'b': 'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a': 'hamburger', 'b': 'fries'}, combine_upd_add)
|
||||
assert result == {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}}
|
||||
assert result == {'delete': {'c': None}}
|
||||
|
||||
result = jay_diff_full({'a': 'hamburger', 'b': 'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a': 'hamburger', 'b': 'fries', 'c': {'f': 'coffee'}}, combine_upd_add)
|
||||
assert result == {'delete': {'c': {'e': 'coke'}}}
|
||||
@@ -284,25 +290,25 @@ def _legacy_assertions() -> None:
|
||||
assert result == {'delete': {'a': 'hamburger', 'b': 'fries', 'c': {'f': 'coffee'}}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a':'hotdog', 'b':'potatoes'}, combine_upd_add)
|
||||
assert result == {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update': {'a': 'hotdog', 'b': 'potatoes'}}
|
||||
assert result == {'delete': {'c': None}, 'update': {'a': 'hotdog', 'b': 'potatoes'}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'c': {'e': 'pepsi'}}, combine_upd_add)
|
||||
assert result == {'delete': {'a': 'hamburger', 'b': 'fries', 'c': {'f': 'coffee'}}, 'update': {'c': {'e': 'pepsi'}}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a':'hotdog', 'b':'fries'}, combine_upd_add)
|
||||
assert result == {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update': {'a': 'hotdog'}}
|
||||
assert result == {'delete': {'c': None}, 'update': {'a': 'hotdog'}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a':'hamburger', 'b':'fries', 'c': {'e': 'pepsi', 'f': 'coffee'}}, combine_upd_add)
|
||||
assert result == {'update': {'c': {'e': 'pepsi'}}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a':'hotdog', 'b':'potatoes', 'd': 'icecream'}, combine_upd_add)
|
||||
assert result == {'add': {'d': 'icecream'}, 'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update': {'a': 'hotdog', 'b': 'potatoes'}}
|
||||
assert result == {'add': {'d': 'icecream'}, 'delete': {'c': None}, 'update': {'a': 'hotdog', 'b': 'potatoes'}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'c': {'e': 'pepsi', 'g': 'tea', 'h': 'cake'}, 'b':'potatoes'}, combine_upd_add)
|
||||
assert result == {'add': {'c': {'g': 'tea', 'h': 'cake'}}, 'delete': {'a': 'hamburger', 'c': {'f': 'coffee'}}, 'update': {'b': 'potatoes', 'c': {'e': 'pepsi'}}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a':'hotdog', 'b':'fries'}, combine_upd_add)
|
||||
assert result == {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update': {'a': 'hotdog'}}
|
||||
assert result == {'delete': {'c': None}, 'update': {'a': 'hotdog'}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a':'hamburger', 'b':'fries', 'c': {'e': 'pepsi', 'f': 'coffee'}}, combine_upd_add)
|
||||
assert result == {'update': {'c': {'e': 'pepsi'}}}
|
||||
@@ -347,7 +353,7 @@ def _legacy_assertions() -> None:
|
||||
assert result == {'update_add': {'c': 'coke'}}
|
||||
|
||||
result = jay_diff_full({'a': 'hamburger', 'b': 'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a': 'hamburger', 'b': 'fries'}, combine_upd_add)
|
||||
assert result == {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}}
|
||||
assert result == {'delete': {'c': None}}
|
||||
|
||||
result = jay_diff_full({'a': 'hamburger', 'b': 'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a': 'hamburger', 'b': 'fries', 'c': {'f': 'coffee'}}, combine_upd_add)
|
||||
assert result == {'delete': {'c': {'e': 'coke'}}}
|
||||
@@ -356,25 +362,25 @@ def _legacy_assertions() -> None:
|
||||
assert result == {'delete': {'a': 'hamburger', 'b': 'fries', 'c': {'f': 'coffee'}}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a':'hotdog', 'b':'potatoes'}, combine_upd_add)
|
||||
assert result == {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update_add': {'a': 'hotdog', 'b': 'potatoes'}}
|
||||
assert result == {'delete': {'c': None}, 'update_add': {'a': 'hotdog', 'b': 'potatoes'}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'c': {'e': 'pepsi'}}, combine_upd_add)
|
||||
assert result == {'delete': {'a': 'hamburger', 'b': 'fries', 'c': {'f': 'coffee'}}, 'update_add': {'c': {'e': 'pepsi'}}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a':'hotdog', 'b':'fries'}, combine_upd_add)
|
||||
assert result == {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update_add': {'a': 'hotdog'}}
|
||||
assert result == {'delete': {'c': None}, 'update_add': {'a': 'hotdog'}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a':'hamburger', 'b':'fries', 'c': {'e': 'pepsi', 'f': 'coffee'}}, combine_upd_add)
|
||||
assert result == {'update_add': {'c': {'e': 'pepsi'}}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a':'hotdog', 'b':'potatoes', 'd': 'icecream'}, combine_upd_add)
|
||||
assert result == {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update_add': {'a': 'hotdog', 'b': 'potatoes', 'd': 'icecream'}}
|
||||
assert result == {'delete': {'c': None}, 'update_add': {'a': 'hotdog', 'b': 'potatoes', 'd': 'icecream'}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'c': {'e': 'pepsi', 'g': 'tea', 'h': 'cake'}, 'b':'potatoes'}, combine_upd_add)
|
||||
assert result == {'delete': {'a': 'hamburger', 'c': {'f': 'coffee'}}, 'update_add': {'b': 'potatoes', 'c': {'e': 'pepsi', 'g': 'tea', 'h': 'cake'}}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a':'hotdog', 'b':'fries'}, combine_upd_add)
|
||||
assert result == {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update_add': {'a': 'hotdog'}}
|
||||
assert result == {'delete': {'c': None}, 'update_add': {'a': 'hotdog'}}
|
||||
|
||||
result = jay_diff_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'a':'hamburger', 'b':'fries', 'c': {'e': 'pepsi', 'f': 'coffee'}}, combine_upd_add)
|
||||
assert result == {'update_add': {'c': {'e': 'pepsi'}}}
|
||||
@@ -443,7 +449,7 @@ def _legacy_assertions() -> None:
|
||||
result = jay_merge_full({'a': 'hamburger', 'b': 'fries', 'c': 'pepsi'}, {'update': {'c': 'coke'}})
|
||||
assert result == {'a': 'hamburger', 'b': 'fries', 'c': 'coke'}
|
||||
|
||||
result = jay_merge_full({'a': 'hamburger', 'b': 'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}})
|
||||
result = jay_merge_full({'a': 'hamburger', 'b': 'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': None}})
|
||||
assert result == {'a': 'hamburger', 'b': 'fries'}
|
||||
|
||||
result = jay_merge_full({'a': 'hamburger', 'b': 'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': {'e': 'coke'}}})
|
||||
@@ -452,25 +458,25 @@ def _legacy_assertions() -> None:
|
||||
result = jay_merge_full({'a': 'hamburger', 'b': 'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'a': 'hamburger', 'b': 'fries', 'c': {'f': 'coffee'}}})
|
||||
assert result == {'c': {'e': 'coke'}}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update': {'a': 'hotdog', 'b': 'potatoes'}})
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': None}, 'update': {'a': 'hotdog', 'b': 'potatoes'}})
|
||||
assert result == {'a':'hotdog', 'b':'potatoes'}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'a': 'hamburger', 'b': 'fries', 'c': {'f': 'coffee'}}, 'update': {'c': {'e': 'pepsi'}}})
|
||||
assert result == {'c': {'e': 'pepsi'}}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update': {'a': 'hotdog'}})
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': None}, 'update': {'a': 'hotdog'}})
|
||||
assert result == {'a':'hotdog', 'b':'fries'}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'update': {'c': {'e': 'pepsi'}}})
|
||||
assert result == {'a':'hamburger', 'b':'fries', 'c': {'e': 'pepsi', 'f': 'coffee'}}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'add': {'d': 'icecream'}, 'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update': {'a': 'hotdog', 'b': 'potatoes'}})
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'add': {'d': 'icecream'}, 'delete': {'c': None}, 'update': {'a': 'hotdog', 'b': 'potatoes'}})
|
||||
assert result == {'a':'hotdog', 'b':'potatoes', 'd': 'icecream'}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'add': {'c': {'g': 'tea', 'h': 'cake'}}, 'delete': {'a': 'hamburger', 'c': {'f': 'coffee'}}, 'update': {'b': 'potatoes', 'c': {'e': 'pepsi'}}})
|
||||
assert result == {'c': {'e': 'pepsi', 'g': 'tea', 'h': 'cake'}, 'b':'potatoes'}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update': {'a': 'hotdog'}})
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': None}, 'update': {'a': 'hotdog'}})
|
||||
assert result == {'a':'hotdog', 'b':'fries'}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'update': {'c': {'e': 'pepsi'}}})
|
||||
@@ -515,7 +521,7 @@ def _legacy_assertions() -> None:
|
||||
result = jay_merge_full({'a': 'hamburger', 'b': 'fries', 'c': 'pepsi'}, {'update_add': {'c': 'coke'}})
|
||||
assert result == {'a': 'hamburger', 'b': 'fries', 'c': 'coke'}
|
||||
|
||||
result = jay_merge_full({'a': 'hamburger', 'b': 'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}})
|
||||
result = jay_merge_full({'a': 'hamburger', 'b': 'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': None}})
|
||||
assert result == {'a': 'hamburger', 'b': 'fries'}
|
||||
|
||||
result = jay_merge_full({'a': 'hamburger', 'b': 'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': {'e': 'coke'}}})
|
||||
@@ -524,25 +530,25 @@ def _legacy_assertions() -> None:
|
||||
result = jay_merge_full({'a': 'hamburger', 'b': 'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'a': 'hamburger', 'b': 'fries', 'c': {'f': 'coffee'}}})
|
||||
assert result == {'c': {'e': 'coke'}}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update_add': {'a': 'hotdog', 'b': 'potatoes'}})
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': None}, 'update_add': {'a': 'hotdog', 'b': 'potatoes'}})
|
||||
assert result == {'a':'hotdog', 'b':'potatoes'}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'a': 'hamburger', 'b': 'fries', 'c': {'f': 'coffee'}}, 'update_add': {'c': {'e': 'pepsi'}}})
|
||||
assert result == {'c': {'e': 'pepsi'}}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update_add': {'a': 'hotdog'}})
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': None}, 'update_add': {'a': 'hotdog'}})
|
||||
assert result == {'a':'hotdog', 'b':'fries'}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'update_add': {'c': {'e': 'pepsi'}}})
|
||||
assert result == {'a':'hamburger', 'b':'fries', 'c': {'e': 'pepsi', 'f': 'coffee'}}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update_add': {'a': 'hotdog', 'b': 'potatoes', 'd': 'icecream'}})
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': None}, 'update_add': {'a': 'hotdog', 'b': 'potatoes', 'd': 'icecream'}})
|
||||
assert result == {'a':'hotdog', 'b':'potatoes', 'd': 'icecream'}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'a': 'hamburger', 'c': {'f': 'coffee'}}, 'update_add': {'b': 'potatoes', 'c': {'e': 'pepsi', 'g': 'tea', 'h': 'cake'}}})
|
||||
assert result == {'b':'potatoes', 'c': {'e': 'pepsi', 'g': 'tea', 'h': 'cake'}}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': {'e': 'coke', 'f': 'coffee'}}, 'update_add': {'a': 'hotdog'}})
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'delete': {'c': None}, 'update_add': {'a': 'hotdog'}})
|
||||
assert result == {'a':'hotdog', 'b':'fries'}
|
||||
|
||||
result = jay_merge_full({'a':'hamburger', 'b':'fries', 'c': {'e': 'coke', 'f': 'coffee'}}, {'update_add': {'c': {'e': 'pepsi'}}})
|
||||
|
||||
Reference in New Issue
Block a user