store difference of vehicle data to file

arielle.py

@@ -7,8 +7,9 @@ from wci import wci
 from credentials import user, password
 import json
 from folder_mgr.folder_mgr import get_date, date_has_changed, to_date_str, to_time_str, FolderMgr, create_timestamp
+from deep_diff import deep_diff, deepcopy
 
-INTERVAL = 5*60
+INTERVAL = 1*60
 
 domains = [
 	"access",
@@ -52,39 +53,55 @@ def main():
 	# Init collecting records
 	f_mgr = FolderMgr()
 	record_id = 0
-	record_list = []
-	_date_last = {}
-
-	initial_timestamp = create_timestamp().replace(':', '-').replace('.', '-')
+	record_list = None
+	_date_last = None
+	initial_timestamp = None
+	vehicle_data_last = None
 
 	# Collecting loop
 	while True:
 		start = time.time()
 		_date = get_date()
 		timestamp = create_timestamp()
+		# Reset on start up or day change
 		if date_has_changed(_date_last, _date):
+			_date_last = _date
 			record_id = 0
 			record_list = []
+			initial_timestamp = create_timestamp().replace(':', '-').replace('.', '-')
+			vehicle_data_last = {}
 
 		try:
-			data = w.get(endpoint)
-			record_list.append({'info': {'record_id': record_id, 'timestamp': timestamp}, 'data': data})
+			print(f"{timestamp}: Retrieve vehicle data")
+
+			# Create data difference
+			vehicle_data = w.get(endpoint)
+			vehicle_data_diff = deep_diff(vehicle_data, vehicle_data_last)
+			vehicle_data_last = vehicle_data
+
+			# Put vehicle data difference onto record list
+			record_list.append({'info': {'record_id': record_id, 'timestamp': timestamp}, 'data': vehicle_data_diff})
+
+			# Store record list
+			print(f"{timestamp}: Store record #{record_id:04d}")
 			folder = f_mgr.update(f"results/{vin}", _date)
 			filename = f"vehicle_{vin}_{initial_timestamp}.json"
 			_path = os.path.join(folder, filename)
 			with open(_path, "w") as fp:
 				json.dump(record_list, fp, indent=4)
 
+			record_id += 1
+
 		except Exception as e:
 			print(f"Exception: {e}")
 
-		_date_last = _date
-		record_id += 1
-
 		# wait until next loop
 		stop = time.time()
-		diff = int(stop - start)
-		time.sleep(max(0, INTERVAL - diff))
+		diff = stop - start
+		time_to_wait = INTERVAL - diff
+		if time_to_wait < 0:
+			time_to_wait = 0
+		time.sleep(time_to_wait)
 
 
 if __name__ == '__main__':

deep_diff.py

@@ -0,0 +1,58 @@
+import json
+from copy import deepcopy
+
+
+def deep_diff(data, temp_data):
+	new_data = {}
+	for k in data.keys():
+		if type(data[k]) == type({}):
+			if k not in temp_data:
+				temp_data[k] = {}
+			temp = deep_diff(data[k], temp_data[k])
+			if temp:
+				new_data[k] = {}
+				new_data[k] = temp
+		else:
+			if k in temp_data:
+				if data[k] != temp_data[k]:
+					new_data[k] = data[k]
+			else:
+				new_data[k] = data[k]
+	return new_data
+
+
+def deep_merge(d, u, only_existing=True):
+	for k, v in u.items():
+		if isinstance(v, dict):
+			if only_existing:
+				r = deep_merge(d.get(k, {}), v)
+				d[k] = r
+			else:
+				d[k] = v
+		else:
+			d[k] = u[k]
+	return d
+
+
+if __name__ == '__main__':
+	with open("results/WVWZZZE1ZMP010760/2024/10/18/vehicle_WVWZZZE1ZMP010760_2024-10-15T12-05-23-268Z.json", "r") as fp:
+		src = json.load(fp)
+		src_last = {}
+		diffed_list = []
+		last = {}
+		for s in src:
+			diffed = deep_diff(s, last)
+			diffed_list.append(deepcopy(diffed))
+			last = deepcopy(s)
+			print(f"Diffed: {diffed}")
+
+		merged_list = []
+		last = {}
+		for d in diffed_list:
+			merged = deep_merge(last, d)
+			last = deepcopy(merged)
+			merged_list.append(deepcopy(merged))
+			print(f"Merged: {merged}")
+
+		assert src == merged_list
+

folder_mgr/folder_mgr.py

@@ -35,8 +35,10 @@ def create_timestamp():
 
 
 def date_has_changed(d1: dict, d2: dict, fields=['year', 'month', 'day']):
-	count = 0
+	if d1 is None or d2 is None:
+		return True
 
+	count = 0
 	try:
 		for field in fields:
 			if d1[field] != d2[field]: