diff --git a/arielle.py b/arielle.py index 9cdbc5b..04d2146 100644 --- a/arielle.py +++ b/arielle.py @@ -7,8 +7,9 @@ from wci import wci from credentials import user, password import json from folder_mgr.folder_mgr import get_date, date_has_changed, to_date_str, to_time_str, FolderMgr, create_timestamp +from deep_diff import deep_diff, deepcopy -INTERVAL = 5*60 +INTERVAL = 1*60 domains = [ "access", @@ -52,39 +53,55 @@ def main(): # Init collecting records f_mgr = FolderMgr() record_id = 0 - record_list = [] - _date_last = {} - - initial_timestamp = create_timestamp().replace(':', '-').replace('.', '-') + record_list = None + _date_last = None + initial_timestamp = None + vehicle_data_last = None # Collecting loop while True: start = time.time() _date = get_date() timestamp = create_timestamp() + # Reset on start up or day change if date_has_changed(_date_last, _date): + _date_last = _date record_id = 0 record_list = [] + initial_timestamp = create_timestamp().replace(':', '-').replace('.', '-') + vehicle_data_last = {} try: - data = w.get(endpoint) - record_list.append({'info': {'record_id': record_id, 'timestamp': timestamp}, 'data': data}) + print(f"{timestamp}: Retrieve vehicle data") + + # Create data difference + vehicle_data = w.get(endpoint) + vehicle_data_diff = deep_diff(vehicle_data, vehicle_data_last) + vehicle_data_last = vehicle_data + + # Put vehicle data difference onto record list + record_list.append({'info': {'record_id': record_id, 'timestamp': timestamp}, 'data': vehicle_data_diff}) + + # Store record list + print(f"{timestamp}: Store record #{record_id:04d}") folder = f_mgr.update(f"results/{vin}", _date) filename = f"vehicle_{vin}_{initial_timestamp}.json" _path = os.path.join(folder, filename) with open(_path, "w") as fp: json.dump(record_list, fp, indent=4) + record_id += 1 + except Exception as e: print(f"Exception: {e}") - _date_last = _date - record_id += 1 - # wait until next loop stop = time.time() - diff = int(stop - start) - time.sleep(max(0, INTERVAL - diff)) + diff = stop - start + time_to_wait = INTERVAL - diff + if time_to_wait < 0: + time_to_wait = 0 + time.sleep(time_to_wait) if __name__ == '__main__': diff --git a/deep_diff.py b/deep_diff.py new file mode 100644 index 0000000..2e0a202 --- /dev/null +++ b/deep_diff.py @@ -0,0 +1,58 @@ +import json +from copy import deepcopy + + +def deep_diff(data, temp_data): + new_data = {} + for k in data.keys(): + if type(data[k]) == type({}): + if k not in temp_data: + temp_data[k] = {} + temp = deep_diff(data[k], temp_data[k]) + if temp: + new_data[k] = {} + new_data[k] = temp + else: + if k in temp_data: + if data[k] != temp_data[k]: + new_data[k] = data[k] + else: + new_data[k] = data[k] + return new_data + + +def deep_merge(d, u, only_existing=True): + for k, v in u.items(): + if isinstance(v, dict): + if only_existing: + r = deep_merge(d.get(k, {}), v) + d[k] = r + else: + d[k] = v + else: + d[k] = u[k] + return d + + +if __name__ == '__main__': + with open("results/WVWZZZE1ZMP010760/2024/10/18/vehicle_WVWZZZE1ZMP010760_2024-10-15T12-05-23-268Z.json", "r") as fp: + src = json.load(fp) + src_last = {} + diffed_list = [] + last = {} + for s in src: + diffed = deep_diff(s, last) + diffed_list.append(deepcopy(diffed)) + last = deepcopy(s) + print(f"Diffed: {diffed}") + + merged_list = [] + last = {} + for d in diffed_list: + merged = deep_merge(last, d) + last = deepcopy(merged) + merged_list.append(deepcopy(merged)) + print(f"Merged: {merged}") + + assert src == merged_list + diff --git a/folder_mgr/folder_mgr.py b/folder_mgr/folder_mgr.py index 7e8daff..ca61429 100644 --- a/folder_mgr/folder_mgr.py +++ b/folder_mgr/folder_mgr.py @@ -35,8 +35,10 @@ def create_timestamp(): def date_has_changed(d1: dict, d2: dict, fields=['year', 'month', 'day']): - count = 0 + if d1 is None or d2 is None: + return True + count = 0 try: for field in fields: if d1[field] != d2[field]: