import json import os from datetime import datetime from jay_diff import jay_merge_full from matplotlib import pyplot as plot BASE = "/media/jens/cifs/jens@moon/projects/we_collect/results" VIN = 'WVWZZZE1ZMP010760' USER = 'alex' YEAR = 2024 MONTH = 10 DAY = ['15', '16', '17', '18', '19', '20', '21', '22', '23'] def has_diff_entries(diff): handled = False if "update" in diff: handled = True if "update_add" in diff: handled = True if "add" in diff: handled = True if "delete" in diff: handled = True return handled class Endpoint: def __init__(self, _ep_path: str, _ts_path: str, _ts_format='%Y-%m-%dT%H:%M:%S.%f%z'): self._ep_path = _ep_path self._ep_path_keys = self.to_keys(_ep_path) self._timestamps = [] self._values = [] self._ts_format = _ts_format self._ts_path_keys = self.to_keys(_ts_path) def timestamps_str(self): return f"{self._ep_path}:{len(self._timestamps)}: {self._timestamps}" @property def timestamps(self): return self._timestamps @property def values(self): return self._values @property def path(self): return self._ep_path def assign(self, _record: dict): value = self.deref_multi(_record, self._ep_path_keys) self._values += [value] # Extract timestamp capture_time = self.deref_multi(_record, self._ts_path_keys) timestamp_int = convert(capture_time, self._ts_format) self._timestamps += [timestamp_int] def remove_dups(self): new_dict = {} index = 0 unique_indices = [] for ts in self._timestamps: if ts not in new_dict: new_dict[ts] = index unique_indices.append(index) index += 1 t_list = self._timestamps v_list = self._values self._timestamps = [] self._values = {} self._timestamps = [t_list[i] for i in unique_indices] self._values = [v_list[i] for i in unique_indices] @staticmethod def to_keys(key_path: str): keys = key_path.split('/') return keys @staticmethod def deref_multi(data, keys): return Endpoint.deref_multi(data[keys[0]], keys[1:]) if keys else data end_points = { 'odo': Endpoint('data/measurements/odometerStatus/value/odometer', 'info/timestamp'), 'soc': Endpoint('data/measurements/fuelLevelStatus/value/currentSOC_pct', 'info/timestamp'), 'engine': Endpoint('data/fuelStatus/rangeStatus/value/primaryEngine', 'info/timestamp') } def convert(timestamp_str: str, pattern: str = "%Y-%m-%dT%H:%M:%S.%f%z"): ts = datetime.strptime(timestamp_str, pattern) result = ts.year result = 100*result + ts.month result = 100*result + ts.day result = 100*result + ts.hour result = 100*result + ts.minute result = 100*result + ts.second result = 1000000*result + ts.microsecond return result path = os.path.join(BASE, USER) path = os.path.join(path, VIN) path = os.path.join(path, str(YEAR)) path = os.path.join(path, str(MONTH)) day_list = sorted(os.listdir(path), key=lambda f: int(f)) print("Days in '", path, "' :") # prints all files print(day_list) month_path = path record_list = [] last = {} for day in day_list: if DAY is not None: if day not in DAY: continue day_path = os.path.join(month_path, day) # Sort out files with unknown nickname file_list = os.listdir(day_path) file_list_sorted = sorted(file_list, key=lambda f: convert(f, pattern=f"records_%Y-%m-%dT%H-%M-%S.json")) print("Files in '", day_path, "' :") # prints all files print(file_list_sorted) for filename in file_list_sorted: file_path = os.path.join(day_path, filename) print(f"Open file: {filename}") with open(file_path, "r") as fp: records = json.load(fp) print(f"Found {len(records)} records") for record in records: try: vehicle_diff = record['data'] is_header = True if 'is_delta' in record['info']: if record['info']['is_delta']: is_header = False print(f"Found delta record #{record['info']['record_id']}") if is_header: print(f"Found full record #{record['info']['record_id']}") last = {} except KeyError as e: print(f"KeyError: {e}") if not has_diff_entries(vehicle_diff): vehicle_diff = {'update_add': vehicle_diff} vehicle_data = jay_merge_full(last, vehicle_diff) last = vehicle_data try: for k in end_points.keys(): end_points[k].assign({'info': record['info'], 'data': vehicle_data}) except KeyError: pass for k in end_points.keys(): end_points[k].remove_dups() plot.subplot(2, 1, 1) plot.plot(end_points['odo'].timestamps, end_points['odo'].values) plot.title("Kilometerstand") plot.grid() plot.subplot(2, 1, 2) plot.plot(end_points['odo'].timestamps, end_points['soc'].values) plot.title("Akku in %") plot.grid() plot.show()