diff --git a/eval_records.py b/eval_records.py index f5b8932..c9afbc1 100644 --- a/eval_records.py +++ b/eval_records.py @@ -4,13 +4,7 @@ import numpy as np import dateutil.parser as dup from jay_diff import jay_merge_full from matplotlib import pyplot as plot - -BASE = "/media/jens/cifs/jens@moon/projects/we_collect/results" -VIN = 'WVWZZZE1ZMP010760' -USER = 'alex' -YEAR = 2024 -MONTH = 10 -DAYS = [15, 16, 17, 18, 19, 20, 21, 22, 23, 24] +import eval_records_settings as settings def has_diff_entries(diff): @@ -27,29 +21,51 @@ def has_diff_entries(diff): return handled +def convert(_filename: str) -> int: + name = _filename + if "records_" in _filename: + name = os.path.splitext(_filename)[0].split("_")[1] + date_str = name.split("T")[0] + time_str = name.split("T")[1] + if "-" in time_str: + time_str = time_str.replace("-", ":") + + timestamp_str = f"{date_str}T{time_str}" + ts = dup.parse(timestamp_str) + result = ts.year + result = 100*result + ts.month + result = 100*result + ts.day + result = 100*result + ts.hour + result = 100*result + ts.minute + result = 100*result + ts.second + result = 1000000*result + ts.microsecond + + return int(result) + + class Endpoint: - def __init__(self, _ep_path: str, _ts_path: str, _ts_format='%Y-%m-%dT%H:%M:%S.%f%z'): - self._ep_path = _ep_path - self._ep_path_keys = self.to_keys(_ep_path) + def __init__(self, _ep_spec: dict, _ts_format='%Y-%m-%dT%H:%M:%S.%f%z'): + self._ep_path = _ep_spec["values"] + self._ep_path_keys = self.to_keys(self._ep_path) self._timestamps = [] self._values = [] self._ts_format = _ts_format - self._ts_path_keys = self.to_keys(_ts_path) + self._ts_path_keys = self.to_keys(_ep_spec["timestamps"]) - def timestamps_str(self): + def timestamps_str(self) -> str: return f"{self._ep_path}:{len(self._timestamps)}: {self._timestamps}" @property - def timestamps(self): + def timestamps(self) -> list[int]: return self._timestamps @property - def values(self): + def values(self) -> list[float]: return self._values @property - def path(self): + def path(self) -> str: return self._ep_path def assign(self, _record: dict): @@ -89,115 +105,112 @@ class Endpoint: return Endpoint.deref_multi(data[keys[0]], keys[1:]) if keys else data -end_points = { -# 'odo': Endpoint('data/measurements/odometerStatus/value/odometer', 'data/measurements/odometerStatus/value/carCapturedTimestamp'), -# 'soc': Endpoint('data/measurements/fuelLevelStatus/value/currentSOC_pct', 'data/measurements/fuelLevelStatus/value/carCapturedTimestamp'), - 'odo': Endpoint('data/measurements/odometerStatus/value/odometer','info/timestamp'), - 'soc': Endpoint('data/measurements/fuelLevelStatus/value/currentSOC_pct','info/timestamp'), -} +def list_folder_ymd(_path: str): + day_list = sorted(os.listdir(_path), key=lambda f: int(f)) + return day_list -def convert(_filename: str): - name = _filename - if "records_" in _filename: - name = os.path.splitext(_filename)[0].split("_")[1] - date_str = name.split("T")[0] - time_str = name.split("T")[1] - if "-" in time_str: - time_str = time_str.replace("-", ":") +def check_sel(root_path, sel: list): + res = [] + dir_entries = list_folder_ymd(root_path) + if sel is None: + res = dir_entries + else: + if isinstance(sel, tuple): + if len(sel) == 2: + res = [str(e) for e in range(sel[0], sel[1]+1, 1)] + else: + res = [str(e) for e in dir_entries if int(e) >= int(sel[0])] - timestamp_str = f"{date_str}T{time_str}" - ts = dup.parse(timestamp_str) - result = ts.year - result = 100*result + ts.month - result = 100*result + ts.day - result = 100*result + ts.hour - result = 100*result + ts.minute - result = 100*result + ts.second - result = 1000000*result + ts.microsecond - - return result + return res -path = os.path.join(BASE, USER) -path = os.path.join(path, VIN) -path = os.path.join(path, str(YEAR)) -path = os.path.join(path, str(MONTH)) +def process(end_points, base_path: str, user: str, vin: str, sel_years=None, sel_months=None, sel_days=None, sel_hours=None): + _path = os.path.join(base_path, user) + root_path = os.path.join(_path, vin) -day_list = sorted(os.listdir(path), key=lambda f: int(f)) -print("Days in '", path, "' :") -# prints all files -print(day_list) + for _year in check_sel(root_path, sel_years): + year_path = os.path.join(root_path, _year) + if not os.path.exists(year_path): + continue + for _month in check_sel(year_path, sel_months): + month_path = os.path.join(year_path, _month) + if not os.path.exists(month_path): + continue + for _day in check_sel(month_path, sel_days): + day_path = os.path.join(month_path, _day) + if not os.path.exists(day_path): + continue -month_path = path -record_list = [] -last = {} + # Sort out files with unknown nickname + file_list_sorted = sorted(os.listdir(day_path), key=lambda f: convert(f)) + vehicle_diff = {} + last = {} + for filename in file_list_sorted: + file_path = os.path.join(day_path, filename) + print(f"Open file: {filename}") + with open(file_path, "r") as fp: + records = json.load(fp) + print(f"Found {len(records)} records") + for record in records: + try: + vehicle_diff = record['data'] + is_header = True + if 'is_delta' in record['info']: + if record['info']['is_delta']: + is_header = False -for _day in DAYS: - day = str(_day) - if day not in day_list: - continue - day_path = os.path.join(month_path, day) + if is_header: + last = {} - # Sort out files with unknown nickname - file_list = os.listdir(day_path) - file_list_sorted = sorted(file_list, key=lambda f: convert(f)) - print("Files in '", day_path, "' :") - # prints all files - print(file_list_sorted) + except KeyError as e: + print(f"KeyError: {e}") - for filename in file_list_sorted: - file_path = os.path.join(day_path, filename) - print(f"Open file: {filename}") - with open(file_path, "r") as fp: - records = json.load(fp) - print(f"Found {len(records)} records") - for record in records: - try: - vehicle_diff = record['data'] - is_header = True - if 'is_delta' in record['info']: - if record['info']['is_delta']: - is_header = False + if not has_diff_entries(vehicle_diff): + vehicle_diff = {'update_add': vehicle_diff} - if is_header: - last = {} + vehicle_data = jay_merge_full(last, vehicle_diff) + last = vehicle_data + try: + for k in end_points.keys(): + end_points[k].assign({'info': record['info'], 'data': vehicle_data}) + except KeyError: + pass - except KeyError as e: - print(f"KeyError: {e}") + for k in end_points.keys(): + end_points[k].remove_dups() - if not has_diff_entries(vehicle_diff): - vehicle_diff = {'update_add': vehicle_diff} - vehicle_data = jay_merge_full(last, vehicle_diff) - last = vehicle_data - try: - for k in end_points.keys(): - end_points[k].assign({'info': record['info'], 'data': vehicle_data}) - except KeyError: - pass +if __name__ == '__main__': + end_points = {} + for k in settings.eps.keys(): + end_points[k] = Endpoint(settings.eps[k]) -for k in end_points.keys(): - end_points[k].remove_dups() + process(end_points, settings.BASE, settings.USER, settings.VIN, sel_days=settings.sel_days) -speed = [0] -for i in range(1, len(end_points['odo'].timestamps)): - dt = float(end_points['odo'].timestamps[i] - end_points['odo'].timestamps[i-1])/(60*100000000) - dv = end_points['odo'].values[i] - end_points['odo'].values[i-1] - _speed = dv/dt - speed.append(_speed) + speed = [0] + _speed = 0 + for i in range(1, len(end_points['odo'].timestamps)): + dt = float(end_points['odo'].timestamps[i] - end_points['odo'].timestamps[i-1])/(60*100000000) + dv = end_points['odo'].values[i] - end_points['odo'].values[i-1] + _speed = 0.9*_speed + 0.1*dv/dt + speed.append(_speed) -plot.subplot(3, 1, 1) -plot.plot(end_points['odo'].timestamps, np.array(end_points['odo'].values) - end_points['odo'].values[0]) -plot.title("Kilometerstand") -plot.grid() -plot.subplot(3, 1, 2) -plot.plot(end_points['odo'].timestamps, speed) -plot.title("Speed") -plot.grid() -plot.subplot(3, 1, 3) -plot.plot(end_points['soc'].timestamps, end_points['soc'].values) -plot.title("Akkustand") -plot.grid() + plot.subplot(4, 1, 1) + plot.plot(np.array(end_points['odo'].timestamps) - end_points['odo'].timestamps[0], np.array(end_points['odo'].values) - end_points['odo'].values[0]) + plot.title("Kilometerstand") + plot.grid() + plot.subplot(4, 1, 2) + plot.plot(np.array(end_points['odo'].timestamps) - end_points['odo'].timestamps[0], speed) + plot.title("Speed") + plot.grid() + plot.subplot(4, 1, 3) + plot.plot(np.array(end_points['soc'].timestamps) - end_points['soc'].timestamps[0], end_points['soc'].values) + plot.title("Akkustand") + plot.grid() + plot.subplot(4, 1, 4) + plot.plot(np.array(end_points['range'].timestamps) - end_points['range'].timestamps[0], end_points['range'].values) + plot.title("Range") + plot.grid() -plot.show() + plot.show() diff --git a/eval_records_settings.py b/eval_records_settings.py new file mode 100644 index 0000000..60f0a3e --- /dev/null +++ b/eval_records_settings.py @@ -0,0 +1,28 @@ +# Settings for eval_record + +BASE = "/media/jens/cifs/jens@moon/projects/we_collect/results" +VIN = 'WVWZZZE1ZMP010760' +USER = 'alex' + +sel_days = (23, 24) + +eps = { + "odo": + { + "values": "data/measurements/odometerStatus/value/odometer", + "timestamps": "data/measurements/odometerStatus/value/carCapturedTimestamp" + }, + "soc": + { + "values": "data/measurements/fuelLevelStatus/value/currentSOC_pct", + "timestamps": "data/measurements/fuelLevelStatus/value/carCapturedTimestamp" + }, + "range": + { + "values": "data/fuelStatus/rangeStatus/value/totalRange_km", + "timestamps": "data/fuelStatus/rangeStatus/value/carCapturedTimestamp" + } +} + +# 'odo': Endpoint('data/measurements/odometerStatus/value/odometer','info/timestamp'), +# 'soc': Endpoint('data/measurements/fuelLevelStatus/value/currentSOC_pct','info/timestamp'),