jens/we_collect
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored

Browse Source
This commit is contained in:
jens
2024-10-24 20:59:14 +02:00
parent 6e41921c37
commit c0f76617f7
2 changed files with 150 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
eval_records.py
+122 -109
View File
@@ -4,13 +4,7 @@ import numpy as np
import dateutil.parser as dup import dateutil.parser as dup
from jay_diff import jay_merge_full from jay_diff import jay_merge_full
from matplotlib import pyplot as plot from matplotlib import pyplot as plot
import eval_records_settings as settings
BASE = "/media/jens/cifs/jens@moon/projects/we_collect/results"
VIN = 'WVWZZZE1ZMP010760'
USER = 'alex'
YEAR = 2024
MONTH = 10
DAYS = [15, 16, 17, 18, 19, 20, 21, 22, 23, 24]
def has_diff_entries(diff): def has_diff_entries(diff):
@@ -27,29 +21,51 @@ def has_diff_entries(diff):
return handled return handled
def convert(_filename: str) -> int:
name = _filename
if "records_" in _filename:
name = os.path.splitext(_filename)[0].split("_")[1]
date_str = name.split("T")[0]
time_str = name.split("T")[1]
if "-" in time_str:
time_str = time_str.replace("-", ":")
timestamp_str = f"{date_str}T{time_str}"
ts = dup.parse(timestamp_str)
result = ts.year
result = 100*result + ts.month
result = 100*result + ts.day
result = 100*result + ts.hour
result = 100*result + ts.minute
result = 100*result + ts.second
result = 1000000*result + ts.microsecond
return int(result)
class Endpoint: class Endpoint:
def __init__(self, _ep_path: str, _ts_path: str, _ts_format='%Y-%m-%dT%H:%M:%S.%f%z'): def __init__(self, _ep_spec: dict, _ts_format='%Y-%m-%dT%H:%M:%S.%f%z'):
self._ep_path = _ep_path self._ep_path = _ep_spec["values"]
self._ep_path_keys = self.to_keys(_ep_path) self._ep_path_keys = self.to_keys(self._ep_path)
self._timestamps = [] self._timestamps = []
self._values = [] self._values = []
self._ts_format = _ts_format self._ts_format = _ts_format
self._ts_path_keys = self.to_keys(_ts_path) self._ts_path_keys = self.to_keys(_ep_spec["timestamps"])
def timestamps_str(self): def timestamps_str(self) -> str:
return f"{self._ep_path}:{len(self._timestamps)}: {self._timestamps}" return f"{self._ep_path}:{len(self._timestamps)}: {self._timestamps}"
@property @property
def timestamps(self): def timestamps(self) -> list[int]:
return self._timestamps return self._timestamps
@property @property
def values(self): def values(self) -> list[float]:
return self._values return self._values
@property @property
def path(self): def path(self) -> str:
return self._ep_path return self._ep_path
def assign(self, _record: dict): def assign(self, _record: dict):
@@ -89,115 +105,112 @@ class Endpoint:
return Endpoint.deref_multi(data[keys[0]], keys[1:]) if keys else data return Endpoint.deref_multi(data[keys[0]], keys[1:]) if keys else data
end_points = { def list_folder_ymd(_path: str):
# 'odo': Endpoint('data/measurements/odometerStatus/value/odometer', 'data/measurements/odometerStatus/value/carCapturedTimestamp'), day_list = sorted(os.listdir(_path), key=lambda f: int(f))
# 'soc': Endpoint('data/measurements/fuelLevelStatus/value/currentSOC_pct', 'data/measurements/fuelLevelStatus/value/carCapturedTimestamp'), return day_list
'odo': Endpoint('data/measurements/odometerStatus/value/odometer','info/timestamp'),
'soc': Endpoint('data/measurements/fuelLevelStatus/value/currentSOC_pct','info/timestamp'),
}
def convert(_filename: str): def check_sel(root_path, sel: list):
name = _filename res = []
if "records_" in _filename: dir_entries = list_folder_ymd(root_path)
name = os.path.splitext(_filename)[0].split("_")[1] if sel is None:
date_str = name.split("T")[0] res = dir_entries
time_str = name.split("T")[1] else:
if "-" in time_str: if isinstance(sel, tuple):
time_str = time_str.replace("-", ":") if len(sel) == 2:
res = [str(e) for e in range(sel[0], sel[1]+1, 1)]
else:
res = [str(e) for e in dir_entries if int(e) >= int(sel[0])]
timestamp_str = f"{date_str}T{time_str}" return res
ts = dup.parse(timestamp_str)
result = ts.year
result = 100*result + ts.month
result = 100*result + ts.day
result = 100*result + ts.hour
result = 100*result + ts.minute
result = 100*result + ts.second
result = 1000000*result + ts.microsecond
return result
path = os.path.join(BASE, USER) def process(end_points, base_path: str, user: str, vin: str, sel_years=None, sel_months=None, sel_days=None, sel_hours=None):
path = os.path.join(path, VIN) _path = os.path.join(base_path, user)
path = os.path.join(path, str(YEAR)) root_path = os.path.join(_path, vin)
path = os.path.join(path, str(MONTH))
day_list = sorted(os.listdir(path), key=lambda f: int(f)) for _year in check_sel(root_path, sel_years):
print("Days in '", path, "' :") year_path = os.path.join(root_path, _year)
# prints all files if not os.path.exists(year_path):
print(day_list) continue
for _month in check_sel(year_path, sel_months):
month_path = os.path.join(year_path, _month)
if not os.path.exists(month_path):
continue
for _day in check_sel(month_path, sel_days):
day_path = os.path.join(month_path, _day)
if not os.path.exists(day_path):
continue
month_path = path # Sort out files with unknown nickname
record_list = [] file_list_sorted = sorted(os.listdir(day_path), key=lambda f: convert(f))
last = {} vehicle_diff = {}
last = {}
for filename in file_list_sorted:
file_path = os.path.join(day_path, filename)
print(f"Open file: {filename}")
with open(file_path, "r") as fp:
records = json.load(fp)
print(f"Found {len(records)} records")
for record in records:
try:
vehicle_diff = record['data']
is_header = True
if 'is_delta' in record['info']:
if record['info']['is_delta']:
is_header = False
for _day in DAYS: if is_header:
day = str(_day) last = {}
if day not in day_list:
continue
day_path = os.path.join(month_path, day)
# Sort out files with unknown nickname except KeyError as e:
file_list = os.listdir(day_path) print(f"KeyError: {e}")
file_list_sorted = sorted(file_list, key=lambda f: convert(f))
print("Files in '", day_path, "' :")
# prints all files
print(file_list_sorted)
for filename in file_list_sorted: if not has_diff_entries(vehicle_diff):
file_path = os.path.join(day_path, filename) vehicle_diff = {'update_add': vehicle_diff}
print(f"Open file: {filename}")
with open(file_path, "r") as fp:
records = json.load(fp)
print(f"Found {len(records)} records")
for record in records:
try:
vehicle_diff = record['data']
is_header = True
if 'is_delta' in record['info']:
if record['info']['is_delta']:
is_header = False
if is_header: vehicle_data = jay_merge_full(last, vehicle_diff)
last = {} last = vehicle_data
try:
for k in end_points.keys():
end_points[k].assign({'info': record['info'], 'data': vehicle_data})
except KeyError:
pass
except KeyError as e: for k in end_points.keys():
print(f"KeyError: {e}") end_points[k].remove_dups()
if not has_diff_entries(vehicle_diff):
vehicle_diff = {'update_add': vehicle_diff}
vehicle_data = jay_merge_full(last, vehicle_diff) if __name__ == '__main__':
last = vehicle_data end_points = {}
try: for k in settings.eps.keys():
for k in end_points.keys(): end_points[k] = Endpoint(settings.eps[k])
end_points[k].assign({'info': record['info'], 'data': vehicle_data})
except KeyError:
pass
for k in end_points.keys(): process(end_points, settings.BASE, settings.USER, settings.VIN, sel_days=settings.sel_days)
end_points[k].remove_dups()
speed = [0] speed = [0]
for i in range(1, len(end_points['odo'].timestamps)): _speed = 0
dt = float(end_points['odo'].timestamps[i] - end_points['odo'].timestamps[i-1])/(60*100000000) for i in range(1, len(end_points['odo'].timestamps)):
dv = end_points['odo'].values[i] - end_points['odo'].values[i-1] dt = float(end_points['odo'].timestamps[i] - end_points['odo'].timestamps[i-1])/(60*100000000)
_speed = dv/dt dv = end_points['odo'].values[i] - end_points['odo'].values[i-1]
speed.append(_speed) _speed = 0.9*_speed + 0.1*dv/dt
speed.append(_speed)
plot.subplot(3, 1, 1) plot.subplot(4, 1, 1)
plot.plot(end_points['odo'].timestamps, np.array(end_points['odo'].values) - end_points['odo'].values[0]) plot.plot(np.array(end_points['odo'].timestamps) - end_points['odo'].timestamps[0], np.array(end_points['odo'].values) - end_points['odo'].values[0])
plot.title("Kilometerstand") plot.title("Kilometerstand")
plot.grid() plot.grid()
plot.subplot(3, 1, 2) plot.subplot(4, 1, 2)
plot.plot(end_points['odo'].timestamps, speed) plot.plot(np.array(end_points['odo'].timestamps) - end_points['odo'].timestamps[0], speed)
plot.title("Speed") plot.title("Speed")
plot.grid() plot.grid()
plot.subplot(3, 1, 3) plot.subplot(4, 1, 3)
plot.plot(end_points['soc'].timestamps, end_points['soc'].values) plot.plot(np.array(end_points['soc'].timestamps) - end_points['soc'].timestamps[0], end_points['soc'].values)
plot.title("Akkustand") plot.title("Akkustand")
plot.grid() plot.grid()
plot.subplot(4, 1, 4)
plot.plot(np.array(end_points['range'].timestamps) - end_points['range'].timestamps[0], end_points['range'].values)
plot.title("Range")
plot.grid()
plot.show() plot.show()
eval_records_settings.py
+28
View File
@@ -0,0 +1,28 @@
# Settings for eval_record
BASE = "/media/jens/cifs/jens@moon/projects/we_collect/results"
VIN = 'WVWZZZE1ZMP010760'
USER = 'alex'
sel_days = (23, 24)
eps = {
"odo":
{
"values": "data/measurements/odometerStatus/value/odometer",
"timestamps": "data/measurements/odometerStatus/value/carCapturedTimestamp"
},
"soc":
{
"values": "data/measurements/fuelLevelStatus/value/currentSOC_pct",
"timestamps": "data/measurements/fuelLevelStatus/value/carCapturedTimestamp"
},
"range":
{
"values": "data/fuelStatus/rangeStatus/value/totalRange_km",
"timestamps": "data/fuelStatus/rangeStatus/value/carCapturedTimestamp"
}
}
# 'odo': Endpoint('data/measurements/odometerStatus/value/odometer','info/timestamp'),
# 'soc': Endpoint('data/measurements/fuelLevelStatus/value/currentSOC_pct','info/timestamp'),