jens/we_collect
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored

Browse Source
This commit is contained in:
jens
2024-10-24 20:59:14 +02:00
parent 6e41921c37
commit c0f76617f7
2 changed files with 150 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
eval_records.py
+122 -109
View File
@@ -4,13 +4,7 @@ import numpy as np
import dateutil.parser as dup
from jay_diff import jay_merge_full
from matplotlib import pyplot as plot
BASE = "/media/jens/cifs/jens@moon/projects/we_collect/results"
VIN = 'WVWZZZE1ZMP010760'
USER = 'alex'
YEAR = 2024
MONTH = 10
DAYS = [15, 16, 17, 18, 19, 20, 21, 22, 23, 24]
import eval_records_settings as settings
def has_diff_entries(diff):
@@ -27,29 +21,51 @@ def has_diff_entries(diff):
return handled
def convert(_filename: str) -> int:
name = _filename
if "records_" in _filename:
name = os.path.splitext(_filename)[0].split("_")[1]
date_str = name.split("T")[0]
time_str = name.split("T")[1]
if "-" in time_str:
time_str = time_str.replace("-", ":")
timestamp_str = f"{date_str}T{time_str}"
ts = dup.parse(timestamp_str)
result = ts.year
result = 100*result + ts.month
result = 100*result + ts.day
result = 100*result + ts.hour
result = 100*result + ts.minute
result = 100*result + ts.second
result = 1000000*result + ts.microsecond
return int(result)
class Endpoint:
def __init__(self, _ep_path: str, _ts_path: str, _ts_format='%Y-%m-%dT%H:%M:%S.%f%z'):
self._ep_path = _ep_path
self._ep_path_keys = self.to_keys(_ep_path)
def __init__(self, _ep_spec: dict, _ts_format='%Y-%m-%dT%H:%M:%S.%f%z'):
self._ep_path = _ep_spec["values"]
self._ep_path_keys = self.to_keys(self._ep_path)
self._timestamps = []
self._values = []
self._ts_format = _ts_format
self._ts_path_keys = self.to_keys(_ts_path)
self._ts_path_keys = self.to_keys(_ep_spec["timestamps"])
def timestamps_str(self):
def timestamps_str(self) -> str:
return f"{self._ep_path}:{len(self._timestamps)}: {self._timestamps}"
@property
def timestamps(self):
def timestamps(self) -> list[int]:
return self._timestamps
@property
def values(self):
def values(self) -> list[float]:
return self._values
@property
def path(self):
def path(self) -> str:
return self._ep_path
def assign(self, _record: dict):
@@ -89,115 +105,112 @@ class Endpoint:
return Endpoint.deref_multi(data[keys[0]], keys[1:]) if keys else data
end_points = {
# 'odo': Endpoint('data/measurements/odometerStatus/value/odometer', 'data/measurements/odometerStatus/value/carCapturedTimestamp'),
# 'soc': Endpoint('data/measurements/fuelLevelStatus/value/currentSOC_pct', 'data/measurements/fuelLevelStatus/value/carCapturedTimestamp'),
'odo': Endpoint('data/measurements/odometerStatus/value/odometer','info/timestamp'),
'soc': Endpoint('data/measurements/fuelLevelStatus/value/currentSOC_pct','info/timestamp'),
}
def list_folder_ymd(_path: str):
day_list = sorted(os.listdir(_path), key=lambda f: int(f))
return day_list
def convert(_filename: str):
name = _filename
if "records_" in _filename:
name = os.path.splitext(_filename)[0].split("_")[1]
date_str = name.split("T")[0]
time_str = name.split("T")[1]
if "-" in time_str:
time_str = time_str.replace("-", ":")
def check_sel(root_path, sel: list):
res = []
dir_entries = list_folder_ymd(root_path)
if sel is None:
res = dir_entries
else:
if isinstance(sel, tuple):
if len(sel) == 2:
res = [str(e) for e in range(sel[0], sel[1]+1, 1)]
else:
res = [str(e) for e in dir_entries if int(e) >= int(sel[0])]
timestamp_str = f"{date_str}T{time_str}"
ts = dup.parse(timestamp_str)
result = ts.year
result = 100*result + ts.month
result = 100*result + ts.day
result = 100*result + ts.hour
result = 100*result + ts.minute
result = 100*result + ts.second
result = 1000000*result + ts.microsecond
return result
return res
path = os.path.join(BASE, USER)
path = os.path.join(path, VIN)
path = os.path.join(path, str(YEAR))
path = os.path.join(path, str(MONTH))
def process(end_points, base_path: str, user: str, vin: str, sel_years=None, sel_months=None, sel_days=None, sel_hours=None):
_path = os.path.join(base_path, user)
root_path = os.path.join(_path, vin)
day_list = sorted(os.listdir(path), key=lambda f: int(f))
print("Days in '", path, "' :")
# prints all files
print(day_list)
for _year in check_sel(root_path, sel_years):
year_path = os.path.join(root_path, _year)
if not os.path.exists(year_path):
continue
for _month in check_sel(year_path, sel_months):
month_path = os.path.join(year_path, _month)
if not os.path.exists(month_path):
continue
for _day in check_sel(month_path, sel_days):
day_path = os.path.join(month_path, _day)
if not os.path.exists(day_path):
continue
month_path = path
record_list = []
last = {}
# Sort out files with unknown nickname
file_list_sorted = sorted(os.listdir(day_path), key=lambda f: convert(f))
vehicle_diff = {}
last = {}
for filename in file_list_sorted:
file_path = os.path.join(day_path, filename)
print(f"Open file: {filename}")
with open(file_path, "r") as fp:
records = json.load(fp)
print(f"Found {len(records)} records")
for record in records:
try:
vehicle_diff = record['data']
is_header = True
if 'is_delta' in record['info']:
if record['info']['is_delta']:
is_header = False
for _day in DAYS:
day = str(_day)
if day not in day_list:
continue
day_path = os.path.join(month_path, day)
if is_header:
last = {}
# Sort out files with unknown nickname
file_list = os.listdir(day_path)
file_list_sorted = sorted(file_list, key=lambda f: convert(f))
print("Files in '", day_path, "' :")
# prints all files
print(file_list_sorted)
except KeyError as e:
print(f"KeyError: {e}")
for filename in file_list_sorted:
file_path = os.path.join(day_path, filename)
print(f"Open file: {filename}")
with open(file_path, "r") as fp:
records = json.load(fp)
print(f"Found {len(records)} records")
for record in records:
try:
vehicle_diff = record['data']
is_header = True
if 'is_delta' in record['info']:
if record['info']['is_delta']:
is_header = False
if not has_diff_entries(vehicle_diff):
vehicle_diff = {'update_add': vehicle_diff}
if is_header:
last = {}
vehicle_data = jay_merge_full(last, vehicle_diff)
last = vehicle_data
try:
for k in end_points.keys():
end_points[k].assign({'info': record['info'], 'data': vehicle_data})
except KeyError:
pass
except KeyError as e:
print(f"KeyError: {e}")
for k in end_points.keys():
end_points[k].remove_dups()
if not has_diff_entries(vehicle_diff):
vehicle_diff = {'update_add': vehicle_diff}
vehicle_data = jay_merge_full(last, vehicle_diff)
last = vehicle_data
try:
for k in end_points.keys():
end_points[k].assign({'info': record['info'], 'data': vehicle_data})
except KeyError:
pass
if __name__ == '__main__':
end_points = {}
for k in settings.eps.keys():
end_points[k] = Endpoint(settings.eps[k])
for k in end_points.keys():
end_points[k].remove_dups()
process(end_points, settings.BASE, settings.USER, settings.VIN, sel_days=settings.sel_days)
speed = [0]
for i in range(1, len(end_points['odo'].timestamps)):
dt = float(end_points['odo'].timestamps[i] - end_points['odo'].timestamps[i-1])/(60*100000000)
dv = end_points['odo'].values[i] - end_points['odo'].values[i-1]
_speed = dv/dt
speed.append(_speed)
speed = [0]
_speed = 0
for i in range(1, len(end_points['odo'].timestamps)):
dt = float(end_points['odo'].timestamps[i] - end_points['odo'].timestamps[i-1])/(60*100000000)
dv = end_points['odo'].values[i] - end_points['odo'].values[i-1]
_speed = 0.9*_speed + 0.1*dv/dt
speed.append(_speed)
plot.subplot(3, 1, 1)
plot.plot(end_points['odo'].timestamps, np.array(end_points['odo'].values) - end_points['odo'].values[0])
plot.title("Kilometerstand")
plot.grid()
plot.subplot(3, 1, 2)
plot.plot(end_points['odo'].timestamps, speed)
plot.title("Speed")
plot.grid()
plot.subplot(3, 1, 3)
plot.plot(end_points['soc'].timestamps, end_points['soc'].values)
plot.title("Akkustand")
plot.grid()
plot.subplot(4, 1, 1)
plot.plot(np.array(end_points['odo'].timestamps) - end_points['odo'].timestamps[0], np.array(end_points['odo'].values) - end_points['odo'].values[0])
plot.title("Kilometerstand")
plot.grid()
plot.subplot(4, 1, 2)
plot.plot(np.array(end_points['odo'].timestamps) - end_points['odo'].timestamps[0], speed)
plot.title("Speed")
plot.grid()
plot.subplot(4, 1, 3)
plot.plot(np.array(end_points['soc'].timestamps) - end_points['soc'].timestamps[0], end_points['soc'].values)
plot.title("Akkustand")
plot.grid()
plot.subplot(4, 1, 4)
plot.plot(np.array(end_points['range'].timestamps) - end_points['range'].timestamps[0], end_points['range'].values)
plot.title("Range")
plot.grid()
plot.show()
plot.show()
eval_records_settings.py
+28
View File
@@ -0,0 +1,28 @@
# Settings for eval_record
BASE = "/media/jens/cifs/jens@moon/projects/we_collect/results"
VIN = 'WVWZZZE1ZMP010760'
USER = 'alex'
sel_days = (23, 24)
eps = {
"odo":
{
"values": "data/measurements/odometerStatus/value/odometer",
"timestamps": "data/measurements/odometerStatus/value/carCapturedTimestamp"
},
"soc":
{
"values": "data/measurements/fuelLevelStatus/value/currentSOC_pct",
"timestamps": "data/measurements/fuelLevelStatus/value/carCapturedTimestamp"
},
"range":
{
"values": "data/fuelStatus/rangeStatus/value/totalRange_km",
"timestamps": "data/fuelStatus/rangeStatus/value/carCapturedTimestamp"
}
}
# 'odo': Endpoint('data/measurements/odometerStatus/value/odometer','info/timestamp'),
# 'soc': Endpoint('data/measurements/fuelLevelStatus/value/currentSOC_pct','info/timestamp'),