we_collect/eval_records.py

import json
import os
import numpy as np
import dateutil.parser as dup
from jay_diff import jay_merge_full
from matplotlib import pyplot as plot

BASE = "/media/jens/cifs/jens@moon/projects/we_collect/results"
VIN = 'WVWZZZE1ZMP010760'
USER = 'alex'
YEAR = 2024
MONTH = 10
DAYS = [15, 16, 17, 18, 19, 20, 21, 22, 23, 24]


def has_diff_entries(diff):
	handled = False
	if "update" in diff:
		handled = True
	if "update_add" in diff:
		handled = True
	if "add" in diff:
		handled = True
	if "delete" in diff:
		handled = True

	return handled


class Endpoint:
	def __init__(self, _ep_path: str, _ts_path: str, _ts_format='%Y-%m-%dT%H:%M:%S.%f%z'):
		self._ep_path = _ep_path
		self._ep_path_keys = self.to_keys(_ep_path)
		self._timestamps = []
		self._values = []

		self._ts_format = _ts_format
		self._ts_path_keys = self.to_keys(_ts_path)

	def timestamps_str(self):
		return f"{self._ep_path}:{len(self._timestamps)}: {self._timestamps}"

	@property
	def timestamps(self):
		return self._timestamps

	@property
	def values(self):
		return self._values

	@property
	def path(self):
		return self._ep_path

	def assign(self, _record: dict):
		value = self.deref_multi(_record, self._ep_path_keys)
		self._values += [value]

		# Extract timestamp
		capture_time = self.deref_multi(_record, self._ts_path_keys)
		timestamp_int = convert(capture_time)
		self._timestamps += [timestamp_int]

	def remove_dups(self):
		new_dict = {}
		index = 0
		unique_indices = []
		for ts in self._timestamps:
			if ts not in new_dict:
				new_dict[ts] = index
				unique_indices.append(index)
			index += 1

		t_list = self._timestamps
		v_list = self._values
		self._timestamps = []
		self._values = {}

		self._timestamps = [t_list[i] for i in unique_indices]
		self._values = [v_list[i] for i in unique_indices]

	@staticmethod
	def to_keys(key_path: str):
		keys = key_path.split('/')
		return keys

	@staticmethod
	def deref_multi(data, keys):
		return Endpoint.deref_multi(data[keys[0]], keys[1:]) if keys else data


end_points = {
#	'odo': Endpoint('data/measurements/odometerStatus/value/odometer', 'data/measurements/odometerStatus/value/carCapturedTimestamp'),
#	'soc': Endpoint('data/measurements/fuelLevelStatus/value/currentSOC_pct', 'data/measurements/fuelLevelStatus/value/carCapturedTimestamp'),
	'odo': Endpoint('data/measurements/odometerStatus/value/odometer','info/timestamp'),
	'soc': Endpoint('data/measurements/fuelLevelStatus/value/currentSOC_pct','info/timestamp'),
}


def convert(_filename: str):
	name = _filename
	if "records_" in _filename:
		name = os.path.splitext(_filename)[0].split("_")[1]
	date_str = name.split("T")[0]
	time_str = name.split("T")[1]
	if "-" in time_str:
		time_str = time_str.replace("-", ":")

	timestamp_str = f"{date_str}T{time_str}"
	ts = dup.parse(timestamp_str)
	result = ts.year
	result = 100*result + ts.month
	result = 100*result + ts.day
	result = 100*result + ts.hour
	result = 100*result + ts.minute
	result = 100*result + ts.second
	result = 1000000*result + ts.microsecond

	return result


path = os.path.join(BASE, USER)
path = os.path.join(path, VIN)
path = os.path.join(path, str(YEAR))
path = os.path.join(path, str(MONTH))

day_list = sorted(os.listdir(path), key=lambda f: int(f))
print("Days in '", path, "' :")
# prints all files
print(day_list)

month_path = path
record_list = []
last = {}

for _day in DAYS:
	day = str(_day)
	if day not in day_list:
		continue
	day_path = os.path.join(month_path, day)

	# Sort out files with unknown nickname
	file_list = os.listdir(day_path)
	file_list_sorted = sorted(file_list, key=lambda f: convert(f))
	print("Files in '", day_path, "' :")
	# prints all files
	print(file_list_sorted)

	for filename in file_list_sorted:
		file_path = os.path.join(day_path, filename)
		print(f"Open file: {filename}")
		with open(file_path, "r") as fp:
			records = json.load(fp)
			print(f"Found {len(records)} records")
			for record in records:
				try:
					vehicle_diff = record['data']
					is_header = True
					if 'is_delta' in record['info']:
						if record['info']['is_delta']:
							is_header = False

					if is_header:
						last = {}

				except KeyError as e:
					print(f"KeyError: {e}")

				if not has_diff_entries(vehicle_diff):
					vehicle_diff = {'update_add': vehicle_diff}

				vehicle_data = jay_merge_full(last, vehicle_diff)
				last = vehicle_data
				try:
					for k in end_points.keys():
						end_points[k].assign({'info': record['info'], 'data': vehicle_data})
				except KeyError:
					pass

for k in end_points.keys():
	end_points[k].remove_dups()

speed = [0]
for i in range(1, len(end_points['odo'].timestamps)):
	dt = float(end_points['odo'].timestamps[i] - end_points['odo'].timestamps[i-1])/(60*100000000)
	dv = end_points['odo'].values[i] - end_points['odo'].values[i-1]
	_speed = dv/dt
	speed.append(_speed)

plot.subplot(3, 1, 1)
plot.plot(end_points['odo'].timestamps, np.array(end_points['odo'].values) - end_points['odo'].values[0])
plot.title("Kilometerstand")
plot.grid()
plot.subplot(3, 1, 2)
plot.plot(end_points['odo'].timestamps, speed)
plot.title("Speed")
plot.grid()
plot.subplot(3, 1, 3)
plot.plot(end_points['soc'].timestamps, end_points['soc'].values)
plot.title("Akkustand")
plot.grid()

plot.show()