we_collect/eval_records.py

import json
import os
from datetime import datetime
from jay_diff import jay_merge_full

BASE = "/media/jens/cifs/jens@moon/projects/we_collect/results"
VIN = 'WVWZZZE1ZMP010760'
YEAR = 2024
MONTH = 10
#DAY = ['17', '18', '19', '20']
DAY = ['22']

def has_diff_entries(diff):
	handled = False
	if "update" in diff:
		handled = True
	if "update_add" in diff:
		handled = True
	if "add" in diff:
		handled = True
	if "delete" in diff:
		handled = True

	return handled


class Endpoint:
	def __init__(self, _ep_path: str, _ts_path: str, _ts_format='%Y-%m-%dT%H:%M:%S.%f%z'):
		self._ep_path = _ep_path
		self._ep_path_keys = self.to_keys(_ep_path)
		self._timestamps = []
		self._values = []

		self._ts_format = _ts_format
		self._ts_path_keys = self.to_keys(_ts_path)

	def __str__(self):
		return f"{self._ep_path}:{len(self._values)}: {self._values}"

	def timestamps_str(self):
		return f"{self._ep_path}:{len(self._timestamps)}: {self._timestamps}"

	@property
	def path(self):
		return self._ep_path

	def assign(self, _record: dict):
		value = self.deref_multi(_record, self._ep_path_keys)
		self._values += [value]

		# Extract timestamp
		capture_time = self.deref_multi(_record, self._ts_path_keys)
		timestamp_int = convert(capture_time, self._ts_format)
		self._timestamps += [timestamp_int]

	def remove_dups(self):
		new_dict = {}
		index = 0
		unique_indices = []
		for ts in self._timestamps:
			if ts not in new_dict:
				new_dict[ts] = index
				unique_indices.append(index)
			index += 1

		t_list = self._timestamps
		v_list = self._values
		self._timestamps = []
		self._values = {}

		self._timestamps = [t_list[i] for i in unique_indices]
		self._values = [v_list[i] for i in unique_indices]

	@staticmethod
	def to_keys(key_path: str):
		keys = key_path.split('/')
		return keys

	@staticmethod
	def deref_multi(data, keys):
		return Endpoint.deref_multi(data[keys[0]], keys[1:]) if keys else data


end_points = [
	Endpoint('data/measurements/odometerStatus/value/odometer', 'info/timestamp'),
	Endpoint('data/measurements/fuelLevelStatus/value/currentSOC_pct', 'info/timestamp'),
	Endpoint('data/vehicleLights/lightsStatus/value/lights', 'info/timestamp'),
	Endpoint('data/fuelStatus/rangeStatus/value/primaryEngine', 'info/timestamp')
]


def convert(timestamp_str: str, pattern: str = "%Y-%m-%dT%H:%M:%S.%f%z"):
	ts = datetime.strptime(timestamp_str, pattern)
	result = ts.year
	result = 100*result + ts.month
	result = 100*result + ts.day
	result = 100*result + ts.hour
	result = 100*result + ts.minute
	result = 100*result + ts.second
	result = 1000000*result + ts.microsecond

	return result


path = os.path.join(BASE, VIN)
path = os.path.join(path, str(YEAR))
path = os.path.join(path, str(MONTH))

day_list = sorted(os.listdir(path), key=lambda f: int(f))
print("Days in '", path, "' :")
# prints all files
print(day_list)

month_path = path
record_list = []
last = {}

for day in day_list:
	if DAY is not None:
		if day not in DAY:
			continue
	day_path = os.path.join(month_path, day)
	file_list = sorted(os.listdir(day_path), key=lambda f: convert(f, pattern=f"vehicle_{VIN}_%Y-%m-%dT%H-%M-%S.json"))
	print("Files in '", day_path, "' :")
	# prints all files
	print(file_list)

	for filename in file_list:
		file_path = os.path.join(day_path, filename)
		print(f"Open file: {filename}")
		with open(file_path, "r") as fp:
			records = json.load(fp)
			print(f"Found {len(records)} records")
			for record in records:
				try:
					vehicle_diff = record['data']
					is_header = True
					if 'is_delta' in record['info']:
						if record['info']['is_delta']:
							is_header = False
							print(f"Found delta record #{record['info']['record_id']}")

					if is_header:
						print(f"Found full record #{record['info']['record_id']}")
						last = {}

				except KeyError as e:
					print(f"KeyError: {e}")

				if not has_diff_entries(vehicle_diff):
					vehicle_diff = {'update_add': vehicle_diff}

				vehicle_data = jay_merge_full(last, vehicle_diff)
				last = vehicle_data
				try:
					for end_point in end_points:
						end_point.assign({'info': record['info'], 'data': vehicle_data})
				except KeyError:
					pass

for end_point in end_points:
	end_point.remove_dups()
	print(end_point)
	print(end_point.timestamps_str())

d = convert('2024-10-17T18:37:17.067Z')