diff --git a/collect.py b/collect.py index 1a621c1..2670e6f 100644 --- a/collect.py +++ b/collect.py @@ -57,7 +57,7 @@ from pathlib import Path from server import log_config, network, we_connect from server.data_model import ALL_DOMAINS, collect_snapshot -from server.storage_helpers import auto_out_path, load_store, load_today_records, save_store +from server.storage_helpers import auto_out_path, load_last_24h_records, load_store, save_store log = logging.getLogger(__name__) @@ -95,9 +95,9 @@ def run(args: argparse.Namespace) -> None: push_clients, push_lock, push_store_ref = network.start_push_server(args.host, args.port) if logs_dir is not None: - push_store_ref["preloaded"] = load_today_records(logs_dir, vin) + push_store_ref["preloaded"] = load_last_24h_records(logs_dir, vin) if push_store_ref["preloaded"]: - log.info("Pre-loaded %d record(s) from today's log files", len(push_store_ref["preloaded"])) + log.info("Pre-loaded %d record(s) from the last 24 h", len(push_store_ref["preloaded"])) current_day = datetime.now(timezone.utc).date() store = load_store(out, vin, args.interval) diff --git a/server/storage_helpers.py b/server/storage_helpers.py index 7ca5e46..9544a08 100644 --- a/server/storage_helpers.py +++ b/server/storage_helpers.py @@ -1,6 +1,6 @@ import json import logging -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import Path log = logging.getLogger(__name__) @@ -35,17 +35,29 @@ def save_store(path: Path, store: dict, max_records: int | None) -> None: path.write_text(json.dumps(store, indent=2, default=str)) -def load_today_records(logs_dir: Path, vin: str) -> list: - """Load and merge all records from today's log files for this VIN.""" - today_prefix = datetime.now(timezone.utc).strftime("%Y_%m_%d_") +def load_last_24h_records(logs_dir: Path, vin: str) -> list: + """Load records from the last 24 hours across log files (may span midnight).""" + now = datetime.now(timezone.utc) + cutoff = now - timedelta(hours=24) + cutoff_str = cutoff.isoformat() + + # Log files are named YYYY_MM_DD_…_VIN.json; the 24 h window can span two calendar days. + prefixes = { + cutoff.strftime("%Y_%m_%d_"), + now.strftime("%Y_%m_%d_"), + } + records = [] - for path in sorted(logs_dir.glob(f"{today_prefix}*_{vin}.json")): - try: - data = json.loads(path.read_text()) - if isinstance(data, dict) and "records" in data: - records.extend(data["records"]) - log.debug("Loaded %d records from %s", len(data["records"]), path.name) - except (json.JSONDecodeError, OSError): - log.warning("Could not load %s", path) + for prefix in sorted(prefixes): + for path in sorted(logs_dir.glob(f"{prefix}*_{vin}.json")): + try: + data = json.loads(path.read_text()) + if isinstance(data, dict) and "records" in data: + kept = [r for r in data["records"] if r.get("ts", "") >= cutoff_str] + records.extend(kept) + log.debug("Loaded %d/%d records from %s", len(kept), len(data["records"]), path.name) + except (json.JSONDecodeError, OSError): + log.warning("Could not load %s", path) + records.sort(key=lambda r: r.get("ts", "")) return records \ No newline at end of file