diff --git a/components/actor/hendiCtrl.py b/components/actor/hendiCtrl.py index bb4c8a0..afec45a 100755 --- a/components/actor/hendiCtrl.py +++ b/components/actor/hendiCtrl.py @@ -213,9 +213,23 @@ class HendiCtrl: if self.ser.is_open: try: self.remoteEnable(False) - except HendiException: + except Exception: + # Not just HendiException (a protocol-level NAK) - a genuine + # unplug fails deep inside pyserial itself (termios.error/ + # OSError from flushInput()/readline() on a dead fd), and + # disconnect() must never let that escape: every caller here + # (HeaterHendi.process()/activate()'s own except handlers, + # AHeater.open()'s context-manager exit) calls disconnect() + # specifically to recover from a comm failure, so if this + # raises too it propagates all the way out of HeaterTask's + # on_process() and permanently kills that task's loop - the + # device then never gets process()'d again even after a + # later successful reconnect. + pass + try: + self.ser.close() + except Exception: pass - self.ser.close() self.sw_id = None self.sw_ver = None diff --git a/tasks/heater.py b/tasks/heater.py index ed67cef..24b7c92 100755 --- a/tasks/heater.py +++ b/tasks/heater.py @@ -122,35 +122,57 @@ class HeaterTask(ATask): # connected. await self.send({'Connected': self.device.connected}) - with self.device.open(): - while True: - # Closed-loop: TC has full control. Open-loop: direct manual power only. - power_soll = self.power_actor if self.closed_loop else self.power_soll - self.power_set_changed(power_soll) + # Outer retry loop - belt-and-suspenders around the tick loop's own + # per-call guards below. A comm failure a device-level guard didn't + # anticipate (e.g. a raw pyserial/termios exception escaping from + # somewhere other than device.process(), the one call already + # wrapped below) must never be allowed to fall out of this + # coroutine entirely: nothing restarts a dead ATask, so a single + # uncaught exception here would permanently stop this device from + # ever being process()'d again - even after a later, otherwise- + # successful reconnect. See the incident this was added for: a + # mid-brew unplug raised termios.error from deep inside + # HendiCtrl.disconnect() itself (fixed there too), which escaped + # all the way out of this loop and killed it for the rest of the + # process's life. + while True: + try: + with self.device.open(): + while True: + # Closed-loop: TC has full control. Open-loop: direct manual power only. + power_soll = self.power_actor if self.closed_loop else self.power_soll + self.power_set_changed(power_soll) - # Calculate duty cycle - power_low = self.get_next_smaller_power(power_soll) - power_high = self.get_next_greater_power(power_soll) - power_step = power_high - power_low - power_diff = power_soll - power_low - duty = power_diff / power_step + # Calculate duty cycle + power_low = self.get_next_smaller_power(power_soll) + power_high = self.get_next_greater_power(power_soll) + power_step = power_high - power_low + power_diff = power_soll - power_low + duty = power_diff / power_step - on_count = pulse_period_count*duty + on_count = pulse_period_count*duty - self.pulse_counter += 1 - if self.pulse_counter >= pulse_period_count: - self.pulse_counter = 0 + self.pulse_counter += 1 + if self.pulse_counter >= pulse_period_count: + self.pulse_counter = 0 - if power_soll == 0 or self.pulse_counter >= on_count: - self.device.set_power(power_low) - elif self.pulse_counter < on_count: - self.device.set_power(power_high) + if power_soll == 0 or self.pulse_counter >= on_count: + self.device.set_power(power_low) + elif self.pulse_counter < on_count: + self.device.set_power(power_high) + try: + self.device.process() + except Exception as e: + print(f"HeaterTask: comm error, marking disconnected: {e}") + self.device.disconnect() + await asyncio.sleep(self.interval) + except Exception as e: + print(f"HeaterTask: unexpected error, recovering: {e}") try: - self.device.process() - except Exception as e: - print(f"HeaterTask: comm error, marking disconnected: {e}") self.device.disconnect() + except Exception: + pass await asyncio.sleep(self.interval) diff --git a/tasks/stirrer.py b/tasks/stirrer.py index ed847d9..b52218b 100644 --- a/tasks/stirrer.py +++ b/tasks/stirrer.py @@ -66,11 +66,24 @@ class StirrerTask(ATask): # observer. await self.send({'Connected': self.device.connected}) - with self.device.open(): - while True: + # Outer retry loop - see HeaterTask.on_process()'s own comment for + # why: nothing restarts a dead ATask, so an exception escaping this + # coroutine entirely would permanently stop this device from ever + # being process()'d again, even after a later successful reconnect. + while True: + try: + with self.device.open(): + while True: + try: + self.device.process() + except Exception as e: + print(f"StirrerTask: comm error, marking disconnected: {e}") + self.device.disconnect() + await asyncio.sleep(self.interval) + except Exception as e: + print(f"StirrerTask: unexpected error, recovering: {e}") try: - self.device.process() - except Exception as e: - print(f"StirrerTask: comm error, marking disconnected: {e}") self.device.disconnect() + except Exception: + pass await asyncio.sleep(self.interval)