fix: add zombie radio recovery for TX fail reset and RX stuck reboot

examples/companion_radio/DataStore.cpp

@@ -231,6 +231,13 @@ void DataStore::loadPrefsInt(const char *filename, NodePrefs& _prefs, double& no
     file.read((uint8_t *)&_prefs.autoadd_config, sizeof(_prefs.autoadd_config));           // 87
     file.read((uint8_t *)&_prefs.autoadd_max_hops, sizeof(_prefs.autoadd_max_hops));       // 88
     file.read((uint8_t *)&_prefs.rx_boosted_gain, sizeof(_prefs.rx_boosted_gain)); // 89
+    if (!file.read((uint8_t *)&_prefs.tx_fail_reset_threshold, sizeof(_prefs.tx_fail_reset_threshold))) { // 90
+      _prefs.tx_fail_reset_threshold = 3;
+    }
+    if (!file.read((uint8_t *)&_prefs.rx_fail_reboot_threshold, sizeof(_prefs.rx_fail_reboot_threshold))) { // 91
+      _prefs.rx_fail_reboot_threshold = 3;
+    }
+    // next: 92
 
     file.close();
   }
@@ -269,6 +276,9 @@ void DataStore::savePrefs(const NodePrefs& _prefs, double node_lat, double node_
     file.write((uint8_t *)&_prefs.autoadd_config, sizeof(_prefs.autoadd_config));           // 87
     file.write((uint8_t *)&_prefs.autoadd_max_hops, sizeof(_prefs.autoadd_max_hops));       // 88
     file.write((uint8_t *)&_prefs.rx_boosted_gain, sizeof(_prefs.rx_boosted_gain)); // 89
+    file.write((uint8_t *)&_prefs.tx_fail_reset_threshold, sizeof(_prefs.tx_fail_reset_threshold)); // 90
+    file.write((uint8_t *)&_prefs.rx_fail_reboot_threshold, sizeof(_prefs.rx_fail_reboot_threshold)); // 91
+    // next: 92
 
     file.close();
   }

examples/companion_radio/MyMesh.cpp

@@ -257,6 +257,15 @@ float MyMesh::getAirtimeBudgetFactor() const {
 int MyMesh::getInterferenceThreshold() const {
   return 0; // disabled for now, until currentRSSI() problem is resolved
 }
+uint8_t MyMesh::getTxFailResetThreshold() const {
+  return _prefs.tx_fail_reset_threshold;
+}
+uint8_t MyMesh::getRxFailRebootThreshold() const {
+  return _prefs.rx_fail_reboot_threshold;
+}
+void MyMesh::onRxUnrecoverable() {
+  board.reboot();
+}
 
 int MyMesh::calcRxDelay(float score, uint32_t air_time) const {
   if (_prefs.rx_delay_base <= 0.0f) return 0;

examples/companion_radio/MyMesh.h

@@ -105,6 +105,9 @@ class MyMesh : public BaseChatMesh, public DataStoreHost {
 protected:
   float getAirtimeBudgetFactor() const override;
   int getInterferenceThreshold() const override;
+  uint8_t getTxFailResetThreshold() const override;
+  uint8_t getRxFailRebootThreshold() const override;
+  void onRxUnrecoverable() override;
   int calcRxDelay(float score, uint32_t air_time) const override;
   uint32_t getRetransmitDelay(const mesh::Packet *packet) override;
   uint32_t getDirectRetransmitDelay(const mesh::Packet *packet) override;

examples/companion_radio/NodePrefs.h

@@ -32,4 +32,6 @@ struct NodePrefs {  // persisted to file
   uint8_t client_repeat;
   uint8_t path_hash_mode;    // which path mode to use when sending
   uint8_t autoadd_max_hops;  // 0 = no limit, 1 = direct (0 hops), N = up to N-1 hops (max 64)
+  uint8_t tx_fail_reset_threshold;  // 0=disabled, 1-10, default 3
+  uint8_t rx_fail_reboot_threshold; // 0=disabled, 1-10, default 3
 };

examples/simple_repeater/MyMesh.cpp

@@ -1066,6 +1066,10 @@ void MyMesh::removeNeighbor(const uint8_t *pubkey, int key_len) {
 #endif
 }
 
+void MyMesh::onRxUnrecoverable() {
+  board.reboot();
+}
+
 void MyMesh::formatStatsReply(char *reply) {
   StatsFormatHelper::formatCoreStats(reply, board, *_ms, _err_flags, _mgr);
 }

examples/simple_repeater/MyMesh.h

@@ -153,6 +153,13 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks {
   int getAGCResetInterval() const override {
     return ((int)_prefs.agc_reset_interval) * 4000;   // milliseconds
   }
+  uint8_t getTxFailResetThreshold() const override {
+    return _prefs.tx_fail_reset_threshold;
+  }
+  uint8_t getRxFailRebootThreshold() const override {
+    return _prefs.rx_fail_reboot_threshold;
+  }
+  void onRxUnrecoverable() override;
   uint8_t getExtraAckTransmitCount() const override {
     return _prefs.multi_acks;
   }

examples/simple_room_server/MyMesh.cpp

@@ -199,6 +199,10 @@ int MyMesh::handleRequest(ClientInfo *sender, uint32_t sender_timestamp, uint8_t
   return 0; // unknown command
 }
 
+void MyMesh::onRxUnrecoverable() {
+  board.reboot();
+}
+
 void MyMesh::logRxRaw(float snr, float rssi, const uint8_t raw[], int len) {
 #if MESH_PACKET_LOGGING
   Serial.print(getLogDateTime());

examples/simple_room_server/MyMesh.h

@@ -140,6 +140,13 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks {
   int getAGCResetInterval() const override {
     return ((int)_prefs.agc_reset_interval) * 4000;   // milliseconds
   }
+  uint8_t getTxFailResetThreshold() const override {
+    return _prefs.tx_fail_reset_threshold;
+  }
+  uint8_t getRxFailRebootThreshold() const override {
+    return _prefs.rx_fail_reboot_threshold;
+  }
+  void onRxUnrecoverable() override;
   uint8_t getExtraAckTransmitCount() const override {
     return _prefs.multi_acks;
   }

examples/simple_sensor/SensorMesh.cpp

@@ -326,6 +326,15 @@ int SensorMesh::getInterferenceThreshold() const {
 int SensorMesh::getAGCResetInterval() const {
   return ((int)_prefs.agc_reset_interval) * 4000;   // milliseconds
 }
+uint8_t SensorMesh::getTxFailResetThreshold() const {
+  return _prefs.tx_fail_reset_threshold;
+}
+uint8_t SensorMesh::getRxFailRebootThreshold() const {
+  return _prefs.rx_fail_reboot_threshold;
+}
+void SensorMesh::onRxUnrecoverable() {
+  board.reboot();
+}
 
 uint8_t SensorMesh::handleLoginReq(const mesh::Identity& sender, const uint8_t* secret, uint32_t sender_timestamp, const uint8_t* data, bool is_flood) {
   ClientInfo* client;

examples/simple_sensor/SensorMesh.h

@@ -120,6 +120,9 @@ class SensorMesh : public mesh::Mesh, public CommonCLICallbacks {
   uint32_t getDirectRetransmitDelay(const mesh::Packet* packet) override;
   int getInterferenceThreshold() const override;
   int getAGCResetInterval() const override;
+  uint8_t getTxFailResetThreshold() const override;
+  uint8_t getRxFailRebootThreshold() const override;
+  void onRxUnrecoverable() override;
   void onAnonDataRecv(mesh::Packet* packet, const uint8_t* secret, const mesh::Identity& sender, uint8_t* data, size_t len) override;
   int searchPeersByHash(const uint8_t* hash) override;
   void getPeerSharedSecret(uint8_t* dest_secret, int peer_idx) override;

src/Dispatcher.cpp

@@ -76,10 +76,28 @@ void Dispatcher::loop() {
     prev_isrecv_mode = is_recv;
     if (!is_recv) {
       radio_nonrx_start = _ms->getMillis();
+    } else {
+      rx_stuck_count = 0;  // radio recovered — reset counter
     }
   }
   if (!is_recv && _ms->getMillis() - radio_nonrx_start > 8000) {   // radio has not been in Rx mode for 8 seconds!
     _err_flags |= ERR_EVENT_STARTRX_TIMEOUT;
+
+    rx_stuck_count++;
+    MESH_DEBUG_PRINTLN("%s Dispatcher::loop(): RX stuck (attempt %d), calling onRxStuck()", getLogDateTime(), rx_stuck_count);
+    onRxStuck();
+
+    uint8_t reboot_threshold = getRxFailRebootThreshold();
+    if (reboot_threshold > 0 && rx_stuck_count >= reboot_threshold) {
+      MESH_DEBUG_PRINTLN("%s Dispatcher::loop(): RX unrecoverable after %d attempts", getLogDateTime(), rx_stuck_count);
+      onRxUnrecoverable();
+    }
+
+    // Reset state to give recovery the full 8s window before re-triggering
+    radio_nonrx_start = _ms->getMillis();
+    prev_isrecv_mode = true;
+    cad_busy_start = 0;
+    next_agc_reset_time = futureMillis(getAGCResetInterval());
   }
 
   if (outbound) {  // waiting for outbound send to be completed
@@ -327,14 +345,31 @@ void Dispatcher::checkSend() {
       outbound_start = _ms->getMillis();
       bool success = _radio->startSendRaw(raw, len);
       if (!success) {
-        MESH_DEBUG_PRINTLN("%s Dispatcher::loop(): ERROR: send start failed!", getLogDateTime());
+        MESH_DEBUG_PRINTLN("%s Dispatcher::checkSend(): ERROR: send start failed!", getLogDateTime());
 
         logTxFail(outbound, outbound->getRawLength());
-
-        releasePacket(outbound);  // return to pool
+
+        // re-queue packet for retry instead of dropping it
+        int retry_delay = getCADFailRetryDelay();
+        unsigned long retry_time = futureMillis(retry_delay);
+        _mgr->queueOutbound(outbound, 0, retry_time);
         outbound = NULL;
+        next_tx_time = retry_time;
+
+        // count consecutive failures and reset radio if stuck
+        uint8_t threshold = getTxFailResetThreshold();
+        if (threshold > 0) {
+          tx_fail_count++;
+          if (tx_fail_count >= threshold) {
+            MESH_DEBUG_PRINTLN("%s Dispatcher::checkSend(): TX stuck (%d failures), resetting radio", getLogDateTime(), tx_fail_count);
+            onTxStuck();
+            tx_fail_count = 0;
+            next_tx_time = futureMillis(2000);
+          }
+        }
         return;
       }
+      tx_fail_count = 0;  // clear counter on successful TX start
       outbound_expiry = futureMillis(max_airtime);
 
     #if MESH_PACKET_LOGGING

src/Dispatcher.h

@@ -126,6 +126,8 @@ class Dispatcher {
   unsigned long tx_budget_ms;
   unsigned long last_budget_update;
   unsigned long duty_cycle_window_ms;
+  uint8_t tx_fail_count;
+  uint8_t rx_stuck_count;
 
   void processRecvPacket(Packet* pkt);
   void updateTxBudget();
@@ -150,6 +152,8 @@ class Dispatcher {
     tx_budget_ms = 0;
     last_budget_update = 0;
     duty_cycle_window_ms = 3600000;
+    tx_fail_count = 0;
+    rx_stuck_count = 0;
   }
 
   virtual DispatcherAction onRecvPacket(Packet* pkt) = 0;
@@ -167,6 +171,11 @@ class Dispatcher {
   virtual uint32_t getCADFailMaxDuration() const;
   virtual int getInterferenceThreshold() const { return 0; }    // disabled by default
   virtual int getAGCResetInterval() const { return 0; }    // disabled by default
+  virtual uint8_t getTxFailResetThreshold() const { return 3; }  // reset radio after N consecutive TX failures; 0=disabled
+  virtual void onTxStuck() { _radio->resetAGC(); }               // override to use doFullRadioReset() when available
+  virtual uint8_t getRxFailRebootThreshold() const { return 3; } // reboot after N failed RX recovery attempts; 0=disabled
+  virtual void onRxStuck() { _radio->resetAGC(); }               // called each time RX stuck for 8s; override for deeper reset
+  virtual void onRxUnrecoverable() { }                           // called when reboot threshold exceeded; override to call _board->reboot()
   virtual unsigned long getDutyCycleWindowMs() const { return 3600000; }
 
 public:

src/helpers/CommonCLI.cpp

@@ -88,7 +88,13 @@ void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) {
     file.read((uint8_t *)&_prefs->adc_multiplier, sizeof(_prefs->adc_multiplier));                 // 166
     file.read((uint8_t *)_prefs->owner_info, sizeof(_prefs->owner_info));                          // 170
     file.read((uint8_t *)&_prefs->rx_boosted_gain, sizeof(_prefs->rx_boosted_gain));              // 290
-    // next: 291
+    if (!file.read((uint8_t *)&_prefs->tx_fail_reset_threshold, sizeof(_prefs->tx_fail_reset_threshold))) { // 291
+      _prefs->tx_fail_reset_threshold = 3;
+    }
+    if (!file.read((uint8_t *)&_prefs->rx_fail_reboot_threshold, sizeof(_prefs->rx_fail_reboot_threshold))) { // 292
+      _prefs->rx_fail_reboot_threshold = 3;
+    }
+    // next: 293
 
     // sanitise bad pref values
     _prefs->rx_delay_base = constrain(_prefs->rx_delay_base, 0, 20.0f);
@@ -179,7 +185,9 @@ void CommonCLI::savePrefs(FILESYSTEM* fs) {
     file.write((uint8_t *)&_prefs->adc_multiplier, sizeof(_prefs->adc_multiplier));                 // 166
     file.write((uint8_t *)_prefs->owner_info, sizeof(_prefs->owner_info));                          // 170
     file.write((uint8_t *)&_prefs->rx_boosted_gain, sizeof(_prefs->rx_boosted_gain));              // 290
-    // next: 291
+    file.write((uint8_t *)&_prefs->tx_fail_reset_threshold, sizeof(_prefs->tx_fail_reset_threshold)); // 291
+    file.write((uint8_t *)&_prefs->rx_fail_reboot_threshold, sizeof(_prefs->rx_fail_reboot_threshold)); // 292
+    // next: 293
 
     file.close();
   }
@@ -300,6 +308,10 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, const char* command, ch
         sprintf(reply, "> %d", (uint32_t) _prefs->interference_threshold);
       } else if (memcmp(config, "agc.reset.interval", 18) == 0) {
         sprintf(reply, "> %d", ((uint32_t) _prefs->agc_reset_interval) * 4);
+      } else if (memcmp(config, "tx.fail.threshold", 17) == 0) {
+        sprintf(reply, "> %d", (uint32_t) _prefs->tx_fail_reset_threshold);
+      } else if (memcmp(config, "rx.fail.threshold", 17) == 0) {
+        sprintf(reply, "> %d", (uint32_t) _prefs->rx_fail_reboot_threshold);
       } else if (memcmp(config, "multi.acks", 10) == 0) {
         sprintf(reply, "> %d", (uint32_t) _prefs->multi_acks);
       } else if (memcmp(config, "allow.read.only", 15) == 0) {
@@ -463,6 +475,28 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, const char* command, ch
         _prefs->agc_reset_interval = atoi(&config[19]) / 4;
         savePrefs();
         sprintf(reply, "OK - interval rounded to %d", ((uint32_t) _prefs->agc_reset_interval) * 4);
+      } else if (memcmp(config, "tx.fail.threshold ", 18) == 0) {
+        int val = atoi(&config[18]);
+        if (val < 0) val = 0;
+        if (val > 10) val = 10;
+        _prefs->tx_fail_reset_threshold = (uint8_t) val;
+        savePrefs();
+        if (val == 0) {
+          strcpy(reply, "OK - tx fail reset disabled");
+        } else {
+          sprintf(reply, "OK - tx fail reset after %d failures", val);
+        }
+      } else if (memcmp(config, "rx.fail.threshold ", 18) == 0) {
+        int val = atoi(&config[18]);
+        if (val < 0) val = 0;
+        if (val > 10) val = 10;
+        _prefs->rx_fail_reboot_threshold = (uint8_t) val;
+        savePrefs();
+        if (val == 0) {
+          strcpy(reply, "OK - rx fail reboot disabled");
+        } else {
+          sprintf(reply, "OK - reboot after %d rx recovery failures", val);
+        }
       } else if (memcmp(config, "multi.acks ", 11) == 0) {
         _prefs->multi_acks = atoi(&config[11]);
         savePrefs();

src/helpers/CommonCLI.h

@@ -60,6 +60,8 @@ struct NodePrefs { // persisted to file
   uint8_t rx_boosted_gain; // power settings
   uint8_t path_hash_mode;   // which path mode to use when sending
   uint8_t loop_detect;
+  uint8_t tx_fail_reset_threshold; // 0=disabled, 1-10, default 3
+  uint8_t rx_fail_reboot_threshold; // 0=disabled, 1-10, default 3
 };
 
 class CommonCLICallbacks {