Improve duplicate suppression hot paths and remove redundant outbound queue scans

src/Dispatcher.cpp

@@ -224,7 +224,6 @@ void Dispatcher::processRecvPacket(Packet* pkt) {
 }
 
 void Dispatcher::checkSend() {
-  if (_mgr->getOutboundCount(_ms->getMillis()) == 0) return;  // nothing waiting to send
   if (!millisHasNowPassed(next_tx_time)) return;   // still in 'radio silence' phase (from airtime budget setting)
   if (_radio->isReceiving()) {   // LBT - check if radio is currently mid-receive, or if channel activity
     if (cad_busy_start == 0) {
@@ -303,6 +302,7 @@ Packet* Dispatcher::obtainNewPacket() {
   } else {
     pkt->payload_len = pkt->path_len = 0;
     pkt->_snr = 0;
+    pkt->invalidateCachedHash();
   }
   return pkt;
 }
@@ -330,4 +330,4 @@ unsigned long Dispatcher::futureMillis(int millis_from_now) const {
   return _ms->getMillis() + millis_from_now;
 }
 
-}
+}

src/Mesh.cpp

@@ -633,6 +633,7 @@ void Mesh::sendFlood(Packet* packet, uint32_t delay_millis) {
   packet->header &= ~PH_ROUTE_MASK;
   packet->header |= ROUTE_TYPE_FLOOD;
   packet->path_len = 0;
+  packet->invalidateCachedHash();
 
   _tables->hasSeen(packet); // mark this packet as already sent in case it is rebroadcast back to us
 
@@ -658,6 +659,7 @@ void Mesh::sendFlood(Packet* packet, uint16_t* transport_codes, uint32_t delay_m
   packet->transport_codes[0] = transport_codes[0];
   packet->transport_codes[1] = transport_codes[1];
   packet->path_len = 0;
+  packet->invalidateCachedHash();
 
   _tables->hasSeen(packet); // mark this packet as already sent in case it is rebroadcast back to us
 
@@ -692,6 +694,7 @@ void Mesh::sendDirect(Packet* packet, const uint8_t* path, uint8_t path_len, uin
       pri = 0;
     }
   }
+  packet->invalidateCachedHash();
   _tables->hasSeen(packet); // mark this packet as already sent in case it is rebroadcast back to us
   sendPacket(packet, pri, delay_millis);
 }
@@ -701,6 +704,7 @@ void Mesh::sendZeroHop(Packet* packet, uint32_t delay_millis) {
   packet->header |= ROUTE_TYPE_DIRECT;
 
   packet->path_len = 0;  // path_len of zero means Zero Hop
+  packet->invalidateCachedHash();
 
   _tables->hasSeen(packet); // mark this packet as already sent in case it is rebroadcast back to us
 
@@ -714,6 +718,7 @@ void Mesh::sendZeroHop(Packet* packet, uint16_t* transport_codes, uint32_t delay
   packet->transport_codes[1] = transport_codes[1];
 
   packet->path_len = 0;  // path_len of zero means Zero Hop
+  packet->invalidateCachedHash();
 
   _tables->hasSeen(packet); // mark this packet as already sent in case it is rebroadcast back to us
 

src/Packet.cpp

@@ -8,21 +8,29 @@ Packet::Packet() {
   header = 0;
   path_len = 0;
   payload_len = 0;
+  _hash_cached = false;
 }
 
 int Packet::getRawLength() const {
   return 2 + path_len + payload_len + (hasTransportCodes() ? 4 : 0);
 }
 
 void Packet::calculatePacketHash(uint8_t* hash) const {
+  if (_hash_cached) {
+    memcpy(hash, _cached_hash, MAX_HASH_SIZE);
+    return;
+  }
+
   SHA256 sha;
   uint8_t t = getPayloadType();
   sha.update(&t, 1);
   if (t == PAYLOAD_TYPE_TRACE) {
     sha.update(&path_len, sizeof(path_len));   // CAVEAT: TRACE packets can revisit same node on return path
   }
   sha.update(payload, payload_len);
-  sha.finalize(hash, MAX_HASH_SIZE);
+  sha.finalize(_cached_hash, MAX_HASH_SIZE);
+  _hash_cached = true;
+  memcpy(hash, _cached_hash, MAX_HASH_SIZE);
 }
 
 uint8_t Packet::writeTo(uint8_t dest[]) const {
@@ -39,6 +47,7 @@ uint8_t Packet::writeTo(uint8_t dest[]) const {
 }
 
 bool Packet::readFrom(const uint8_t src[], uint8_t len) {
+  _hash_cached = false;
   uint8_t i = 0;
   header = src[i++];
   if (hasTransportCodes()) {
@@ -57,4 +66,4 @@ bool Packet::readFrom(const uint8_t src[], uint8_t len) {
   return true;   // success
 }
 
-}
+}

src/Packet.h

@@ -55,6 +55,7 @@ class Packet {
    * \param  dest_hash   destination to store the hash (must be MAX_HASH_SIZE bytes)
    */
   void calculatePacketHash(uint8_t* dest_hash) const;
+  void invalidateCachedHash() const { _hash_cached = false; }
 
   /**
    * \returns  one of ROUTE_ values
@@ -76,7 +77,7 @@ class Packet {
    */
   uint8_t getPayloadVer() const { return (header >> PH_VER_SHIFT) & PH_VER_MASK; }
 
-  void markDoNotRetransmit() { header = 0xFF; }
+  void markDoNotRetransmit() { header = 0xFF; invalidateCachedHash(); }
   bool isMarkedDoNotRetransmit() const { return header == 0xFF; }
 
   float getSNR() const { return ((float)_snr) / 4.0f; }
@@ -99,6 +100,11 @@ class Packet {
    * \param  len  the packet length (as returned by writeTo())
    */
   bool readFrom(const uint8_t src[], uint8_t len);
+
+private:
+  // Cache packet hash for repeated duplicate checks while packet content is unchanged.
+  mutable bool _hash_cached;
+  mutable uint8_t _cached_hash[MAX_HASH_SIZE];
 };
 
 }

src/helpers/BaseChatMesh.cpp

@@ -468,6 +468,7 @@ bool BaseChatMesh::importContact(const uint8_t src_buf[], uint8_t len) {
   if (pkt) {
     if (pkt->readFrom(src_buf, len) && pkt->getPayloadType() == PAYLOAD_TYPE_ADVERT) {
       pkt->header |= ROUTE_TYPE_FLOOD;   // simulate it being received flood-mode
+      pkt->invalidateCachedHash();
       getTables()->clear(pkt);  // remove packet hash from table, so we can receive/process it again
       _pendingLoopback = pkt;  // loop-back, as if received over radio
       return true;  // success

src/helpers/SimpleMeshTables.h

@@ -14,14 +14,40 @@ class SimpleMeshTables : public mesh::MeshTables {
   int _next_idx;
   uint32_t _acks[MAX_PACKET_ACKS];
   int _next_ack_idx;
+  uint32_t _hash_prefix_bits[8];
+  uint32_t _ack_lsb_bits[8];
   uint32_t _direct_dups, _flood_dups;
 
+  static bool hasFastBit(const uint32_t bits[], uint8_t key) {
+    return (bits[key >> 5] & (1UL << (key & 31))) != 0;
+  }
+  static void setFastBit(uint32_t bits[], uint8_t key) {
+    bits[key >> 5] |= (1UL << (key & 31));
+  }
+  void rebuildFastBits() {
+    memset(_hash_prefix_bits, 0, sizeof(_hash_prefix_bits));
+    memset(_ack_lsb_bits, 0, sizeof(_ack_lsb_bits));
+    for (int i = 0; i < MAX_PACKET_HASHES; i++) {
+      const uint8_t* hp = &_hashes[i * MAX_HASH_SIZE];
+      if (hp[0] | hp[1] | hp[2] | hp[3] | hp[4] | hp[5] | hp[6] | hp[7]) {
+        setFastBit(_hash_prefix_bits, hp[0]);
+      }
+    }
+    for (int i = 0; i < MAX_PACKET_ACKS; i++) {
+      if (_acks[i] != 0) {
+        setFastBit(_ack_lsb_bits, (uint8_t)_acks[i]);
+      }
+    }
+  }
+
 public:
   SimpleMeshTables() { 
     memset(_hashes, 0, sizeof(_hashes));
     _next_idx = 0;
     memset(_acks, 0, sizeof(_acks));
     _next_ack_idx = 0;
+    memset(_hash_prefix_bits, 0, sizeof(_hash_prefix_bits));
+    memset(_ack_lsb_bits, 0, sizeof(_ack_lsb_bits));
     _direct_dups = _flood_dups = 0;
   }
 
@@ -31,6 +57,7 @@ class SimpleMeshTables : public mesh::MeshTables {
     f.read((uint8_t *) &_next_idx, sizeof(_next_idx));
     f.read((uint8_t *) &_acks[0], sizeof(_acks));
     f.read((uint8_t *) &_next_ack_idx, sizeof(_next_ack_idx));
+    rebuildFastBits();
   }
   void saveTo(File f) {
     f.write(_hashes, sizeof(_hashes));
@@ -44,6 +71,12 @@ class SimpleMeshTables : public mesh::MeshTables {
     if (packet->getPayloadType() == PAYLOAD_TYPE_ACK) {
       uint32_t ack;
       memcpy(&ack, packet->payload, 4);
+      if (!hasFastBit(_ack_lsb_bits, (uint8_t)ack)) {
+        _acks[_next_ack_idx] = ack;
+        _next_ack_idx = (_next_ack_idx + 1) % MAX_PACKET_ACKS;  // cyclic table
+        setFastBit(_ack_lsb_bits, (uint8_t)ack);
+        return false;
+      }
       for (int i = 0; i < MAX_PACKET_ACKS; i++) {
         if (ack == _acks[i]) { 
           if (packet->isRouteDirect()) {
@@ -56,12 +89,19 @@ class SimpleMeshTables : public mesh::MeshTables {
       }
 
       _acks[_next_ack_idx] = ack;
-      _next_ack_idx = (_next_ack_idx + 1) % MAX_PACKET_ACKS;  // cyclic table  
+      _next_ack_idx = (_next_ack_idx + 1) % MAX_PACKET_ACKS;  // cyclic table
+      setFastBit(_ack_lsb_bits, (uint8_t)ack);
       return false;
     }
 
     uint8_t hash[MAX_HASH_SIZE];
     packet->calculatePacketHash(hash);
+    if (!hasFastBit(_hash_prefix_bits, hash[0])) {
+      memcpy(&_hashes[_next_idx * MAX_HASH_SIZE], hash, MAX_HASH_SIZE);
+      _next_idx = (_next_idx + 1) % MAX_PACKET_HASHES;  // cyclic table
+      setFastBit(_hash_prefix_bits, hash[0]);
+      return false;
+    }
 
     const uint8_t* sp = _hashes;
     for (int i = 0; i < MAX_PACKET_HASHES; i++, sp += MAX_HASH_SIZE) {
@@ -77,6 +117,7 @@ class SimpleMeshTables : public mesh::MeshTables {
 
     memcpy(&_hashes[_next_idx*MAX_HASH_SIZE], hash, MAX_HASH_SIZE);
     _next_idx = (_next_idx + 1) % MAX_PACKET_HASHES;  // cyclic table
+    setFastBit(_hash_prefix_bits, hash[0]);
     return false;
   }