Prevent Memory Buildup by Clearing Joinsets

simln-lib/src/lib.rs

@@ -1348,7 +1348,7 @@ async fn produce_simulation_results(
 ) -> Result<(), SimulationError> {
     let mut set = tokio::task::JoinSet::new();
 
-    let result = loop {
+    let result = 'outer: loop {
         tokio::select! {
             biased;
             _ = listener.clone() => {
@@ -1388,6 +1388,16 @@ async fn produce_simulation_results(
                 }
             }
         }
+
+        // Do a best-effort attempt to clear out any completed tasks in the joinset before proceeding to process
+        // more results. This prevents the joinset from building up over time and never emptying out any completed
+        // tasks.
+        while let Some(res) = set.try_join_next() {
+            if let Err(e) = res {
+                log::error!("Error collecting joinset: {e}");
+                break 'outer Err(SimulationError::TaskError);
+            }
+        }
     };
 
     log::debug!("Simulation results producer exiting.");

simln-lib/src/sim_node.rs

@@ -817,6 +817,14 @@ impl SimNetwork for SimGraph {
             },
         };
 
+        // This fixes it, but is a super weird way to do this!
+        while let Some(res) = self.tasks.try_join_next() {
+            if let Err(e) = res {
+                log::error!("Error completing task {e}");
+                self.shutdown_trigger.trigger();
+            }
+        }
+
         self.tasks.spawn(propagate_payment(
             self.channels.clone(),
             source,