rt: Introduce rust_sched_reaper

This just moves the responsibility for joining with scheduler threads
off to a worker thread. This will be needed when we allow tasks to be
scheduled on the main thread.

mk/rt.mk

@@ -53,6 +53,7 @@ RUNTIME_CS_$(1) := \
               rt/rust_sched_loop.cpp \
               rt/rust_sched_launcher.cpp \
               rt/rust_scheduler.cpp \
+              rt/rust_sched_reaper.cpp \
               rt/rust_task.cpp \
               rt/rust_stack.cpp \
               rt/rust_port.cpp \

src/rt/rust.cpp

@@ -93,7 +93,7 @@ rust_start(uintptr_t main_fn, int argc, char **argv, void* crate_map) {
     root_task->start((spawn_fn)main_fn, NULL, args->args);
     root_task = NULL;
 
-    int ret = kernel->wait_for_schedulers();
+    int ret = kernel->wait_for_exit();
     delete args;
     delete kernel;
     delete srv;

src/rt/rust_kernel.cpp

@@ -17,6 +17,7 @@ rust_kernel::rust_kernel(rust_srv *srv) :
     max_port_id(0),
     rval(0),
     max_sched_id(0),
+    sched_reaper(this),
     env(srv->env)
 {
 }
@@ -62,6 +63,9 @@ rust_kernel::create_scheduler(size_t num_threads) {
     rust_scheduler *sched;
     {
         scoped_lock with(sched_lock);
+        // If this is the first scheduler then we need to launch
+        // the scheduler reaper.
+        bool start_reaper = sched_table.empty();
         id = max_sched_id++;
         K(srv, id != INTPTR_MAX, "Hit the maximum scheduler id");
         sched = new (this, "rust_scheduler")
@@ -69,6 +73,9 @@ rust_kernel::create_scheduler(size_t num_threads) {
         bool is_new = sched_table
             .insert(std::pair<rust_sched_id, rust_scheduler*>(id, sched)).second;
         A(this, is_new, "Reusing a sched id?");
+        if (start_reaper) {
+            sched_reaper.start();
+        }
     }
     sched->start_task_threads();
     return id;
@@ -96,12 +103,12 @@ rust_kernel::release_scheduler_id(rust_sched_id id) {
 }
 
 /*
-Called on the main thread to wait for the kernel to exit. This function is
-also used to join on every terminating scheduler thread, so that we can be
-sure they have completely exited before the process exits.  If we don't join
-them then we can see valgrind errors due to un-freed pthread memory.
+Called by rust_sched_reaper to join every every terminating scheduler thread,
+so that we can be sure they have completely exited before the process exits.
+If we don't join them then we can see valgrind errors due to un-freed pthread
+memory.
  */
-int
+void
 rust_kernel::wait_for_schedulers()
 {
     scoped_lock with(sched_lock);
@@ -120,6 +127,12 @@ rust_kernel::wait_for_schedulers()
             sched_lock.wait();
         }
     }
+}
+
+/* Called on the main thread to wait for the kernel to exit */
+int
+rust_kernel::wait_for_exit() {
+    sched_reaper.join();
     return rval;
 }
 

src/rt/rust_kernel.h

@@ -6,6 +6,7 @@
 #include <vector>
 #include "memory_region.h"
 #include "rust_log.h"
+#include "rust_sched_reaper.h"
 
 struct rust_task_thread;
 class rust_scheduler;
@@ -46,6 +47,8 @@ private:
     // A list of scheduler ids that are ready to exit
     std::vector<rust_sched_id> join_list;
 
+    rust_sched_reaper sched_reaper;
+
 public:
 
     struct rust_env *env;
@@ -66,7 +69,8 @@ public:
     rust_scheduler* get_scheduler_by_id(rust_sched_id id);
     // Called by a scheduler to indicate that it is terminating
     void release_scheduler_id(rust_sched_id id);
-    int wait_for_schedulers();
+    void wait_for_schedulers();
+    int wait_for_exit();
 
 #ifdef __WIN32__
     void win32_require(LPCTSTR fn, BOOL ok);

src/rt/rust_sched_reaper.cpp

@@ -0,0 +1,15 @@
+#include "rust_internal.h"
+#include "rust_kernel.h"
+#include "rust_sched_reaper.h"
+
+// NB: We're using a very small stack here
+const size_t STACK_SIZE = 1024*20;
+
+rust_sched_reaper::rust_sched_reaper(rust_kernel *kernel)
+    : rust_thread(STACK_SIZE), kernel(kernel) {
+}
+
+void
+rust_sched_reaper::run() {
+    kernel->wait_for_schedulers();
+}

src/rt/rust_sched_reaper.h

@@ -0,0 +1,17 @@
+#ifndef RUST_SCHED_REAPER_H
+#define RUST_SCHED_REAPER_H
+
+#include "sync/rust_thread.h"
+
+class rust_kernel;
+
+/* Responsible for joining with rust_schedulers */
+class rust_sched_reaper : public rust_thread {
+private:
+    rust_kernel *kernel;
+public:
+    rust_sched_reaper(rust_kernel *kernel);
+    virtual void run();
+};
+
+#endif /* RUST_SCHED_REAPER_H */