Merge pull request #4619 from brson/exchange

Some work on freestanding Rust: foreign calls, exchange allocator

mk/rt.mk

@@ -63,6 +63,7 @@ RUNTIME_CXXS_$(1) := \
               rt/rust_log.cpp \
               rt/rust_gc_metadata.cpp \
               rt/rust_util.cpp \
+              rt/rust_exchange_alloc.cpp \
               rt/isaac/randport.cpp \
               rt/miniz.cpp \
               rt/rust_kernel.cpp \

src/libcore/private.rs

@@ -30,6 +30,8 @@ pub mod global;
 pub mod finally;
 #[path = "private/weak_task.rs"]
 pub mod weak_task;
+#[path = "private/exchange_alloc.rs"]
+pub mod exchange_alloc;
 
 extern mod rustrt {
     pub unsafe fn rust_create_little_lock() -> rust_little_lock;
@@ -86,6 +88,17 @@ fn test_run_in_bare_thread() {
     }
 }
 
+#[test]
+fn test_run_in_bare_thread_exchange() {
+    unsafe {
+        // Does the exchange heap work without the runtime?
+        let i = ~100;
+        do run_in_bare_thread {
+            assert i == ~100;
+        }
+    }
+}
+
 fn compare_and_swap(address: &mut int, oldval: int, newval: int) -> bool {
     unsafe {
         let old = rusti::atomic_cxchg(address, oldval, newval);

src/libcore/private/exchange_alloc.rs

@@ -0,0 +1,79 @@
+// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use sys::{TypeDesc, size_of};
+use libc::{c_void, size_t, uintptr_t};
+use c_malloc = libc::malloc;
+use c_free = libc::free;
+use managed::raw::{BoxHeaderRepr, BoxRepr};
+use cast::transmute;
+use ptr::{set_memory, null};
+use intrinsic::TyDesc;
+
+pub unsafe fn malloc(td: *TypeDesc, size: uint) -> *c_void {
+    unsafe {
+        assert td.is_not_null();
+
+        let total_size = get_box_size(size, (*td).align);
+        let p = c_malloc(total_size as size_t);
+        assert p.is_not_null();
+
+        // FIXME #4761: Would be very nice to not memset all allocations
+        let p: *mut u8 = transmute(p);
+        set_memory(p, 0, total_size);
+
+        // FIXME #3475: Converting between our two different tydesc types
+        let td: *TyDesc = transmute(td);
+
+        let box: &mut BoxRepr = transmute(p);
+        box.header.ref_count = -1; // Exchange values not ref counted
+        box.header.type_desc = td;
+        box.header.prev = null();
+        box.header.next = null();
+
+        let exchange_count = &mut *rust_get_exchange_count_ptr();
+        rusti::atomic_xadd(exchange_count, 1);
+
+        return transmute(box);
+    }
+}
+
+pub unsafe fn free(ptr: *c_void) {
+    let exchange_count = &mut *rust_get_exchange_count_ptr();
+    rusti::atomic_xsub(exchange_count, 1);
+
+    assert ptr.is_not_null();
+    c_free(ptr);
+}
+
+fn get_box_size(body_size: uint, body_align: uint) -> uint {
+    let header_size = size_of::<BoxHeaderRepr>();
+    // FIXME (#2699): This alignment calculation is suspicious. Is it right?
+    let total_size = align_to(header_size, body_align) + body_size;
+    return total_size;
+}
+
+// Rounds |size| to the nearest |alignment|. Invariant: |alignment| is a power
+// of two.
+fn align_to(size: uint, align: uint) -> uint {
+    assert align != 0;
+    (size + align - 1) & !(align - 1)
+}
+
+extern {
+    #[rust_stack]
+    fn rust_get_exchange_count_ptr() -> *mut int;
+}
+
+#[abi = "rust-intrinsic"]
+extern mod rusti {
+    fn atomic_xadd(dst: &mut int, src: int) -> int;
+    fn atomic_xsub(dst: &mut int, src: int) -> int;
+}

src/libcore/rt.rs

@@ -15,6 +15,8 @@ use libc::{c_char, c_uchar, c_void, size_t, uintptr_t};
 use managed::raw::BoxRepr;
 use str;
 use sys;
+use private::exchange_alloc;
+use cast::transmute;
 
 use gc::{cleanup_stack_for_failure, gc, Word};
 
@@ -27,13 +29,6 @@ pub const FROZEN_BIT: uint = 0x80000000;
 pub const FROZEN_BIT: uint = 0x8000000000000000;
 
 pub extern mod rustrt {
-    #[rust_stack]
-    unsafe fn rust_upcall_exchange_malloc(td: *c_char, size: uintptr_t)
-                                       -> *c_char;
-
-    #[rust_stack]
-    unsafe fn rust_upcall_exchange_free(ptr: *c_char);
-
     #[rust_stack]
     unsafe fn rust_upcall_malloc(td: *c_char, size: uintptr_t) -> *c_char;
 
@@ -67,10 +62,11 @@ pub unsafe fn rt_fail_borrowed() {
     }
 }
 
+// XXX: Make these signatures agree with exchange_alloc's signatures
 #[rt(exchange_malloc)]
 #[lang="exchange_malloc"]
 pub unsafe fn rt_exchange_malloc(td: *c_char, size: uintptr_t) -> *c_char {
-    return rustrt::rust_upcall_exchange_malloc(td, size);
+    transmute(exchange_alloc::malloc(transmute(td), transmute(size)))
 }
 
 // NB: Calls to free CANNOT be allowed to fail, as throwing an exception from
@@ -79,7 +75,7 @@ pub unsafe fn rt_exchange_malloc(td: *c_char, size: uintptr_t) -> *c_char {
 #[rt(exchange_free)]
 #[lang="exchange_free"]
 pub unsafe fn rt_exchange_free(ptr: *c_char) {
-    rustrt::rust_upcall_exchange_free(ptr);
+    exchange_alloc::free(transmute(ptr))
 }
 
 #[rt(malloc)]

src/rt/rust_exchange_alloc.cpp

@@ -0,0 +1,63 @@
+// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#include "rust_exchange_alloc.h"
+#include "sync/sync.h"
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+uintptr_t exchange_count = 0;
+
+void *
+rust_exchange_alloc::malloc(size_t size, bool zero) {
+  void *value = ::malloc(size);
+  assert(value);
+  if (zero) {
+    memset(value, 0, size);
+  }
+
+  sync::increment(exchange_count);
+
+  return value;
+}
+
+void *
+rust_exchange_alloc::calloc(size_t size) {
+  return this->malloc(size);
+}
+
+void *
+rust_exchange_alloc::realloc(void *ptr, size_t size) {
+  void *new_ptr = ::realloc(ptr, size);
+  assert(new_ptr);
+  return new_ptr;
+}
+
+void
+rust_exchange_alloc::free(void *ptr) {
+  sync::decrement(exchange_count);
+  ::free(ptr);
+}
+
+extern "C" uintptr_t *
+rust_get_exchange_count_ptr() {
+  return &exchange_count;
+}
+
+void
+rust_check_exchange_count_on_exit() {
+  if (exchange_count != 0) {
+    printf("exchange heap not empty on on exit");
+    printf("%d dangling allocations", (int)exchange_count);
+    abort();
+  }
+}

src/rt/rust_exchange_alloc.h

@@ -0,0 +1,31 @@
+// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#ifndef RUST_EXCHANGE_ALLOC_H
+#define RUST_EXCHANGE_ALLOC_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+class rust_exchange_alloc {
+ public:
+    void *malloc(size_t size, bool zero = true);
+    void *calloc(size_t size);
+    void *realloc(void *mem, size_t size);
+    void free(void *mem);
+};
+
+extern "C" uintptr_t *
+rust_get_exchange_count_ptr();
+
+void
+rust_check_exchange_count_on_exit();
+
+#endif

src/rt/rust_kernel.cpp

@@ -22,7 +22,6 @@
     KLOG_LVL(this, field, log_err, __VA_ARGS__)
 
 rust_kernel::rust_kernel(rust_env *env) :
-    _region(env, true),
     _log(NULL),
     max_task_id(INIT_TASK_ID-1), // sync_add_and_fetch increments first
     rval(0),
@@ -77,21 +76,21 @@ rust_kernel::fatal(char const *fmt, ...) {
 
 void *
 rust_kernel::malloc(size_t size, const char *tag) {
-    return _region.malloc(size, tag);
+    return exchange_alloc.malloc(size);
 }
 
 void *
 rust_kernel::calloc(size_t size, const char *tag) {
-    return _region.calloc(size, tag);
+    return exchange_alloc.calloc(size);
 }
 
 void *
 rust_kernel::realloc(void *mem, size_t size) {
-    return _region.realloc(mem, size);
+    return exchange_alloc.realloc(mem, size);
 }
 
 void rust_kernel::free(void *mem) {
-    _region.free(mem);
+    exchange_alloc.free(mem);
 }
 
 rust_sched_id
@@ -217,6 +216,7 @@ rust_kernel::run() {
     assert(osmain_driver != NULL);
     osmain_driver->start_main_loop();
     sched_reaper.join();
+    rust_check_exchange_count_on_exit();
     return rval;
 }
 

src/rt/rust_kernel.h

@@ -45,11 +45,12 @@
 #include <map>
 #include <vector>
 
-#include "memory_region.h"
+#include "rust_exchange_alloc.h"
 #include "rust_log.h"
 #include "rust_sched_reaper.h"
 #include "rust_type.h"
 #include "util/hash_map.h"
+#include "sync/lock_and_signal.h"
 
 class rust_scheduler;
 class rust_sched_driver;
@@ -71,7 +72,7 @@ struct exit_functions {
 };
 
 class rust_kernel {
-    memory_region _region;
+    rust_exchange_alloc exchange_alloc;
     rust_log _log;
 
     // The next task id
@@ -135,7 +136,7 @@ public:
     void *calloc(size_t size, const char *tag);
     void *realloc(void *mem, size_t size);
     void free(void *mem);
-    memory_region *region() { return &_region; }
+    rust_exchange_alloc *region() { return &exchange_alloc; }
 
     void fail();
 

src/rt/rust_sched_loop.cpp

@@ -260,7 +260,7 @@ rust_sched_loop::run_single_turn() {
 
         assert(!extra_c_stack);
         if (cached_c_stack) {
-            destroy_stack(kernel->region(), cached_c_stack);
+            destroy_exchange_stack(kernel->region(), cached_c_stack);
             cached_c_stack = NULL;
         }
 
@@ -389,14 +389,15 @@ void
 rust_sched_loop::prepare_c_stack(rust_task *task) {
     assert(!extra_c_stack);
     if (!cached_c_stack && !task->have_c_stack()) {
-        cached_c_stack = create_stack(kernel->region(), C_STACK_SIZE);
+        cached_c_stack = create_exchange_stack(kernel->region(),
+                                               C_STACK_SIZE);
     }
 }
 
 void
 rust_sched_loop::unprepare_c_stack() {
     if (extra_c_stack) {
-        destroy_stack(kernel->region(), extra_c_stack);
+        destroy_exchange_stack(kernel->region(), extra_c_stack);
         extra_c_stack = NULL;
     }
 }

src/rt/rust_sched_loop.h

@@ -135,6 +135,7 @@ public:
     void place_task_in_tls(rust_task *task);
 
     static rust_task *get_task_tls();
+    static rust_task *try_get_task_tls();
 
     // Called by each task when they are ready to be destroyed
     void release_task(rust_task *task);
@@ -154,7 +155,7 @@ rust_sched_loop::get_log() {
     return _log;
 }
 
-inline rust_task* rust_sched_loop::get_task_tls()
+inline rust_task* rust_sched_loop::try_get_task_tls()
 {
     if (!tls_initialized)
         return NULL;
@@ -165,6 +166,12 @@ inline rust_task* rust_sched_loop::get_task_tls()
     rust_task *task = reinterpret_cast<rust_task *>
         (pthread_getspecific(task_key));
 #endif
+    return task;
+}
+
+inline rust_task* rust_sched_loop::get_task_tls()
+{
+    rust_task *task = try_get_task_tls();
     assert(task && "Couldn't get the task from TLS!");
     return task;
 }

src/rt/rust_stack.cpp

@@ -53,6 +53,8 @@ check_stack_canary(stk_seg *stk) {
     assert(stk->canary == canary_value && "Somebody killed the canary");
 }
 
+// XXX: Duplication here between the local and exchange heap constructors
+
 stk_seg *
 create_stack(memory_region *region, size_t sz) {
     size_t total_sz = sizeof(stk_seg) + sz;
@@ -69,3 +71,20 @@ destroy_stack(memory_region *region, stk_seg *stk) {
     deregister_valgrind_stack(stk);
     region->free(stk);
 }
+
+stk_seg *
+create_exchange_stack(rust_exchange_alloc *exchange, size_t sz) {
+    size_t total_sz = sizeof(stk_seg) + sz;
+    stk_seg *stk = (stk_seg *)exchange->malloc(total_sz, false);
+    memset(stk, 0, sizeof(stk_seg));
+    stk->end = (uintptr_t) &stk->data[sz];
+    add_stack_canary(stk);
+    register_valgrind_stack(stk);
+    return stk;
+}
+
+void
+destroy_exchange_stack(rust_exchange_alloc *exchange, stk_seg *stk) {
+    deregister_valgrind_stack(stk);
+    exchange->free(stk);
+}

src/rt/rust_stack.h

@@ -12,6 +12,7 @@
 #define RUST_STACK_H
 
 #include "rust_globals.h"
+#include "rust_exchange_alloc.h"
 #include "memory_region.h"
 
 struct rust_task;
@@ -37,6 +38,12 @@ create_stack(memory_region *region, size_t sz);
 void
 destroy_stack(memory_region *region, stk_seg *stk);
 
+stk_seg *
+create_exchange_stack(rust_exchange_alloc *exchange, size_t sz);
+
+void
+destroy_exchange_stack(rust_exchange_alloc *exchange, stk_seg *stk);
+
 // Must be called before each time a stack is reused to tell valgrind
 // that the stack is accessible.
 void

src/rt/rust_task.cpp

@@ -181,7 +181,7 @@ void task_start_wrapper(spawn_args *a)
         // free the environment (which should be a unique closure).
         const type_desc *td = env->td;
         td->drop_glue(NULL, NULL, NULL, box_body(env));
-        upcall_exchange_free(env);
+        task->kernel->region()->free(env);
     }
 
     // The cleanup work needs lots of stack

src/rt/rust_task.h

@@ -619,14 +619,14 @@ rust_task::record_stack_limit() {
     record_sp_limit(stk->data + LIMIT_OFFSET + RED_ZONE_SIZE);
 }
 
-inline rust_task* rust_get_current_task() {
+inline rust_task* rust_try_get_current_task() {
     uintptr_t sp_limit = get_sp_limit();
 
     // FIXME (#1226) - Because of a hack in upcall_call_shim_on_c_stack this
     // value is sometimes inconveniently set to 0, so we can't use this
     // method of retreiving the task pointer and need to fall back to TLS.
     if (sp_limit == 0)
-        return rust_sched_loop::get_task_tls();
+        return rust_sched_loop::try_get_task_tls();
 
     // The stack pointer boundary is stored in a quickly-accessible location
     // in the TCB. From that we can calculate the address of the stack segment
@@ -642,6 +642,12 @@ inline rust_task* rust_get_current_task() {
     return stk->task;
 }
 
+inline rust_task* rust_get_current_task() {
+    rust_task* task = rust_try_get_current_task();
+    assert(task != NULL && "no current task");
+    return task;
+}
+
 //
 // Local Variables:
 // mode: C++

src/rt/rust_upcall.cpp

@@ -45,6 +45,8 @@ call_upcall_on_c_stack(rust_task *task, void *args, void *fn_ptr) {
     task->call_on_c_stack(args, fn_ptr);
 }
 
+typedef void (*CDECL stack_switch_shim)(void*);
+
 /**********************************************************************
  * Switches to the C-stack and invokes |fn_ptr|, passing |args| as argument.
  * This is used by the C compiler to call foreign functions and by other
@@ -54,13 +56,20 @@ call_upcall_on_c_stack(rust_task *task, void *args, void *fn_ptr) {
  */
 extern "C" CDECL void
 upcall_call_shim_on_c_stack(void *args, void *fn_ptr) {
-    rust_task *task = rust_get_current_task();
+    rust_task *task = rust_try_get_current_task();
 
-    try {
-        task->call_on_c_stack(args, fn_ptr);
-    } catch (...) {
-        // Logging here is not reliable
-        assert(false && "Foreign code threw an exception");
+    if (task) {
+        // We're running in task context, do a stack switch
+        try {
+            task->call_on_c_stack(args, fn_ptr);
+        } catch (...) {
+            // Logging here is not reliable
+            assert(false && "Foreign code threw an exception");
+        }
+    } else {
+        // There's no task. Call the function and hope for the best
+        stack_switch_shim f = (stack_switch_shim)fn_ptr;
+        f(args);
     }
 }
 
@@ -70,15 +79,22 @@ upcall_call_shim_on_c_stack(void *args, void *fn_ptr) {
  */
 extern "C" CDECL void
 upcall_call_shim_on_rust_stack(void *args, void *fn_ptr) {
-    rust_task *task = rust_get_current_task();
+    rust_task *task = rust_try_get_current_task();
 
-    try {
-        task->call_on_rust_stack(args, fn_ptr);
-    } catch (...) {
-        // We can't count on being able to unwind through arbitrary
-        // code. Our best option is to just fail hard.
-        // Logging here is not reliable
-        assert(false && "Rust task failed after reentering the Rust stack");
+    if (task) {
+        try {
+            task->call_on_rust_stack(args, fn_ptr);
+        } catch (...) {
+            // We can't count on being able to unwind through arbitrary
+            // code. Our best option is to just fail hard.
+            // Logging here is not reliable
+            assert(false
+                   && "Rust task failed after reentering the Rust stack");
+        }
+    } else {
+        // There's no task. Call the function and hope for the best
+        stack_switch_shim f = (stack_switch_shim)fn_ptr;
+        f(args);
     }
 }
 
@@ -124,81 +140,6 @@ struct s_trace_args {
     size_t line;
 };
 
-/**********************************************************************
- * Allocate an object in the exchange heap
- */
-
-struct s_exchange_malloc_args {
-    rust_task *task;
-    uintptr_t retval;
-    type_desc *td;
-    uintptr_t size;
-};
-
-extern "C" CDECL void
-upcall_s_exchange_malloc(s_exchange_malloc_args *args) {
-    rust_task *task = args->task;
-    LOG_UPCALL_ENTRY(task);
-
-    size_t total_size = get_box_size(args->size, args->td->align);
-    void *p = task->kernel->malloc(total_size, "exchange malloc");
-
-    rust_opaque_box *header = static_cast<rust_opaque_box*>(p);
-    header->ref_count = -1; // This is not ref counted
-    header->td = args->td;
-    header->prev = 0;
-    header->next = 0;
-
-    LOG(task, mem, "exchange malloced %p of size %" PRIuPTR,
-        header, args->size);
-
-    args->retval = (uintptr_t)header;
-}
-
-extern "C" CDECL uintptr_t
-upcall_exchange_malloc(type_desc *td, uintptr_t size) {
-    rust_task *task = rust_get_current_task();
-    s_exchange_malloc_args args = {task, 0, td, size};
-    UPCALL_SWITCH_STACK(task, &args, upcall_s_exchange_malloc);
-    return args.retval;
-}
-
-// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
-// autogenerated wrappers for upcall_exchange_malloc. Remove this when we
-// fully move away away from the C upcall path.
-extern "C" CDECL uintptr_t
-rust_upcall_exchange_malloc(type_desc *td, uintptr_t size) {
-    return upcall_exchange_malloc(td, size);
-}
-
-struct s_exchange_free_args {
-    rust_task *task;
-    void *ptr;
-};
-
-extern "C" CDECL void
-upcall_s_exchange_free(s_exchange_free_args *args) {
-    rust_task *task = args->task;
-    LOG_UPCALL_ENTRY(task);
-    LOG(task, mem, "exchange freed %p", args->ptr);
-    task->kernel->free(args->ptr);
-}
-
-extern "C" CDECL void
-upcall_exchange_free(void *ptr) {
-    rust_task *task = rust_get_current_task();
-    s_exchange_free_args args = {task,ptr};
-    UPCALL_SWITCH_STACK(task, &args, upcall_s_exchange_free);
-}
-
-// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
-// autogenerated wrappers for upcall_exchange_free. Remove this when we fully
-// move away away from the C upcall path.
-extern "C" CDECL void
-rust_upcall_exchange_free(void *ptr) {
-    return upcall_exchange_free(ptr);
-}
-
 /**********************************************************************
  * Allocate an object in the task-local heap.
  */

src/rt/rustrt.def.in

@@ -68,10 +68,6 @@ upcall_call_shim_on_rust_stack
 upcall_new_stack
 upcall_del_stack
 upcall_reset_stack_limit
-upcall_exchange_malloc
-upcall_exchange_free
-rust_upcall_exchange_free
-rust_upcall_exchange_malloc
 rust_upcall_fail
 rust_upcall_free
 rust_upcall_malloc
@@ -194,4 +190,5 @@ rust_raw_thread_join_delete
 rust_register_exit_function
 rust_get_global_data_ptr
 rust_inc_weak_task_count
-rust_dec_weak_task_count
+rust_dec_weak_task_count
+rust_get_exchange_count_ptr

src/test/run-pass/foreign-call-no-runtime.rs

@@ -0,0 +1,24 @@
+use core::private::run_in_bare_thread;
+
+extern {
+    pub fn rust_dbg_call(cb: *u8,
+                         data: libc::uintptr_t) -> libc::uintptr_t;
+}
+
+pub fn main() {
+    unsafe {
+        do run_in_bare_thread() {
+            unsafe {
+                let i = &100;
+                rust_dbg_call(callback, cast::transmute(i));
+            }
+        }
+    }
+}
+
+extern fn callback(data: libc::uintptr_t) {
+    unsafe {
+        let data: *int = cast::transmute(data);
+        assert *data == 100;
+    }
+}