diff --git a/compiler/rustc_target/src/spec/x86_64_fortanix_unknown_sgx.rs b/compiler/rustc_target/src/spec/x86_64_fortanix_unknown_sgx.rs index cba6fda19dc3e..b4b1296b03569 100644 --- a/compiler/rustc_target/src/spec/x86_64_fortanix_unknown_sgx.rs +++ b/compiler/rustc_target/src/spec/x86_64_fortanix_unknown_sgx.rs @@ -1,6 +1,6 @@ use std::borrow::Cow; -use super::{cvs, Cc, LinkerFlavor, Lld, Target, TargetOptions}; +use super::{cvs, Cc, LinkerFlavor, Lld, Target, TargetOptions, TlsModel}; pub fn target() -> Target { let pre_link_args = TargetOptions::link_args( @@ -53,6 +53,9 @@ pub fn target() -> Target { "EH_FRM_LEN", "TEXT_BASE", "TEXT_SIZE", + "TLS_INIT_BASE", + "TLS_INIT_SIZE", + "TLS_OFFSET", ]; let opts = TargetOptions { os: "unknown".into(), @@ -69,6 +72,8 @@ pub fn target() -> Target { pre_link_args, override_export_symbols: Some(EXPORT_SYMBOLS.iter().cloned().map(Cow::from).collect()), relax_elf_relocations: true, + has_thread_local: true, + tls_model: TlsModel::LocalExec, ..Default::default() }; Target { diff --git a/library/std/src/sys/sgx/abi/entry.S b/library/std/src/sys/sgx/abi/entry.S index f61bcf06f0815..cbbad7c9c5e2f 100644 --- a/library/std/src/sys/sgx/abi/entry.S +++ b/library/std/src/sys/sgx/abi/entry.S @@ -68,6 +68,12 @@ IMAGE_BASE: globvar EH_FRM_OFFSET 8 /* The size in bytes of enclacve .eh_frame section */ globvar EH_FRM_LEN 8 + /* The base address (relative to enclave start) of the TLS initialization image */ + globvar TLS_INIT_BASE 8 + /* The size in bytes of the TLS initialization image */ + globvar TLS_INIT_SIZE 8 + /* The TLS module offset of this enclave (see the ELF-TLS specification for more details) */ + globvar TLS_OFFSET 8 .org .Lxsave_clear+512 .Lxsave_header: @@ -80,24 +86,24 @@ IMAGE_BASE: .byte 0 /* TCS local storage section */ -.equ tcsls_tos, 0x00 /* initialized by loader to *offset* from image base to TOS */ -.equ tcsls_flags, 0x08 /* initialized by loader */ +.equ tcsls_tp, 0x00 /* initialized by loader to *offset* from image base to this field */ +.equ tcsls_tos, 0x08 /* initialized by loader to *offset* from image base to TOS */ +.equ tcsls_flags, 0x10 /* initialized by loader */ .equ tcsls_flag_secondary, 0 /* initialized by loader; 0 = standard TCS, 1 = secondary TCS */ .equ tcsls_flag_init_once, 1 /* initialized by loader to 0 */ /* 14 unused bits */ -.equ tcsls_user_fcw, 0x0a -.equ tcsls_user_mxcsr, 0x0c -.equ tcsls_last_rsp, 0x10 /* initialized by loader to 0 */ -.equ tcsls_panic_last_rsp, 0x18 /* initialized by loader to 0 */ -.equ tcsls_debug_panic_buf_ptr, 0x20 /* initialized by loader to 0 */ -.equ tcsls_user_rsp, 0x28 -.equ tcsls_user_retip, 0x30 -.equ tcsls_user_rbp, 0x38 -.equ tcsls_user_r12, 0x40 -.equ tcsls_user_r13, 0x48 -.equ tcsls_user_r14, 0x50 -.equ tcsls_user_r15, 0x58 -.equ tcsls_tls_ptr, 0x60 +.equ tcsls_user_fcw, 0x12 +.equ tcsls_user_mxcsr, 0x14 +.equ tcsls_last_rsp, 0x18 /* initialized by loader to 0 */ +.equ tcsls_panic_last_rsp, 0x20 /* initialized by loader to 0 */ +.equ tcsls_debug_panic_buf_ptr, 0x28 /* initialized by loader to 0 */ +.equ tcsls_user_rsp, 0x30 +.equ tcsls_user_retip, 0x38 +.equ tcsls_user_rbp, 0x40 +.equ tcsls_user_r12, 0x48 +.equ tcsls_user_r13, 0x50 +.equ tcsls_user_r14, 0x58 +.equ tcsls_user_r15, 0x60 .equ tcsls_tcs_addr, 0x68 .macro load_tcsls_flag_secondary_bool reg:req comments:vararg @@ -105,7 +111,7 @@ IMAGE_BASE: .abort .endif mov $(1< EntryReturn { // FIXME: how to support TLS in library mode? - let tls = Box::new(tls::Tls::new()); - let tls_guard = unsafe { tls.activate() }; + let tls = unsafe { tls::Tls::init() }; if secondary { let join_notifier = super::thread::Thread::entry(); - drop(tls_guard); + drop(tls); drop(join_notifier); EntryReturn(0, 0) diff --git a/library/std/src/sys/sgx/abi/tls.rs b/library/std/src/sys/sgx/abi/tls.rs new file mode 100644 index 0000000000000..1c08b950f0987 --- /dev/null +++ b/library/std/src/sys/sgx/abi/tls.rs @@ -0,0 +1,49 @@ +use super::mem; +use crate::arch::asm; + +extern "C" { + static TLS_INIT_BASE: u64; + static TLS_INIT_SIZE: usize; + static TLS_OFFSET: usize; +} + +pub struct Tls {} + +impl Tls { + /// Initialize the thread local storage to a fresh state. + /// + /// # Safety + /// * may only be called once per thread + /// * must be dropped before thread exit + /// * must be called before any `#[thread_local]` variable is used + pub unsafe fn init() -> Tls { + // The thread pointer points to the end of the TLS section. It is stored + // both in the `fs` segment base address (initialized by the loader) and + // at the address it itself points to (initialized during TCS + // initialization). + let tp: *mut u8; + unsafe { + asm!("mov fs:0, {}", out(reg) tp, options(preserves_flags, readonly)); + } + + // Initialize the TLS data. + unsafe { + let init_base = mem::rel_ptr_mut(TLS_INIT_BASE); + // The first `TLS_INIT_SIZE` bytes of the TLS section hold non-trivial + // data that needs to be copied from the initialization image. + tp.sub(TLS_OFFSET).copy_from_nonoverlapping(init_base, TLS_INIT_SIZE); + // All remaining bytes are initialized to zero. + tp.sub(TLS_OFFSET).add(TLS_INIT_SIZE).write_bytes(0, TLS_OFFSET - TLS_INIT_SIZE); + } + + Tls {} + } +} + +impl Drop for Tls { + fn drop(&mut self) { + unsafe { + crate::sys::thread_local_dtor::run_dtors(); + } + } +} diff --git a/library/std/src/sys/sgx/abi/tls/mod.rs b/library/std/src/sys/sgx/abi/tls/mod.rs deleted file mode 100644 index 09c4ab3d3e901..0000000000000 --- a/library/std/src/sys/sgx/abi/tls/mod.rs +++ /dev/null @@ -1,133 +0,0 @@ -mod sync_bitset; - -use self::sync_bitset::*; -use crate::cell::Cell; -use crate::mem; -use crate::num::NonZeroUsize; -use crate::ptr; -use crate::sync::atomic::{AtomicUsize, Ordering}; - -#[cfg(target_pointer_width = "64")] -const USIZE_BITS: usize = 64; -const TLS_KEYS: usize = 128; // Same as POSIX minimum -const TLS_KEYS_BITSET_SIZE: usize = (TLS_KEYS + (USIZE_BITS - 1)) / USIZE_BITS; - -#[cfg_attr(test, linkage = "available_externally")] -#[export_name = "_ZN16__rust_internals3std3sys3sgx3abi3tls14TLS_KEY_IN_USEE"] -static TLS_KEY_IN_USE: SyncBitset = SYNC_BITSET_INIT; -macro_rules! dup { - ((* $($exp:tt)*) $($val:tt)*) => (dup!( ($($exp)*) $($val)* $($val)* )); - (() $($val:tt)*) => ([$($val),*]) -} -#[cfg_attr(test, linkage = "available_externally")] -#[export_name = "_ZN16__rust_internals3std3sys3sgx3abi3tls14TLS_DESTRUCTORE"] -static TLS_DESTRUCTOR: [AtomicUsize; TLS_KEYS] = dup!((* * * * * * *) (AtomicUsize::new(0))); - -extern "C" { - fn get_tls_ptr() -> *const u8; - fn set_tls_ptr(tls: *const u8); -} - -#[derive(Copy, Clone)] -#[repr(C)] -pub struct Key(NonZeroUsize); - -impl Key { - fn to_index(self) -> usize { - self.0.get() - 1 - } - - fn from_index(index: usize) -> Self { - Key(NonZeroUsize::new(index + 1).unwrap()) - } - - pub fn as_usize(self) -> usize { - self.0.get() - } - - pub fn from_usize(index: usize) -> Self { - Key(NonZeroUsize::new(index).unwrap()) - } -} - -#[repr(C)] -pub struct Tls { - data: [Cell<*mut u8>; TLS_KEYS], -} - -pub struct ActiveTls<'a> { - tls: &'a Tls, -} - -impl<'a> Drop for ActiveTls<'a> { - fn drop(&mut self) { - let value_with_destructor = |key: usize| { - let ptr = TLS_DESTRUCTOR[key].load(Ordering::Relaxed); - unsafe { mem::transmute::<_, Option>(ptr) } - .map(|dtor| (&self.tls.data[key], dtor)) - }; - - let mut any_non_null_dtor = true; - while any_non_null_dtor { - any_non_null_dtor = false; - for (value, dtor) in TLS_KEY_IN_USE.iter().filter_map(&value_with_destructor) { - let value = value.replace(ptr::null_mut()); - if !value.is_null() { - any_non_null_dtor = true; - unsafe { dtor(value) } - } - } - } - } -} - -impl Tls { - pub fn new() -> Tls { - Tls { data: dup!((* * * * * * *) (Cell::new(ptr::null_mut()))) } - } - - pub unsafe fn activate(&self) -> ActiveTls<'_> { - // FIXME: Needs safety information. See entry.S for `set_tls_ptr` definition. - unsafe { set_tls_ptr(self as *const Tls as _) }; - ActiveTls { tls: self } - } - - #[allow(unused)] - pub unsafe fn activate_persistent(self: Box) { - // FIXME: Needs safety information. See entry.S for `set_tls_ptr` definition. - unsafe { set_tls_ptr((&*self) as *const Tls as _) }; - mem::forget(self); - } - - unsafe fn current<'a>() -> &'a Tls { - // FIXME: Needs safety information. See entry.S for `set_tls_ptr` definition. - unsafe { &*(get_tls_ptr() as *const Tls) } - } - - pub fn create(dtor: Option) -> Key { - let index = if let Some(index) = TLS_KEY_IN_USE.set() { - index - } else { - rtabort!("TLS limit exceeded") - }; - TLS_DESTRUCTOR[index].store(dtor.map_or(0, |f| f as usize), Ordering::Relaxed); - unsafe { Self::current() }.data[index].set(ptr::null_mut()); - Key::from_index(index) - } - - pub fn set(key: Key, value: *mut u8) { - let index = key.to_index(); - rtassert!(TLS_KEY_IN_USE.get(index)); - unsafe { Self::current() }.data[index].set(value); - } - - pub fn get(key: Key) -> *mut u8 { - let index = key.to_index(); - rtassert!(TLS_KEY_IN_USE.get(index)); - unsafe { Self::current() }.data[index].get() - } - - pub fn destroy(key: Key) { - TLS_KEY_IN_USE.clear(key.to_index()); - } -} diff --git a/library/std/src/sys/sgx/abi/tls/sync_bitset.rs b/library/std/src/sys/sgx/abi/tls/sync_bitset.rs deleted file mode 100644 index 4eeff8f6ef773..0000000000000 --- a/library/std/src/sys/sgx/abi/tls/sync_bitset.rs +++ /dev/null @@ -1,85 +0,0 @@ -#[cfg(test)] -mod tests; - -use super::{TLS_KEYS_BITSET_SIZE, USIZE_BITS}; -use crate::iter::{Enumerate, Peekable}; -use crate::slice::Iter; -use crate::sync::atomic::{AtomicUsize, Ordering}; - -/// A bitset that can be used synchronously. -pub(super) struct SyncBitset([AtomicUsize; TLS_KEYS_BITSET_SIZE]); - -pub(super) const SYNC_BITSET_INIT: SyncBitset = - SyncBitset([AtomicUsize::new(0), AtomicUsize::new(0)]); - -impl SyncBitset { - pub fn get(&self, index: usize) -> bool { - let (hi, lo) = Self::split(index); - (self.0[hi].load(Ordering::Relaxed) & lo) != 0 - } - - /// Not atomic. - pub fn iter(&self) -> SyncBitsetIter<'_> { - SyncBitsetIter { iter: self.0.iter().enumerate().peekable(), elem_idx: 0 } - } - - pub fn clear(&self, index: usize) { - let (hi, lo) = Self::split(index); - self.0[hi].fetch_and(!lo, Ordering::Relaxed); - } - - /// Sets any unset bit. Not atomic. Returns `None` if all bits were - /// observed to be set. - pub fn set(&self) -> Option { - 'elems: for (idx, elem) in self.0.iter().enumerate() { - let mut current = elem.load(Ordering::Relaxed); - loop { - if 0 == !current { - continue 'elems; - } - let trailing_ones = (!current).trailing_zeros() as usize; - match elem.compare_exchange( - current, - current | (1 << trailing_ones), - Ordering::AcqRel, - Ordering::Relaxed, - ) { - Ok(_) => return Some(idx * USIZE_BITS + trailing_ones), - Err(previous) => current = previous, - } - } - } - None - } - - fn split(index: usize) -> (usize, usize) { - (index / USIZE_BITS, 1 << (index % USIZE_BITS)) - } -} - -pub(super) struct SyncBitsetIter<'a> { - iter: Peekable>>, - elem_idx: usize, -} - -impl<'a> Iterator for SyncBitsetIter<'a> { - type Item = usize; - - fn next(&mut self) -> Option { - self.iter.peek().cloned().and_then(|(idx, elem)| { - let elem = elem.load(Ordering::Relaxed); - let low_mask = (1 << self.elem_idx) - 1; - let next = elem & !low_mask; - let next_idx = next.trailing_zeros() as usize; - self.elem_idx = next_idx + 1; - if self.elem_idx >= 64 { - self.elem_idx = 0; - self.iter.next(); - } - match next_idx { - 64 => self.next(), - _ => Some(idx * USIZE_BITS + next_idx), - } - }) - } -} diff --git a/library/std/src/sys/sgx/abi/tls/sync_bitset/tests.rs b/library/std/src/sys/sgx/abi/tls/sync_bitset/tests.rs deleted file mode 100644 index d7eb2e139d011..0000000000000 --- a/library/std/src/sys/sgx/abi/tls/sync_bitset/tests.rs +++ /dev/null @@ -1,25 +0,0 @@ -use super::*; - -fn test_data(bitset: [usize; 2], bit_indices: &[usize]) { - let set = SyncBitset([AtomicUsize::new(bitset[0]), AtomicUsize::new(bitset[1])]); - assert_eq!(set.iter().collect::>(), bit_indices); - for &i in bit_indices { - assert!(set.get(i)); - } -} - -#[test] -fn iter() { - test_data([0b0110_1001, 0], &[0, 3, 5, 6]); - test_data([0x8000_0000_0000_0000, 0x8000_0000_0000_0001], &[63, 64, 127]); - test_data([0, 0], &[]); -} - -#[test] -fn set_get_clear() { - let set = SYNC_BITSET_INIT; - let key = set.set().unwrap(); - assert!(set.get(key)); - set.clear(key); - assert!(!set.get(key)); -} diff --git a/library/std/src/sys/sgx/mod.rs b/library/std/src/sys/sgx/mod.rs index 63e070207cd70..afe838a57462a 100644 --- a/library/std/src/sys/sgx/mod.rs +++ b/library/std/src/sys/sgx/mod.rs @@ -33,6 +33,8 @@ pub mod pipe; pub mod process; pub mod stdio; pub mod thread; +pub mod thread_local_dtor; +#[path = "../unsupported/thread_local_key.rs"] pub mod thread_local_key; pub mod thread_parker; pub mod time; diff --git a/library/std/src/sys/sgx/thread_local_dtor.rs b/library/std/src/sys/sgx/thread_local_dtor.rs new file mode 100644 index 0000000000000..27e9c1fe2eb19 --- /dev/null +++ b/library/std/src/sys/sgx/thread_local_dtor.rs @@ -0,0 +1,22 @@ +use crate::mem; + +#[thread_local] +static mut DTORS: Vec<(*mut u8, unsafe extern "C" fn(*mut u8))> = Vec::new(); + +pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { + let dtors = unsafe { &mut DTORS }; + dtors.push((t, dtor)); +} + +pub(super) unsafe fn run_dtors() { + let mut dtors = mem::take(unsafe { &mut DTORS }); + while !dtors.is_empty() { + for (t, dtor) in dtors { + unsafe { + dtor(t); + } + } + + dtors = mem::take(unsafe { &mut DTORS }); + } +} diff --git a/library/std/src/sys/sgx/thread_local_key.rs b/library/std/src/sys/sgx/thread_local_key.rs deleted file mode 100644 index c7a57d3a3d47e..0000000000000 --- a/library/std/src/sys/sgx/thread_local_key.rs +++ /dev/null @@ -1,23 +0,0 @@ -use super::abi::tls::{Key as AbiKey, Tls}; - -pub type Key = usize; - -#[inline] -pub unsafe fn create(dtor: Option) -> Key { - Tls::create(dtor).as_usize() -} - -#[inline] -pub unsafe fn set(key: Key, value: *mut u8) { - Tls::set(AbiKey::from_usize(key), value) -} - -#[inline] -pub unsafe fn get(key: Key) -> *mut u8 { - Tls::get(AbiKey::from_usize(key)) -} - -#[inline] -pub unsafe fn destroy(key: Key) { - Tls::destroy(AbiKey::from_usize(key)) -}