[profile.release]
lto = true
opt-level = 3
+debug = 2
[profile.wasm-dev]
inherits = "dev"
#[cfg(feature = "http")]
use crate::http::{HttpListener, HttpResponse};
use crate::machine::loader::LiveLoadState;
-use crate::machine::machine_indices::*;
use crate::machine::streams::*;
-use crate::raw_block::*;
+use crate::offset_table::*;
use crate::read::*;
use crate::types::UntypedArenaPtr;
use crate::parser::dashu::{Integer, Rational};
-use arcu::atomic::Arcu;
-use arcu::epoch_counters::GlobalEpochCounterPool;
-use arcu::rcu_ref::RcuRef;
-use arcu::Rcu;
use ordered_float::OrderedFloat;
-use std::cell::UnsafeCell;
use std::fmt;
use std::fmt::Debug;
use std::hash::{Hash, Hasher};
use std::ptr;
use std::ptr::addr_of_mut;
use std::ptr::NonNull;
-use std::sync::RwLock;
+use std::sync::Arc;
macro_rules! arena_alloc {
($e:expr, $arena:expr) => {{
macro_rules! float_alloc {
($e:expr, $arena:expr) => {{
let result = $e;
- unsafe { $arena.f64_tbl.build_with(result).as_ptr() }
+ unsafe { $arena.f64_tbl.build_with(OrderedFloat(result)).as_ptr() }
}};
}
payload_offset - header_offset
}
-use std::sync::Arc;
-use std::sync::Mutex;
-use std::sync::Weak;
-
-const F64_TABLE_INIT_SIZE: usize = 1 << 16;
-const F64_TABLE_ALIGN: usize = 8;
-
-#[inline(always)]
-fn global_f64table() -> &'static RwLock<Weak<F64Table>> {
- static GLOBAL_ATOM_TABLE: RwLock<Weak<F64Table>> = RwLock::new(Weak::new());
- &GLOBAL_ATOM_TABLE
-}
-
-impl RawBlockTraits for F64Table {
- #[inline]
- fn init_size() -> usize {
- F64_TABLE_INIT_SIZE
- }
-
- #[inline]
- fn align() -> usize {
- F64_TABLE_ALIGN
- }
-}
-
-#[derive(Debug)]
-pub struct F64Table {
- block: Arcu<RawBlock<F64Table>, GlobalEpochCounterPool>,
- update: Mutex<()>,
-}
-
-#[inline(always)]
-pub fn lookup_float(
- offset: F64Offset,
-) -> RcuRef<RawBlock<F64Table>, UnsafeCell<OrderedFloat<f64>>> {
- let f64table = global_f64table()
- .read()
- .unwrap()
- .upgrade()
- .expect("We should only be looking up floats while there is a float table");
-
- RcuRef::try_map(f64table.block.read(), |raw_block| unsafe {
- raw_block
- .base
- .add(offset.0)
- .cast_mut()
- .cast::<UnsafeCell<OrderedFloat<f64>>>()
- .as_ref()
- })
- .expect("The offset should result in a non-null pointer")
-}
-
-impl F64Table {
- #[inline]
- pub fn new() -> Arc<Self> {
- let upgraded = global_f64table().read().unwrap().upgrade();
- // don't inline upgraded, otherwise temporary will be dropped too late in case of None
- if let Some(atom_table) = upgraded {
- atom_table
- } else {
- let mut guard = global_f64table().write().unwrap();
- // try to upgrade again in case we lost the race on the write lock
- if let Some(atom_table) = guard.upgrade() {
- atom_table
- } else {
- let atom_table = Arc::new(Self {
- block: Arcu::new(RawBlock::new(), GlobalEpochCounterPool),
- update: Mutex::new(()),
- });
- *guard = Arc::downgrade(&atom_table);
- atom_table
- }
- }
- }
-
- #[allow(clippy::missing_safety_doc)]
- pub unsafe fn build_with(&self, value: f64) -> F64Offset {
- let update_guard = self.update.lock();
-
- // we don't have an index table for lookups as AtomTable does so
- // just get the epoch after we take the upgrade lock
- let mut block_epoch = self.block.read();
-
- let mut ptr;
-
- loop {
- ptr = block_epoch.alloc(mem::size_of::<f64>());
-
- if ptr.is_null() {
- let new_block = block_epoch.grow_new().unwrap();
- self.block.replace(new_block);
- block_epoch = self.block.read();
- } else {
- break;
- }
- }
-
- ptr::write(ptr as *mut OrderedFloat<f64>, OrderedFloat(value));
-
- let float = F64Offset(ptr as usize - block_epoch.base as usize);
-
- // atometable would have to update the index table at this point
-
- // expicit drop to ensure we don't accidentally drop it early
- drop(update_guard);
-
- float
- }
-}
-
#[derive(BitfieldSpecifier, Copy, Clone, Debug, PartialEq)]
#[bits = 7]
pub enum ArenaHeaderTag {
HttpListener = 0b1000001,
HttpResponse = 0b1000010,
Dropped = 0b1000100,
- IndexPtrDynamicUndefined = 0b1000101,
- IndexPtrDynamicIndex = 0b1000110,
- IndexPtrIndex = 0b1000111,
- IndexPtrUndefined = 0b1001000,
}
#[bitfield]
}
}
-#[derive(Debug)]
-pub struct F64Ptr(RcuRef<RawBlock<F64Table>, UnsafeCell<OrderedFloat<f64>>>);
-
-impl Clone for F64Ptr {
- fn clone(&self) -> Self {
- Self(RcuRef::clone(&self.0))
- }
-}
-
-impl PartialEq for F64Ptr {
- fn eq(&self, other: &F64Ptr) -> bool {
- RcuRef::ptr_eq(&self.0, &other.0) || self.deref() == other.deref()
- }
-}
-
-impl Eq for F64Ptr {}
-
-impl PartialOrd for F64Ptr {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- Some(self.cmp(other))
- }
-}
-
-impl Ord for F64Ptr {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- (**self).cmp(&**other)
- }
-}
-
-impl Hash for F64Ptr {
- #[inline(always)]
- fn hash<H: Hasher>(&self, hasher: &mut H) {
- (self as &OrderedFloat<f64>).hash(hasher)
- }
-}
-
-impl fmt::Display for F64Ptr {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "{}", self as &OrderedFloat<f64>)
- }
-}
-
-impl Deref for F64Ptr {
- type Target = OrderedFloat<f64>;
-
- #[inline]
- fn deref(&self) -> &Self::Target {
- unsafe { self.0.get().as_ref().unwrap() }
- }
-}
-
-impl DerefMut for F64Ptr {
- #[inline]
- fn deref_mut(&mut self) -> &mut Self::Target {
- unsafe { &mut *self.0.get().as_mut().unwrap() }
- }
-}
-
-impl F64Ptr {
- #[inline(always)]
- pub fn from_offset(offset: F64Offset) -> Self {
- Self(lookup_float(offset))
- }
-
- #[inline(always)]
- pub fn as_offset(&self) -> F64Offset {
- F64Offset(self.0.get() as usize - RcuRef::get_root(&self.0).base as usize)
- }
-}
-
-#[derive(Clone, Copy, Debug)]
-pub struct F64Offset(usize);
-
-impl F64Offset {
- #[inline(always)]
- pub fn new(offset: usize) -> Self {
- Self(offset)
- }
-
- #[inline(always)]
- pub fn from_ptr(ptr: F64Ptr) -> Self {
- ptr.as_offset()
- }
-
- #[inline(always)]
- pub fn as_ptr(self) -> F64Ptr {
- F64Ptr::from_offset(self)
- }
-
- #[inline(always)]
- pub fn to_u64(self) -> u64 {
- self.0 as u64
- }
-}
-
-impl PartialEq for F64Offset {
- #[inline(always)]
- fn eq(&self, other: &F64Offset) -> bool {
- self.as_ptr() == other.as_ptr()
- }
-}
-
-impl Eq for F64Offset {}
-
-impl PartialOrd for F64Offset {
- #[inline(always)]
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- Some(self.cmp(other))
- }
-}
-
-impl Ord for F64Offset {
- #[inline(always)]
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- self.as_ptr().cmp(&other.as_ptr())
- }
-}
-
-impl Hash for F64Offset {
- #[inline(always)]
- fn hash<H: Hasher>(&self, hasher: &mut H) {
- self.as_ptr().hash(hasher)
- }
-}
-
-impl fmt::Display for F64Offset {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "F64Offset({})", self.0)
- }
-}
-
impl ArenaAllocated for Integer {
type Payload = Self;
#[inline]
}
}
-impl ArenaAllocated for IndexPtr {
- type Payload = Self;
- #[inline]
- fn tag() -> ArenaHeaderTag {
- ArenaHeaderTag::IndexPtrUndefined
- }
-
- #[inline]
- fn header_offset_from_payload() -> usize {
- 0
- }
-
- /// # Safety
- /// - the caller must guarantee that the pointee type of UntypedArenaPtr is T
- /// - the pointer must be non-null
- unsafe fn typed_ptr(ptr: UntypedArenaPtr) -> TypedArenaPtr<Self> {
- TypedArenaPtr(NonNull::new_unchecked(
- ptr.get_ptr().cast_mut().cast::<IndexPtr>(),
- ))
- }
-
- #[inline]
- fn alloc(arena: &mut Arena, value: Self) -> TypedArenaPtr<Self> {
- let slab = Box::new(IndexPtrSlab {
- next: arena.base.take(),
- index_ptr: value,
- });
-
- let (allocated_ptr, untyped_slab) = slab.to_untyped();
- arena.base = Some(untyped_slab);
- allocated_ptr
- }
-
- /// # Safety
- /// - ptr points to an allocated slab of the correct kind
- unsafe fn dealloc(ptr: NonNull<TypedAllocSlab<Self>>) {
- drop(unsafe { Box::from_raw(ptr.as_ptr().cast::<IndexPtrSlab>()) });
- }
-}
-
#[repr(C)]
#[derive(Debug)]
pub struct AllocSlab {
header: ArenaHeader,
}
-#[repr(C)]
-#[derive(Debug)]
-pub struct IndexPtrSlab {
- next: Option<UntypedArenaSlab>,
- index_ptr: IndexPtr,
-}
-
const _: () = {
if std::mem::align_of::<AllocSlab>() < std::mem::align_of::<*const ()>() {
panic!("alignment of AllocSlab is too low");
if std::mem::offset_of!(AllocSlab, header) % std::mem::align_of::<*const ()>() != 0 {
panic!("alignment of header not a multiple of pointers alignment");
}
-
- if std::mem::offset_of!(AllocSlab, header) != std::mem::offset_of!(IndexPtrSlab, index_ptr) {
- panic!("IndexPtrSlab.index_ptr and AllocSlab.header are at different offsets");
- }
};
-impl IndexPtrSlab {
- #[inline]
- pub fn to_untyped(self: Box<Self>) -> (TypedArenaPtr<IndexPtr>, UntypedArenaSlab) {
- let raw_box = Box::into_raw(self);
-
- // safety: the pointer from Box::into_raw fullfills addr_of_mut's saftey requirements
- let index_ptr_ptr = unsafe { ptr::addr_of_mut!((*raw_box).index_ptr) };
- let allocated_ptr = TypedArenaPtr(
- // safety: the pointer points into a valid allocation so it is non null
- unsafe { NonNull::new_unchecked(index_ptr_ptr) },
- );
-
- let untyped_arena = UntypedArenaSlab {
- // safety: pointer from Box::into_raw is never null
- slab: unsafe { NonNull::new_unchecked(raw_box.cast::<AllocSlab>()) },
- tag: <IndexPtr as ArenaAllocated>::tag(),
- };
-
- (allocated_ptr, untyped_arena)
- }
-}
-
#[repr(C)]
#[derive(Debug)]
pub struct TypedAllocSlab<T: ?Sized + ArenaAllocated> {
pub struct Arena {
base: Option<UntypedArenaSlab>,
pub f64_tbl: Arc<F64Table>,
+ pub code_index_tbl: Arc<CodeIndexTable>,
}
unsafe impl Send for Arena {}
Arena {
base: None,
f64_tbl: F64Table::new(),
+ code_index_tbl: CodeIndexTable::new(),
}
}
}
ArenaHeaderTag::StandardErrorStream => {
drop_typed_slab_in_place!(StandardErrorStream, value);
}
- ArenaHeaderTag::IndexPtrUndefined
- | ArenaHeaderTag::IndexPtrDynamicUndefined
- | ArenaHeaderTag::IndexPtrDynamicIndex
- | ArenaHeaderTag::IndexPtrIndex => {
- drop_typed_slab_in_place!(IndexPtr, value);
- }
ArenaHeaderTag::NullStream => {
unreachable!("NullStream is never arena allocated!");
}
use crate::forms::*;
use crate::instructions::*;
use crate::iterators::*;
+use crate::offset_table::*;
use crate::targets::QueryInstruction;
use crate::types::*;
use crate::machine::partial_string::*;
use crate::machine::stack::*;
use crate::machine::streams::*;
+use crate::offset_table::*;
use crate::types::*;
use dashu::base::Signed;
}
}
- fn print_index_ptr(&mut self, index_ptr: IndexPtr, max_depth: usize) {
+ fn print_index_ptr(&mut self, idx: CodeIndex, max_depth: usize) {
if self.format_struct(max_depth, 1, atom!("$index_ptr")) {
let atom = self.state_stack.pop().unwrap();
self.state_stack.pop();
self.state_stack.pop();
- let offset = if index_ptr.is_undefined() || index_ptr.is_dynamic_undefined() {
+ let idx_ptr = idx.as_ptr();
+
+ let offset = if idx_ptr.is_undefined() || idx_ptr.is_dynamic_undefined() {
TokenOrRedirect::Atom(atom!("undefined"))
} else {
- let idx = index_ptr.p() as i64;
+ let idx_ptr_p = idx_ptr.p() as i64;
TokenOrRedirect::NumberFocus(
max_depth,
- NumberFocus::Unfocused(Number::Fixnum(Fixnum::build_with(idx))),
+ NumberFocus::Unfocused(Number::Fixnum(Fixnum::build_with(idx_ptr_p))),
None,
)
};
});
}
}
+ (HeapCellValueTag::CodeIndex, idx) => {
+ self.print_index_ptr(idx, self.max_depth);
+ }
(HeapCellValueTag::Fixnum | HeapCellValueTag::CutPoint, n) => {
self.print_number(max_depth, NumberFocus::Unfocused(Number::Fixnum(n)), &op);
}
(ArenaHeaderTag::Dropped, _value) => {
self.print_impromptu_atom(atom!("$dropped_value"));
}
- (ArenaHeaderTag::IndexPtr, index_ptr) => {
- self.print_index_ptr(*index_ptr, max_depth);
- }
_ => {
}
);
pub(crate) mod atom_table;
#[macro_use]
pub(crate) mod arena;
+pub(crate) mod offset_table;
#[macro_use]
pub(crate) mod parser;
#[macro_use]
use crate::heap_iter::*;
use crate::machine::machine_errors::*;
use crate::machine::machine_state::*;
+use crate::offset_table::*;
use crate::parser::ast::*;
use crate::parser::dashu::{Integer, Rational};
use crate::types::*;
.indices
.code_dir
.get_index(predicate_idx)
- .map(|x| x.1.p() as usize)
+ .map(|x| x.1.as_ptr().p() as usize)
.unwrap();
debug_assert!(current_pred_start <= p);
.indices
.code_dir
.get_index(predicate_idx + 1)
- .map(|x| x.1.p() as usize)
+ .map(|x| x.1.as_ptr().p() as usize)
.unwrap_or(self.code.len());
debug_assert!(current_pred_end >= p);
use crate::machine::machine_indices::VarKey;
use crate::machine::mock_wam::CompositeOpDir;
use crate::machine::{
- ArenaHeaderTag, F64Offset, F64Ptr, Fixnum, Number, BREAK_FROM_DISPATCH_LOOP_LOC,
- LIB_QUERY_SUCCESS,
+ ArenaHeaderTag, Fixnum, Number, BREAK_FROM_DISPATCH_LOOP_LOC, LIB_QUERY_SUCCESS,
};
+use crate::offset_table::*;
use crate::parser::ast::{Var, VarPtr};
use crate::parser::parser::{Parser, Tokens};
use crate::read::{write_term_to_heap, TermWriteResult};
src_code_index.get(),
);
- if src_code_index.is_dynamic_undefined() {
+ if src_code_index.as_ptr().is_dynamic_undefined() {
code_dir.insert(key, src_code_index);
}
} else {
continue;
}
- if !code_index.is_undefined() && !code_index.is_dynamic_undefined() {
+ if !code_index.as_ptr().is_undefined() && !code_index.as_ptr().is_dynamic_undefined() {
let old_index_ptr = code_index.replace(IndexPtr::undefined());
self.payload.retraction_info.push_record(
let code_index = self.get_or_insert_code_index(key, compilation_target);
- if code_index.is_undefined() {
+ if code_index.as_ptr().is_undefined() {
set_code_index(
&mut self.payload.retraction_info,
&compilation_target,
while let Some(addr) = iter.next() {
let addr = unmark_cell_bits!(addr);
- if let Ok(literal) = Literal::try_from(addr) {
- term_stack.push(Term::Literal(Cell::default(), literal));
- } else {
- read_heap_cell!(addr,
- (HeapCellValueTag::Lis) => {
- use crate::parser::parser::as_partial_string;
+ read_heap_cell!(addr,
+ (HeapCellValueTag::Lis) => {
+ use crate::parser::parser::as_partial_string;
- let tail = term_stack.pop().unwrap();
- let head = term_stack.pop().unwrap();
+ let tail = term_stack.pop().unwrap();
+ let head = term_stack.pop().unwrap();
- match as_partial_string(head, tail) {
- Ok((string, Some(tail))) => {
- term_stack.push(Term::PartialString(Cell::default(), Rc::new(string), tail));
- }
- Ok((string, None)) => {
- term_stack.push(Term::CompleteString(Cell::default(), Rc::new(string)));
- }
- Err(cons_term) => term_stack.push(cons_term),
+ match as_partial_string(head, tail) {
+ Ok((string, Some(tail))) => {
+ term_stack.push(Term::PartialString(Cell::default(), Rc::new(string), tail));
}
+ Ok((string, None)) => {
+ term_stack.push(Term::CompleteString(Cell::default(), Rc::new(string)));
+ }
+ Err(cons_term) => term_stack.push(cons_term),
}
- (HeapCellValueTag::StackVar, h) => {
- term_stack.push(Term::Var(Cell::default(), VarPtr::from(format!("s_{}", h))));
- }
- (HeapCellValueTag::Var | HeapCellValueTag::AttrVar, h) => {
- term_stack.push(Term::Var(Cell::default(), VarPtr::from(format!("_{}", h))));
+ }
+ (HeapCellValueTag::Cons | HeapCellValueTag::Fixnum | HeapCellValueTag::F64) => {
+ term_stack.push(Term::Literal(Cell::default(), Literal::try_from(addr).unwrap()));
+ }
+ (HeapCellValueTag::StackVar, h) => {
+ term_stack.push(Term::Var(Cell::default(), VarPtr::from(format!("s_{}", h))));
+ }
+ (HeapCellValueTag::Var | HeapCellValueTag::AttrVar, h) => {
+ term_stack.push(Term::Var(Cell::default(), VarPtr::from(format!("_{}", h))));
+ }
+ (HeapCellValueTag::Atom, (name, arity)) => {
+ let h = iter.focus().value() as usize;
+ let mut arity = arity;
+ let value = iter.heap[h.saturating_sub(1)];
+
+ if let Some(idx) = get_structure_index(value) {
+ term_stack.push(Term::Literal(Cell::default(), Literal::CodeIndex(idx)));
+ arity += 1;
}
- (HeapCellValueTag::Atom, (name, arity)) => {
- let h = iter.focus().value() as usize;
- let mut arity = arity;
- let value = iter.heap[h.saturating_sub(1)];
-
- if let Some(idx) = get_structure_index(value) {
- term_stack.push(Term::Literal(Cell::default(), Literal::CodeIndex(idx)));
- arity += 1;
- }
- if arity == 0 {
- term_stack.push(Term::Literal(Cell::default(), Literal::Atom(name)));
- } else {
- let subterms = term_stack
- .drain(term_stack.len() - arity ..)
- .collect();
+ if arity == 0 {
+ term_stack.push(Term::Literal(Cell::default(), Literal::Atom(name)));
+ } else {
+ let subterms = term_stack
+ .drain(term_stack.len() - arity ..)
+ .collect();
- term_stack.push(Term::Clause(Cell::default(), name, subterms));
- }
- }
- (HeapCellValueTag::PStrLoc, h) => {
- let HeapStringScan { string, .. } = iter.heap.scan_slice_to_str(h);
- let tail = term_stack.pop().unwrap();
-
- term_stack.push(if matches!(tail, Term::Literal(_, Literal::Atom(atom!("[]")))) {
- Term::CompleteString(
- Cell::default(),
- Rc::new(string.to_owned()),
- )
- } else {
- Term::PartialString(
- Cell::default(),
- Rc::new(string.to_owned()),
- Box::new(tail),
- )
- });
- }
- _ => {
+ term_stack.push(Term::Clause(Cell::default(), name, subterms));
}
- );
- }
+ }
+ (HeapCellValueTag::PStrLoc, h) => {
+ let HeapStringScan { string, .. } = iter.heap.scan_slice_to_str(h);
+ let tail = term_stack.pop().unwrap();
+
+ term_stack.push(if matches!(tail, Term::Literal(_, Literal::Atom(atom!("[]")))) {
+ Term::CompleteString(
+ Cell::default(),
+ Rc::new(string.to_owned()),
+ )
+ } else {
+ Term::PartialString(
+ Cell::default(),
+ Rc::new(string.to_owned()),
+ Box::new(tail),
+ )
+ });
+ }
+ _ => {
+ }
+ );
}
debug_assert!(term_stack.len() == 1);
use crate::machine::streams::{Stream, StreamOptions};
use crate::machine::ClauseType;
use crate::machine::MachineStubGen;
+use crate::offset_table::*;
use fxhash::FxBuildHasher;
use indexmap::{IndexMap, IndexSet};
use std::cmp::Ordering;
use std::collections::BTreeSet;
-use std::ops::{Deref, DerefMut};
+use std::ops::Deref;
use crate::types::*;
}
#[derive(Debug, Clone, Copy, Ord, Hash, PartialOrd, Eq, PartialEq)]
-pub struct CodeIndex(TypedArenaPtr<IndexPtr>);
+pub struct CodeIndex(CodeIndexOffset);
#[cfg(target_pointer_width = "32")]
const_assert!(std::mem::align_of::<CodeIndex>() == 4);
#[cfg(target_pointer_width = "64")]
const_assert!(std::mem::align_of::<CodeIndex>() == 8);
-impl Deref for CodeIndex {
- type Target = TypedArenaPtr<IndexPtr>;
-
- #[inline(always)]
- fn deref(&self) -> &TypedArenaPtr<IndexPtr> {
- &self.0
- }
-}
-
-impl DerefMut for CodeIndex {
- #[inline(always)]
- fn deref_mut(&mut self) -> &mut TypedArenaPtr<IndexPtr> {
- &mut self.0
- }
-}
-
-impl From<CodeIndex> for UntypedArenaPtr {
- #[inline(always)]
- fn from(ptr: CodeIndex) -> UntypedArenaPtr {
- UntypedArenaPtr::build_with(ptr.0.as_ptr() as usize)
- }
-}
-
-impl From<TypedArenaPtr<IndexPtr>> for CodeIndex {
+impl From<CodeIndex> for HeapCellValue {
#[inline(always)]
- fn from(ptr: TypedArenaPtr<IndexPtr>) -> CodeIndex {
- CodeIndex(ptr)
+ fn from(idx: CodeIndex) -> HeapCellValue {
+ HeapCellValue::from(idx.as_ptr())
}
}
-impl From<CodeIndex> for HeapCellValue {
+impl From<CodeIndexOffset> for CodeIndex {
#[inline(always)]
- fn from(idx: CodeIndex) -> HeapCellValue {
- untyped_arena_ptr_as_cell!(UntypedArenaPtr::from(idx))
+ fn from(offset: CodeIndexOffset) -> CodeIndex {
+ CodeIndex(offset)
}
}
impl CodeIndex {
#[inline]
pub(crate) fn new(ptr: IndexPtr, arena: &mut Arena) -> Self {
- CodeIndex(arena_alloc!(ptr, arena))
+ unsafe { CodeIndex(arena.code_index_tbl.build_with(ptr)) }
}
#[inline(always)]
}
pub(crate) fn local(&self) -> Option<usize> {
- match self.0.tag() {
- IndexPtrTag::Index => Some(self.0.p() as usize),
- IndexPtrTag::DynamicIndex => Some(self.0.p() as usize),
+ match self.0.as_ptr().tag() {
+ IndexPtrTag::Index => Some(self.get().p() as usize),
+ IndexPtrTag::DynamicIndex => Some(self.get().p() as usize),
_ => None,
}
}
#[inline(always)]
pub(crate) fn get(&self) -> IndexPtr {
- *self.0.deref()
+ *self.as_ptr().deref()
}
#[inline(always)]
pub(crate) fn set(&mut self, value: IndexPtr) {
- *self.0.deref_mut() = value;
+ self.as_ptr().set(value);
}
#[inline(always)]
pub(crate) fn get_tag(self) -> IndexPtrTag {
- self.0.tag()
+ self.get().tag()
}
#[inline(always)]
pub(crate) fn replace(&mut self, value: IndexPtr) -> IndexPtr {
- std::mem::replace(self.0.deref_mut(), value)
+ self.as_ptr().replace(value)
}
- /*
#[inline(always)]
- pub(crate) fn as_ptr(&self) -> *const IndexPtr {
+ pub(crate) fn as_ptr(&self) -> CodeIndexPtr {
self.0.as_ptr()
}
- */
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
use crate::machine::partial_string::*;
use crate::machine::stack::*;
use crate::machine::unify::*;
+use crate::offset_table::*;
use crate::parser::ast::*;
use crate::parser::dashu::{Integer, Rational};
use crate::types::*;
use crate::machine::machine_state::*;
use crate::machine::stack::*;
use crate::machine::streams::*;
+use crate::offset_table::*;
use crate::parser::ast::*;
use crate::parser::dashu::{Integer, Rational};
use crate::types::*;
#[inline]
pub(crate) fn get_structure_index(value: HeapCellValue) -> Option<CodeIndex> {
read_heap_cell!(value,
- (HeapCellValueTag::Cons, cons_ptr) => {
- match_untyped_arena_ptr!(cons_ptr,
- (ArenaHeaderTag::IndexPtr, ip) => {
- return Some(CodeIndex::from(ip));
- }
- _ => {}
- );
+ (HeapCellValueTag::CodeIndex, ip) => {
+ return Some(ip);
}
_ => {
}
};
if let Some(code_index) = index_cell {
- if !code_index.is_undefined() {
+ if !code_index.as_ptr().is_undefined() {
load_registers(&mut self.machine_st, goal, goal_arity);
self.machine_st.neck_cut();
return call_at_index(self, name, arity, code_index.get());
let expanded_term = if result.is_simple_goal {
let idx = self.get_or_insert_qualified_code_index(module_name, result.key);
- self.machine_st.heap[result.index_ptr_loc] =
- untyped_arena_ptr_as_cell!(UntypedArenaPtr::from(idx));
+ self.machine_st.heap[result.index_ptr_loc] = HeapCellValue::from(idx);
result.goal
} else {
let mut unexpanded_vars = IndexSet::with_hasher(FxBuildHasher::default());
);
writer.write_with(|section| {
- section.push_cell(untyped_arena_ptr_as_cell!(UntypedArenaPtr::from(idx)));
+ section.push_cell(HeapCellValue::from(idx));
section.push_cell(atom_as_cell!(atom!("$aux"), 0));
for value in unexpanded_vars.difference(&result.supp_vars).cloned() {
let idx_cell = self.machine_st.heap[s.saturating_sub(1)];
- if HeapCellValueTag::Cons == idx_cell.get_tag() {
- match_untyped_arena_ptr!(cell_as_untyped_arena_ptr!(idx_cell),
- (ArenaHeaderTag::IndexPtr, _ip) => {
- return true;
- }
- _ => {
- }
- );
+ if HeapCellValueTag::CodeIndex == idx_cell.get_tag() {
+ return true;
}
}
use crate::heap_iter::{stackful_preorder_iter, NonListElider};
use crate::machine::machine_state::*;
use crate::machine::*;
+use crate::offset_table::*;
use crate::types::*;
use std::ops::{Deref, DerefMut};
macro_rules! cell_as_f64_ptr {
($cell:expr) => {{
let offset = $cell.get_value() as usize;
- F64Ptr::from_offset(F64Offset::new(offset))
+ F64Ptr::from_offset(F64Offset::from(offset))
+ }};
+}
+
+macro_rules! cell_as_code_index {
+ ($cell:expr) => {{
+ let offset = $cell.get_value() as usize;
+ CodeIndex::from(CodeIndexOffset::from(offset))
}};
}
($ptr:expr) => {{
// Cell is 64-bit, but raw ptr is 32-bit in 32-bit systems
// TODO use <*{const,mut} _>::addr instead of as when the strict_provenance feature is stable rust-lang/rust#95228
- // we might need <*{const,mut} _>::expose_provenance for strict provenance, dependening on how we recreate a pointer later
+ // we might need <*{const,mut} _>::expose_provenance for strict provenance, depending on how we recreate a pointer later
let ptr : *const _ = $ptr;
HeapCellValue::from_ptr_addr(ptr as usize)
}};
#[allow(unused_braces)]
$code
}};
- ($ptr:ident, IndexPtr, $ip:ident, $code:expr) => {{
- #[allow(unused_mut)]
- let mut $ip = unsafe { $ptr.as_typed_ptr::<IndexPtr>() };
- #[allow(unused_braces)]
- $code
- }};
($ptr:ident, $($tags:tt)|+, $s:ident, $code:expr) => {{
let $s = Stream::from_tag($ptr.get_tag(), $ptr);
#[allow(unused_braces)]
| ArenaHeaderTag::StandardOutputStream
| ArenaHeaderTag::StandardErrorStream
};
- (IndexPtr) => {
- ArenaHeaderTag::IndexPtrUndefined
- | ArenaHeaderTag::IndexPtrDynamicUndefined
- | ArenaHeaderTag::IndexPtrDynamicIndex
- | ArenaHeaderTag::IndexPtrIndex
- };
($tag:ident) => {
ArenaHeaderTag::$tag
};
#[allow(unused_braces)]
$code
}};
+ ($cell:ident, CodeIndex, $n:ident, $code:expr) => {{
+ let $n = cell_as_code_index!($cell);
+ #[allow(unused_braces)]
+ $code
+ }};
($cell:ident, Atom, ($name:ident, $arity:ident), $code:expr) => {{
let ($name, $arity) = cell_as_atom_cell!($cell).get_name_and_arity();
#[allow(unused_braces)]
--- /dev/null
+use std::cell::UnsafeCell;
+use std::hash::{Hash, Hasher};
+use std::ops::{Deref, DerefMut};
+use std::sync::RwLock;
+use std::sync::Weak;
+use std::sync::{Arc, Mutex};
+use std::{fmt, mem, ptr};
+
+use arcu::atomic::Arcu;
+use arcu::epoch_counters::GlobalEpochCounterPool;
+use arcu::rcu_ref::RcuRef;
+use arcu::Rcu;
+
+use crate::machine::machine_indices::IndexPtr;
+use crate::raw_block::RawBlock;
+use crate::raw_block::RawBlockTraits;
+
+use ordered_float::OrderedFloat;
+
+const F64_TABLE_INIT_SIZE: usize = 1 << 16;
+const F64_TABLE_ALIGN: usize = 8;
+
+const CODE_INDEX_TABLE_INIT_SIZE: usize = 1 << 16;
+const CODE_INDEX_TABLE_ALIGN: usize = 8;
+
+#[derive(Debug)]
+pub struct OffsetTableImpl<T>
+where
+ OffsetTableImpl<T>: RawBlockTraits,
+{
+ block: Arcu<RawBlock<OffsetTableImpl<T>>, GlobalEpochCounterPool>,
+ update: Mutex<()>,
+}
+
+pub type F64Table = OffsetTableImpl<OrderedFloat<f64>>;
+pub type CodeIndexTable = OffsetTableImpl<IndexPtr>;
+
+impl RawBlockTraits for F64Table {
+ #[inline]
+ fn init_size() -> usize {
+ F64_TABLE_INIT_SIZE
+ }
+
+ #[inline]
+ fn align() -> usize {
+ F64_TABLE_ALIGN
+ }
+}
+
+impl RawBlockTraits for CodeIndexTable {
+ #[inline]
+ fn init_size() -> usize {
+ CODE_INDEX_TABLE_INIT_SIZE
+ }
+
+ #[inline]
+ fn align() -> usize {
+ CODE_INDEX_TABLE_ALIGN
+ }
+}
+
+pub trait OffsetTable: RawBlockTraits {
+ type Offset: Copy + From<usize> + Into<usize>;
+ type Stored;
+
+ fn global_table() -> &'static RwLock<Weak<Self>>;
+}
+
+impl OffsetTable for F64Table {
+ type Offset = F64Offset;
+ type Stored = OrderedFloat<f64>;
+
+ #[inline(always)]
+ fn global_table() -> &'static RwLock<Weak<Self>> {
+ static GLOBAL_ATOM_TABLE: RwLock<Weak<F64Table>> = RwLock::new(Weak::new());
+ &GLOBAL_ATOM_TABLE
+ }
+}
+
+impl OffsetTable for CodeIndexTable {
+ type Offset = CodeIndexOffset;
+ type Stored = IndexPtr;
+
+ #[inline(always)]
+ fn global_table() -> &'static RwLock<Weak<CodeIndexTable>> {
+ static GLOBAL_CODE_INDEX_TABLE: RwLock<Weak<CodeIndexTable>> = RwLock::new(Weak::new());
+ &GLOBAL_CODE_INDEX_TABLE
+ }
+}
+
+impl<T: 'static> OffsetTableImpl<T>
+where
+ OffsetTableImpl<T>: OffsetTable<Stored = T>,
+{
+ #[inline]
+ pub fn new() -> Arc<Self> {
+ let upgraded = Self::global_table().read().unwrap().upgrade();
+ // don't inline upgraded, otherwise temporary will be dropped too late in case of None
+ if let Some(atom_table) = upgraded {
+ atom_table
+ } else {
+ let mut guard = Self::global_table().write().unwrap();
+ // try to upgrade again in case we lost the race on the write lock
+ if let Some(atom_table) = guard.upgrade() {
+ atom_table
+ } else {
+ let table = Arc::new(Self {
+ block: Arcu::new(RawBlock::new(), GlobalEpochCounterPool),
+ update: Mutex::new(()),
+ });
+ *guard = Arc::downgrade(&table);
+ table
+ }
+ }
+ }
+
+ #[allow(clippy::missing_safety_doc)]
+ pub unsafe fn build_with(
+ &self,
+ value: <OffsetTableImpl<T> as OffsetTable>::Stored,
+ ) -> <OffsetTableImpl<T> as OffsetTable>::Offset {
+ let update_guard = self.update.lock();
+
+ // we don't have an index table for lookups as AtomTable does so
+ // just get the epoch after we take the upgrade lock
+ let mut block_epoch = self.block.read();
+
+ let mut ptr;
+
+ loop {
+ ptr = block_epoch.alloc(mem::size_of::<T>());
+
+ if ptr.is_null() {
+ let new_block = block_epoch.grow_new().unwrap();
+ self.block.replace(new_block);
+ block_epoch = self.block.read();
+ } else {
+ break;
+ }
+ }
+
+ ptr::write(ptr as *mut T, value);
+
+ let value = <OffsetTableImpl<T> as OffsetTable>::Offset::from(
+ ptr as usize - block_epoch.base as usize,
+ );
+
+ // AtomTable would have to update the index table at this point
+ // explicit drop to ensure we don't accidentally drop it early
+ drop(update_guard);
+
+ value
+ }
+
+ pub fn lookup(offset: <Self as OffsetTable>::Offset) -> RcuRef<RawBlock<Self>, UnsafeCell<T>> {
+ let table = Self::global_table()
+ .read()
+ .unwrap()
+ .upgrade()
+ .expect("We should only be looking up entries when there is a table");
+
+ RcuRef::try_map(table.block.read(), |raw_block| unsafe {
+ raw_block
+ .base
+ .add(offset.into())
+ .cast_mut()
+ .cast::<UnsafeCell<T>>()
+ .as_ref()
+ })
+ .expect("The offset should result in a non-null pointer")
+ }
+}
+
+#[derive(Debug)]
+pub struct TablePtr<T>(RcuRef<RawBlock<OffsetTableImpl<T>>, UnsafeCell<T>>)
+where
+ OffsetTableImpl<T>: RawBlockTraits;
+
+pub type CodeIndexPtr = TablePtr<IndexPtr>;
+pub type F64Ptr = TablePtr<OrderedFloat<f64>>;
+
+impl<T> Clone for TablePtr<T>
+where
+ OffsetTableImpl<T>: RawBlockTraits,
+{
+ fn clone(&self) -> Self {
+ Self(RcuRef::clone(&self.0))
+ }
+}
+
+impl<T: PartialEq> PartialEq for TablePtr<T>
+where
+ OffsetTableImpl<T>: RawBlockTraits,
+{
+ fn eq(&self, other: &TablePtr<T>) -> bool {
+ RcuRef::ptr_eq(&self.0, &other.0) || self.deref() == other.deref()
+ }
+}
+
+impl<T: Eq> Eq for TablePtr<T> where OffsetTableImpl<T>: RawBlockTraits {}
+
+impl<T: PartialOrd + Ord> PartialOrd for TablePtr<T>
+where
+ OffsetTableImpl<T>: RawBlockTraits,
+{
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ Some(self.cmp(other))
+ }
+}
+
+impl<T: Ord> Ord for TablePtr<T>
+where
+ OffsetTableImpl<T>: RawBlockTraits,
+{
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ (**self).cmp(&**other)
+ }
+}
+
+impl<T: Hash> Hash for TablePtr<T>
+where
+ OffsetTableImpl<T>: RawBlockTraits,
+{
+ #[inline(always)]
+ fn hash<H: Hasher>(&self, hasher: &mut H) {
+ (self as &T).hash(hasher)
+ }
+}
+
+impl<T: fmt::Display> fmt::Display for TablePtr<T>
+where
+ OffsetTableImpl<T>: RawBlockTraits,
+{
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ write!(f, "{}", self as &T)
+ }
+}
+
+impl<T> Deref for TablePtr<T>
+where
+ OffsetTableImpl<T>: RawBlockTraits,
+{
+ type Target = T;
+
+ #[inline]
+ fn deref(&self) -> &Self::Target {
+ unsafe { self.0.get().as_ref().unwrap() }
+ }
+}
+
+impl<T> DerefMut for TablePtr<T>
+where
+ OffsetTableImpl<T>: RawBlockTraits,
+{
+ #[inline]
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ unsafe { &mut *self.0.get().as_mut().unwrap() }
+ }
+}
+
+impl<T: 'static> TablePtr<T>
+where
+ OffsetTableImpl<T>: OffsetTable<Stored = T>,
+{
+ #[inline(always)]
+ pub fn from_offset(offset: <OffsetTableImpl<T> as OffsetTable>::Offset) -> Self {
+ Self(OffsetTableImpl::<T>::lookup(offset))
+ }
+
+ #[inline(always)]
+ pub fn as_offset(&self) -> <OffsetTableImpl<T> as OffsetTable>::Offset {
+ <OffsetTableImpl<T> as OffsetTable>::Offset::from(
+ self.0.get() as usize - RcuRef::get_root(&self.0).base as usize,
+ )
+ }
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct F64Offset(usize);
+
+impl From<usize> for F64Offset {
+ #[inline(always)]
+ fn from(offset: usize) -> Self {
+ Self(offset)
+ }
+}
+
+impl Into<usize> for F64Offset {
+ #[inline(always)]
+ fn into(self: Self) -> usize {
+ self.0
+ }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct CodeIndexOffset(usize);
+
+impl From<usize> for CodeIndexOffset {
+ #[inline(always)]
+ fn from(offset: usize) -> Self {
+ Self(offset)
+ }
+}
+
+impl Into<usize> for CodeIndexOffset {
+ #[inline(always)]
+ fn into(self: Self) -> usize {
+ self.0
+ }
+}
+
+impl CodeIndexOffset {
+ #[inline(always)]
+ pub fn from_ptr(ptr: CodeIndexPtr) -> Self {
+ ptr.as_offset()
+ }
+
+ #[inline(always)]
+ pub fn as_ptr(self) -> CodeIndexPtr {
+ CodeIndexPtr::from_offset(self)
+ }
+
+ #[inline(always)]
+ pub fn to_u64(self) -> u64 {
+ self.0 as u64
+ }
+}
+
+impl PartialEq for CodeIndexOffset {
+ #[inline(always)]
+ fn eq(&self, other: &CodeIndexOffset) -> bool {
+ self.as_ptr() == other.as_ptr()
+ }
+}
+
+impl Eq for CodeIndexOffset {}
+
+impl PartialOrd for CodeIndexOffset {
+ #[inline(always)]
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ Some(self.cmp(other))
+ }
+}
+
+impl Ord for CodeIndexOffset {
+ #[inline(always)]
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.as_ptr().cmp(&other.as_ptr())
+ }
+}
+
+impl Hash for CodeIndexOffset {
+ #[inline(always)]
+ fn hash<H: Hasher>(&self, hasher: &mut H) {
+ self.as_ptr().hash(hasher)
+ }
+}
+
+impl fmt::Display for CodeIndexOffset {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ write!(f, "CodeIndexOffset({})", self.0)
+ }
+}
+
+impl CodeIndexPtr {
+ #[inline]
+ pub fn set(&self, val: IndexPtr) {
+ unsafe { *self.0.get() = val };
+ }
+
+ #[inline]
+ pub fn replace(&self, val: IndexPtr) -> IndexPtr {
+ unsafe { self.0.get().replace(val) }
+ }
+}
+
+impl F64Offset {
+ #[inline(always)]
+ pub fn from_ptr(ptr: F64Ptr) -> Self {
+ ptr.as_offset()
+ }
+
+ #[inline(always)]
+ pub fn as_ptr(self) -> F64Ptr {
+ F64Ptr::from_offset(self)
+ }
+
+ #[inline(always)]
+ pub fn to_u64(self) -> u64 {
+ self.0 as u64
+ }
+}
+
+impl PartialEq for F64Offset {
+ #[inline(always)]
+ fn eq(&self, other: &F64Offset) -> bool {
+ self.as_ptr() == other.as_ptr()
+ }
+}
+
+impl Eq for F64Offset {}
+
+impl PartialOrd for F64Offset {
+ #[inline(always)]
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ Some(self.cmp(other))
+ }
+}
+
+impl Ord for F64Offset {
+ #[inline(always)]
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.as_ptr().cmp(&other.as_ptr())
+ }
+}
+
+impl Hash for F64Offset {
+ #[inline(always)]
+ fn hash<H: Hasher>(&self, hasher: &mut H) {
+ self.as_ptr().hash(hasher)
+ }
+}
+
+impl fmt::Display for F64Offset {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ write!(f, "F64Offset({})", self.0)
+ }
+}
use crate::arena::*;
use crate::atom_table::*;
use crate::machine::machine_indices::CodeIndex;
+use crate::offset_table::*;
use crate::parser::char_reader::*;
use crate::types::HeapCellValueTag;
Literal::Atom(ref atom) => {
write!(f, "{}", atom.flat_index())
}
- // Literal::Char(c) => write!(f, "'{}'", *c as u32),
- Literal::CodeIndex(i) => write!(f, "{:x}", i.as_ptr() as u64),
+ Literal::CodeIndex(i) => write!(f, "{:?}", *i.as_ptr()),
Literal::Fixnum(n) => write!(f, "{}", n.get_num()),
Literal::Integer(ref n) => write!(f, "{}", n),
Literal::Rational(ref n) => write!(f, "{}", n),
use crate::parser::char_reader::*;
use crate::parser::dashu::Integer;
+use ordered_float::OrderedFloat;
+
use std::convert::TryFrom;
use std::fmt;
use crate::parser::char_reader::*;
use crate::parser::lexer::*;
+use ordered_float::OrderedFloat;
+
use std::cell::Cell;
use std::mem;
use std::ops::Neg;
use crate::machine::heap::*;
use crate::machine::machine_indices::*;
use crate::machine::streams::*;
+use crate::offset_table::*;
use crate::parser::ast::Fixnum;
use crate::parser::ast::Literal;
Cons = 0b0,
F64 = 0b010101,
Fixnum = 0b011001,
- // Char = 0b011011,
+ CodeIndex = 0b011011,
Atom = 0b011111,
CutPoint = 0b011101,
// trail elements.
Cons = 0b0,
F64 = 0b010101,
Fixnum = 0b011001,
- Char = 0b011011,
+ CodeIndex = 0b011011,
Atom = 0b011111,
CutPoint = 0b011101,
// trail elements.
fn from(literal: Literal) -> Self {
match literal {
Literal::Atom(name) => atom_as_cell!(name),
- Literal::CodeIndex(ptr) => {
- untyped_arena_ptr_as_cell!(UntypedArenaPtr::from(ptr))
- }
+ Literal::CodeIndex(idx) => HeapCellValue::from(idx),
Literal::Fixnum(n) => fixnum_as_cell!(n),
Literal::Integer(bigint_ptr) => {
typed_arena_ptr_as_cell!(bigint_ptr)
(HeapCellValueTag::F64, f) => {
Ok(Literal::Float(f.as_offset()))
}
+ (HeapCellValueTag::CodeIndex, idx) => {
+ Ok(Literal::CodeIndex(idx))
+ }
(HeapCellValueTag::Cons, cons_ptr) => {
match_untyped_arena_ptr!(cons_ptr,
(ArenaHeaderTag::Integer, n) => {
(ArenaHeaderTag::Rational, n) => {
Ok(Literal::Rational(n))
}
- (ArenaHeaderTag::IndexPtr, ip) => {
- Ok(Literal::CodeIndex(CodeIndex::from(ip)))
- }
_ => {
Err(())
}
}
}
+impl From<CodeIndexPtr> for HeapCellValue {
+ #[inline]
+ fn from(code_index_ptr: CodeIndexPtr) -> HeapCellValue {
+ HeapCellValue::build_with(
+ HeapCellValueTag::CodeIndex,
+ code_index_ptr.as_offset().to_u64(),
+ )
+ }
+}
+
impl From<ConsPtr> for HeapCellValue {
#[inline(always)]
fn from(cons_ptr: ConsPtr) -> HeapCellValue {
}
#[inline]
- pub fn get_ptr(self) -> *const ArenaHeader {
- unsafe { mem::transmute::<_, *const ArenaHeader>(self.ptr()) }
+ pub fn get_ptr(self) -> *const u8 {
+ let addr: u64 = self.ptr();
+ addr as usize as *const u8
}
#[inline]
pub fn get_tag(self) -> ArenaHeaderTag {
unsafe {
- let header = *self.get_ptr();
+ let header = *(self.get_ptr() as *const ArenaHeader);
header.get_tag()
}
}
#[inline]
pub fn payload_offset(self) -> *const u8 {
- unsafe { self.get_ptr().byte_add(mem::size_of::<ArenaHeader>()) as *const _ }
+ unsafe { self.get_ptr().add(mem::size_of::<ArenaHeader>()) }
}
/// # Safety
projects / scryer-prolog.git / commitdiff