From: Mark Thom Date: Wed, 25 Jun 2025 04:58:40 +0000 (-0700) Subject: use granular hierarchical locks in offset_table.rs X-Git-Tag: v0.10.0~35^2~8 X-Git-Url: https://git.sagredo.dev/?a=commitdiff_plain;h=762b63e1f41dff5d6d19b154b528ab58bf1e2f6b;p=scryer-prolog.git use granular hierarchical locks in offset_table.rs --- diff --git a/Cargo.lock b/Cargo.lock index 52a3a76a..0feaf573 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1673,9 +1673,9 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "lock_api" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", @@ -2011,9 +2011,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", "parking_lot_core", @@ -2021,9 +2021,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", @@ -2712,6 +2712,7 @@ dependencies = [ "num-order", "ordered-float", "ouroboros", + "parking_lot", "phf", "pprof", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 22609228..3087a3a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,6 +79,7 @@ ego-tree = "0.10.0" serde_json = "1.0.122" serde = "1.0.204" +parking_lot = "0.12.4" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] crossterm = { version = "0.28.1", optional = true } diff --git a/src/arena.rs b/src/arena.rs index 2be3a9f9..2a3fc8b9 100644 --- a/src/arena.rs +++ b/src/arena.rs @@ -570,8 +570,6 @@ const_assert!(mem::size_of::>() == 8); #[cfg(test)] mod tests { - use std::ops::Deref; - use crate::arena::*; use crate::atom_table::*; use crate::machine::mock_wam::*; @@ -590,10 +588,7 @@ mod tests { assert_eq!(cell.get_tag(), HeapCellValueTag::F64Offset); assert!(!cell.get_mark_bit()); - assert_eq!( - wam.machine_st.arena.f64_tbl.lookup(fp).deref(), - &OrderedFloat(f) - ); + assert_eq!(wam.machine_st.arena.f64_tbl.get_entry(fp), OrderedFloat(f)); cell.set_mark_bit(true); @@ -601,8 +596,8 @@ mod tests { read_heap_cell!(cell, (HeapCellValueTag::F64Offset, offset) => { - let fp = wam.machine_st.arena.f64_tbl.lookup(offset.into()); - assert_eq!(*fp, OrderedFloat(0f64)) + let fp = wam.machine_st.arena.f64_tbl.get_entry(offset); + assert_eq!(fp, OrderedFloat(0f64)) } _ => { unreachable!() } ); diff --git a/src/arithmetic.rs b/src/arithmetic.rs index a4aad3f8..f7a86569 100644 --- a/src/arithmetic.rs +++ b/src/arithmetic.rs @@ -166,7 +166,7 @@ fn push_literal( Literal::Fixnum(n) => interm.push(ArithmeticTerm::Number(Number::Fixnum(*n))), Literal::Integer(n) => interm.push(ArithmeticTerm::Number(Number::Integer(*n))), &Literal::F64Offset(offset) => { - let n = *f64_tbl.lookup(offset); + let n = f64_tbl.get_entry(offset); interm.push(ArithmeticTerm::Number(Number::Float(n))); } Literal::Rational(n) => interm.push(ArithmeticTerm::Number(Number::Rational(*n))), @@ -685,7 +685,7 @@ impl TryFrom<(HeapCellValue, &'_ F64Table)> for Number { ) } (HeapCellValueTag::F64Offset, offset) => { - let n = *f64_tbl.lookup(offset.into()); + let n = f64_tbl.get_entry(offset); Ok(Number::Float(n)) } (HeapCellValueTag::Fixnum | HeapCellValueTag::CutPoint, n) => { diff --git a/src/heap_print.rs b/src/heap_print.rs index d45e45a9..9cc3a2d0 100644 --- a/src/heap_print.rs +++ b/src/heap_print.rs @@ -1544,7 +1544,7 @@ impl<'a, Outputter: HCValueOutputter> HCPrinter<'a, Outputter> { self.state_stack.pop(); self.state_stack.pop(); - let idx_ptr = self.arena.code_index_tbl.lookup(idx); + let idx_ptr = self.arena.code_index_tbl.get_entry(idx); let offset = if idx_ptr.is_undefined() || idx_ptr.is_dynamic_undefined() { TokenOrRedirect::Atom(atom!("undefined")) @@ -1749,7 +1749,7 @@ impl<'a, Outputter: HCValueOutputter> HCPrinter<'a, Outputter> { self.print_number(max_depth, NumberFocus::Unfocused(Number::Fixnum(n)), &op); } (HeapCellValueTag::F64Offset, offset) => { - let f = *self.arena.f64_tbl.lookup(offset.into()); + let f = self.arena.f64_tbl.get_entry(offset); self.print_number(max_depth, NumberFocus::Unfocused(Number::Float(f)), &op); } (HeapCellValueTag::PStrLoc) => { diff --git a/src/machine/arithmetic_ops.rs b/src/machine/arithmetic_ops.rs index 8e4c0552..c0c3f2bb 100644 --- a/src/machine/arithmetic_ops.rs +++ b/src/machine/arithmetic_ops.rs @@ -1389,8 +1389,8 @@ impl MachineState { self.interms.push(Number::Fixnum(n)); } (HeapCellValueTag::F64Offset, offset) => { - let fl = self.arena.f64_tbl.lookup(offset); - self.interms.push(Number::Float(*fl)); + let fl = self.arena.f64_tbl.get_entry(offset); + self.interms.push(Number::Float(fl)); } (HeapCellValueTag::Cons, ptr) => { match_untyped_arena_ptr!(ptr, diff --git a/src/machine/compile.rs b/src/machine/compile.rs index fcd2364f..7dcbee14 100644 --- a/src/machine/compile.rs +++ b/src/machine/compile.rs @@ -1328,17 +1328,15 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> { let index_ptr = LS::machine_st(&mut self.payload) .arena .code_index_tbl - .lookup(code_idx.into()); + .get_entry(code_idx.into()); print_overwrite_warning( &predicates.compilation_target, - *index_ptr, + index_ptr, key, settings.is_dynamic(), ); - drop(index_ptr); - let index_ptr = if settings.is_dynamic() { IndexPtr::dynamic_index(code_ptr) } else { @@ -2231,10 +2229,10 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> { if let Some(filename) = self.listing_src_file_name() { if let Some(ref mut module) = self.wam_prelude.indices.modules.get_mut(&filename) { - let index_ptr = *LS::machine_st(&mut self.payload) + let index_ptr = LS::machine_st(&mut self.payload) .arena .code_index_tbl - .lookup_mut(offset.into()); + .get_entry(offset.into()); let offset = *module.code_dir.entry(key).or_insert(offset); diff --git a/src/machine/dispatch.rs b/src/machine/dispatch.rs index 59816f17..2428f7d2 100644 --- a/src/machine/dispatch.rs +++ b/src/machine/dispatch.rs @@ -547,8 +547,8 @@ impl Machine { // Find the boundaries of the current predicate self.indices.code_dir.sort_by(|_, a, _, b| { - let a = *self.machine_st.arena.code_index_tbl.lookup((*a).into()); - let b = *self.machine_st.arena.code_index_tbl.lookup((*b).into()); + let a = self.machine_st.arena.code_index_tbl.get_entry((*a).into()); + let b = self.machine_st.arena.code_index_tbl.get_entry((*b).into()); a.cmp(&b) }); @@ -557,8 +557,11 @@ impl Machine { .indices .code_dir .binary_search_by_key(&p, |_, x| -> usize { - self.machine_st.arena.code_index_tbl.lookup((*x).into()).p() - as usize + self.machine_st + .arena + .code_index_tbl + .get_entry((*x).into()) + .p() as usize }) .unwrap_or_else(|x| x - 1); @@ -570,7 +573,7 @@ impl Machine { self.machine_st .arena .code_index_tbl - .lookup((*idx.1).into()) + .get_entry((*idx.1).into()) .p() as usize }) .unwrap(); @@ -585,7 +588,7 @@ impl Machine { self.machine_st .arena .code_index_tbl - .lookup((*idx.1).into()) + .get_entry((*idx.1).into()) .p() as usize }) .unwrap_or(self.code.len()); @@ -2696,7 +2699,7 @@ impl Machine { } } &Instruction::CallNamed(arity, name, idx) => { - let idx = *self.machine_st.arena.code_index_tbl.lookup(idx.into()); + let idx = self.machine_st.arena.code_index_tbl.get_entry(idx.into()); try_or_throw!(self.machine_st, self.try_call(name, arity, idx)); @@ -2707,7 +2710,7 @@ impl Machine { } } &Instruction::ExecuteNamed(arity, name, idx) => { - let idx = *self.machine_st.arena.code_index_tbl.lookup(idx.into()); + let idx = self.machine_st.arena.code_index_tbl.get_entry(idx.into()); try_or_throw!(self.machine_st, self.try_execute(name, arity, idx)); @@ -2718,7 +2721,7 @@ impl Machine { } } &Instruction::DefaultCallNamed(arity, name, idx) => { - let idx = *self.machine_st.arena.code_index_tbl.lookup(idx.into()); + let idx = self.machine_st.arena.code_index_tbl.get_entry(idx.into()); try_or_throw!(self.machine_st, self.try_call(name, arity, idx)); @@ -2727,7 +2730,7 @@ impl Machine { } } &Instruction::DefaultExecuteNamed(arity, name, idx) => { - let idx = *self.machine_st.arena.code_index_tbl.lookup(idx.into()); + let idx = self.machine_st.arena.code_index_tbl.get_entry(idx.into()); try_or_throw!(self.machine_st, self.try_execute(name, arity, idx)); diff --git a/src/machine/lib_machine/mod.rs b/src/machine/lib_machine/mod.rs index dbaac207..a1ba3399 100644 --- a/src/machine/lib_machine/mod.rs +++ b/src/machine/lib_machine/mod.rs @@ -278,7 +278,7 @@ impl Term { } } (HeapCellValueTag::F64Offset, offset) => { - let f = *machine.machine_st.arena.f64_tbl.lookup(offset); + let f = machine.machine_st.arena.f64_tbl.get_entry(offset); term_stack.push(Term::Float(f.into())); } (HeapCellValueTag::Fixnum, n) => { @@ -602,7 +602,7 @@ impl Machine { self.machine_st .arena .code_index_tbl - .lookup(offset.into()) + .get_entry(offset.into()) .p() as usize }) .expect("couldn't get code index"); diff --git a/src/machine/load_state.rs b/src/machine/load_state.rs index 931ee273..40e9a383 100644 --- a/src/machine/load_state.rs +++ b/src/machine/load_state.rs @@ -22,33 +22,30 @@ pub(super) fn set_code_index<'a, LS: LoadState<'a>>( code_idx: CodeIndex, code_ptr: IndexPtr, ) { - let mut code_idx_ptr = LS::machine_st(payload) - .arena - .code_index_tbl - .lookup_mut(code_idx.into()); - - let record = match compilation_target { - CompilationTarget::User => { - if IndexPtrTag::Undefined == code_idx_ptr.tag() { - code_idx_ptr.set(code_ptr); - RetractionRecord::AddedUserPredicate(key) - } else { - let replaced = code_idx_ptr.replace(code_ptr); - RetractionRecord::ReplacedUserPredicate(key, replaced) + let record = LS::machine_st(payload).arena.code_index_tbl.with_entry_mut( + code_idx.into(), + |code_idx_ptr| match compilation_target { + CompilationTarget::User => { + if IndexPtrTag::Undefined == code_idx_ptr.tag() { + *code_idx_ptr = code_ptr; + RetractionRecord::AddedUserPredicate(key) + } else { + let replaced = mem::replace(code_idx_ptr, code_ptr); + RetractionRecord::ReplacedUserPredicate(key, replaced) + } } - } - CompilationTarget::Module(ref module_name) => { - if IndexPtrTag::Undefined == code_idx_ptr.tag() { - code_idx_ptr.set(code_ptr); - RetractionRecord::AddedModulePredicate(*module_name, key) - } else { - let replaced = code_idx_ptr.replace(code_ptr); - RetractionRecord::ReplacedModulePredicate(*module_name, key, replaced) + CompilationTarget::Module(ref module_name) => { + if IndexPtrTag::Undefined == code_idx_ptr.tag() { + *code_idx_ptr = code_ptr; + RetractionRecord::AddedModulePredicate(*module_name, key) + } else { + let replaced = mem::replace(code_idx_ptr, code_ptr); + RetractionRecord::ReplacedModulePredicate(*module_name, key, replaced) + } } - } - }; + }, + ); - drop(code_idx_ptr); payload.retraction_info.push_record(record); } @@ -145,7 +142,7 @@ pub(super) fn import_module_exports<'a, LS: LoadState<'a>>( .entry(key) .or_insert_with(|| CodeIndex::default(code_idx_tbl)); - let src_code_index_ptr = *code_idx_tbl.lookup(src_code_index.into()); + let src_code_index_ptr = code_idx_tbl.get_entry(src_code_index.into()); set_code_index::( payload, @@ -158,7 +155,7 @@ pub(super) fn import_module_exports<'a, LS: LoadState<'a>>( if LS::machine_st(payload) .arena .code_index_tbl - .lookup(src_code_index.into()) + .get_entry(src_code_index.into()) .is_dynamic_undefined() { code_dir.insert(key, src_code_index); @@ -205,7 +202,7 @@ fn import_module_exports_into_module<'a, LS: LoadState<'a>>( if let Some(src_code_index) = imported_module.code_dir.get(&key).cloned() { let code_index_tbl = &mut LS::machine_st(payload).arena.code_index_tbl; - let src_code_ptr = *code_index_tbl.lookup(src_code_index.into()); + let src_code_ptr = code_index_tbl.get_entry(src_code_index.into()); let target_code_index = *code_dir .entry(key) .or_insert_with(|| CodeIndex::default(code_index_tbl)); @@ -259,7 +256,7 @@ fn import_qualified_module_exports<'a, LS: LoadState<'a>>( if let Some(src_code_index) = imported_module.code_dir.get(&key).cloned() { let code_index_tbl = &mut LS::machine_st(payload).arena.code_index_tbl; - let src_code_ptr = *code_index_tbl.lookup(src_code_index.into()); + let src_code_ptr = code_index_tbl.get_entry(src_code_index.into()); let target_code_index = *wam_prelude.indices.code_dir.entry(key).or_insert_with(|| { CodeIndex::new(IndexPtr::undefined(), code_index_tbl) @@ -320,7 +317,7 @@ fn import_qualified_module_exports_into_module<'a, LS: LoadState<'a>>( if let Some(src_code_index) = imported_module.code_dir.get(&key).cloned() { let code_index_tbl = &mut LS::machine_st(payload).arena.code_index_tbl; - let src_code_ptr = *code_index_tbl.lookup(src_code_index.into()); + let src_code_ptr = code_index_tbl.get_entry(src_code_index.into()); let target_code_index = *code_dir .entry(key) .or_insert_with(|| CodeIndex::new(IndexPtr::undefined(), code_index_tbl)); @@ -505,10 +502,9 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> { } let code_index_tbl = &mut LS::machine_st(&mut self.payload).arena.code_index_tbl; - let code_ptr = code_index_tbl.lookup((*code_index).into()); + let code_ptr = code_index_tbl.get_entry((*code_index).into()); if !code_ptr.is_undefined() && !code_ptr.is_dynamic_undefined() { - drop(code_ptr); let old_index_ptr = code_index.replace(code_index_tbl, IndexPtr::undefined()); self.payload.retraction_info.push_record( @@ -557,13 +553,12 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> { (Some(module_code_idx), Some(target_code_idx)) => { let code_index_tbl = &mut LS::machine_st(payload).arena.code_index_tbl; - let module_code_ptr = code_index_tbl.lookup(module_code_idx.into()); - let target_code_ptr = code_index_tbl.lookup(target_code_idx.into()); + let module_code_ptr = + code_index_tbl.get_entry(module_code_idx.into()); + let target_code_ptr = + code_index_tbl.get_entry(target_code_idx.into()); if module_code_ptr == target_code_ptr { - drop(module_code_ptr); - drop(target_code_ptr); - let old_index_ptr = target_code_idx .replace(code_index_tbl, IndexPtr::undefined()); payload diff --git a/src/machine/loader.rs b/src/machine/loader.rs index 2a65a1f9..02b8a2b0 100644 --- a/src/machine/loader.rs +++ b/src/machine/loader.rs @@ -1225,11 +1225,9 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> { let code_idx_ptr = LS::machine_st(&mut self.payload) .arena .code_index_tbl - .lookup_mut(offset.into()); + .get_entry(offset.into()); if code_idx_ptr.is_undefined() { - drop(code_idx_ptr); - set_code_index::( &mut self.payload, &compilation_target, @@ -2015,8 +2013,7 @@ impl Machine { .machine_st .arena .code_index_tbl - .lookup(offset.into()) - .tag() + .with_entry(offset.into(), |idx| idx.tag()) }) .unwrap_or(IndexPtrTag::DynamicUndefined); @@ -2162,16 +2159,14 @@ impl Machine { let offset = loader.get_or_insert_code_index(key, compilation_target); - let mut code_idx = loader + loader .payload .machine_st .arena .code_index_tbl - .lookup_mut(offset.into()); - - code_idx.set(IndexPtr::undefined()); - - drop(code_idx); + .with_entry_mut(offset.into(), |code_idx| { + *code_idx = IndexPtr::undefined(); + }); loader.payload.compilation_target = clause_clause_compilation_target; diff --git a/src/machine/machine_indices.rs b/src/machine/machine_indices.rs index 66caba46..4f04c2b1 100644 --- a/src/machine/machine_indices.rs +++ b/src/machine/machine_indices.rs @@ -186,12 +186,12 @@ impl CodeIndex { #[inline(always)] pub(crate) fn set(&self, code_index_tbl: &mut CodeIndexTable, value: IndexPtr) { - code_index_tbl.lookup_mut(self.0).set(value); + code_index_tbl.with_entry_mut(self.0, |idx| *idx = value); } #[inline(always)] pub(crate) fn replace(&self, code_index_tbl: &mut CodeIndexTable, value: IndexPtr) -> IndexPtr { - code_index_tbl.lookup_mut(self.0).replace(value) + code_index_tbl.with_entry_mut(self.0, |idx| std::mem::replace(idx, value)) } } diff --git a/src/machine/machine_state_impl.rs b/src/machine/machine_state_impl.rs index 385d9a4e..f663656c 100644 --- a/src/machine/machine_state_impl.rs +++ b/src/machine/machine_state_impl.rs @@ -412,8 +412,8 @@ impl MachineState { let v1 = cell_as_f64_offset!(v1); let v2 = cell_as_f64_offset!(v2); - let v1 = self.arena.f64_tbl.lookup(v1); - let v2 = self.arena.f64_tbl.lookup(v2); + let v1 = self.arena.f64_tbl.get_entry(v1); + let v2 = self.arena.f64_tbl.get_entry(v2); if v1 != v2 { self.pdl.clear(); diff --git a/src/machine/mod.rs b/src/machine/mod.rs index e9e91f89..51ae44e4 100644 --- a/src/machine/mod.rs +++ b/src/machine/mod.rs @@ -253,7 +253,11 @@ impl Machine { ) -> std::process::ExitCode { if let Some(module) = self.indices.modules.get(&module_name) { if let Some(code_idx) = module.code_dir.get(&key) { - let index_ptr = *self.machine_st.arena.code_index_tbl.lookup(code_idx.into()); + let index_ptr = self + .machine_st + .arena + .code_index_tbl + .get_entry(code_idx.into()); let p = index_ptr.local().unwrap(); // Leave a halting choice point to backtrack to in case the predicate fails or throws. @@ -327,7 +331,11 @@ impl Machine { if let Some(module) = self.indices.modules.get(&atom!("$atts")) { if let Some(code_idx) = module.code_dir.get(&(atom!("driver"), 2)) { - let index_ptr = *self.machine_st.arena.code_index_tbl.lookup(code_idx.into()); + let index_ptr = self + .machine_st + .arena + .code_index_tbl + .get_entry(code_idx.into()); self.machine_st.attr_var_init.verify_attrs_loc = index_ptr.local().unwrap(); } } @@ -350,7 +358,7 @@ impl Machine { CodeIndex::new(IndexPtr::undefined(), code_index_tbl) }); - let src_code_ptr = *code_index_tbl.lookup(src_code_index.into()); + let src_code_ptr = code_index_tbl.get_entry(src_code_index.into()); target_code_index.set(code_index_tbl, src_code_ptr); } None => { @@ -1045,14 +1053,14 @@ impl Machine { if module_name == atom!("user") { if let Some(idx) = self.indices.code_dir.get(&(name, arity)).cloned() { - let index_ptr = *self.machine_st.arena.code_index_tbl.lookup(idx.into()); + let index_ptr = self.machine_st.arena.code_index_tbl.get_entry(idx.into()); self.try_call(name, arity, index_ptr) } else { Err(self.machine_st.throw_undefined_error(name, arity)) } } else if let Some(module) = self.indices.modules.get(&module_name) { if let Some(idx) = module.code_dir.get(&(name, arity)).cloned() { - let index_ptr = *self.machine_st.arena.code_index_tbl.lookup(idx.into()); + let index_ptr = self.machine_st.arena.code_index_tbl.get_entry(idx.into()); self.try_call(name, arity, index_ptr) } else { self.undefined_procedure(name, arity) @@ -1076,15 +1084,23 @@ impl Machine { let (name, arity) = key; if module_name == atom!("user") { - if let Some(idx) = self.indices.code_dir.get(&(name, arity)).cloned() { - let index_ptr = *self.machine_st.arena.code_index_tbl.lookup(idx.into()); + if let Some(offset) = self.indices.code_dir.get(&(name, arity)).cloned() { + let index_ptr = self + .machine_st + .arena + .code_index_tbl + .get_entry(offset.into()); self.try_execute(name, arity, index_ptr) } else { self.undefined_procedure(name, arity) } } else if let Some(module) = self.indices.modules.get(&module_name) { - if let Some(idx) = module.code_dir.get(&(name, arity)).cloned() { - let index_ptr = *self.machine_st.arena.code_index_tbl.lookup(idx.into()); + if let Some(offset) = module.code_dir.get(&(name, arity)).cloned() { + let index_ptr = self + .machine_st + .arena + .code_index_tbl + .get_entry(offset.into()); self.try_execute(name, arity, index_ptr) } else { self.undefined_procedure(name, arity) @@ -1131,7 +1147,7 @@ impl Machine { self.machine_st .arena .code_index_tbl - .lookup(code_idx.into()) + .get_entry(code_idx.into()) .local() }) .unwrap(); @@ -1142,7 +1158,7 @@ impl Machine { self.machine_st .arena .code_index_tbl - .lookup(code_idx.into()) + .get_entry(code_idx.into()) .local() }) .unwrap(); diff --git a/src/machine/system_calls.rs b/src/machine/system_calls.rs index 94728dce..24735325 100644 --- a/src/machine/system_calls.rs +++ b/src/machine/system_calls.rs @@ -1246,7 +1246,7 @@ impl Machine { self.machine_st .arena .code_index_tbl - .lookup(idx.into()) + .get_entry(idx.into()) .local() }) .unwrap(); @@ -1477,7 +1477,11 @@ impl Machine { }; if let Some(code_idx) = index_cell_opt { - let index_ptr = *self.machine_st.arena.code_index_tbl.lookup(code_idx.into()); + let index_ptr = self + .machine_st + .arena + .code_index_tbl + .get_entry(code_idx.into()); if !index_ptr.is_undefined() { load_registers(&mut self.machine_st, goal, goal_arity); @@ -6005,7 +6009,7 @@ impl Machine { self.machine_st .arena .code_index_tbl - .lookup(idx.into()) + .get_entry(idx.into()) .local() .is_some() }) @@ -6030,10 +6034,10 @@ impl Machine { arity, module_name, ) - .map(|idx| *self.machine_st + .map(|idx| self.machine_st .arena .code_index_tbl - .lookup(idx.into())) + .get_entry(idx.into())) .unwrap_or(IndexPtr::dynamic_undefined()); !matches!(index.tag(), IndexPtrTag::DynamicUndefined | IndexPtrTag::Undefined) @@ -6050,10 +6054,10 @@ impl Machine { 0, module_name, ) - .map(|idx| *self.machine_st + .map(|idx| self.machine_st .arena .code_index_tbl - .lookup(idx.into())) + .get_entry(idx.into())) .unwrap_or(IndexPtr::dynamic_undefined()); !matches!(index.tag(), IndexPtrTag::DynamicUndefined) @@ -7480,7 +7484,7 @@ impl Machine { self.machine_st .arena .code_index_tbl - .lookup(first_idx.into()) + .get_entry(first_idx.into()) .local() }); diff --git a/src/machine/unify.rs b/src/machine/unify.rs index 252b0f48..6ada6fb1 100644 --- a/src/machine/unify.rs +++ b/src/machine/unify.rs @@ -289,10 +289,10 @@ pub(crate) trait Unifier: DerefMut { (HeapCellValueTag::F64Offset, f2) => { let machine_st = self.deref_mut(); - let f1 = *machine_st.arena.f64_tbl.lookup(f1); - let f2 = *machine_st.arena.f64_tbl.lookup(f2.into()); + let f1 = machine_st.arena.f64_tbl.get_entry(f1); + let f2 = machine_st.arena.f64_tbl.get_entry(f2.into()); - self.fail = *f1 != *f2; + self.fail = f1 != f2; } _ => { self.fail = true; diff --git a/src/offset_table.rs b/src/offset_table.rs index f7d9efdf..c2b2ea8b 100644 --- a/src/offset_table.rs +++ b/src/offset_table.rs @@ -1,13 +1,12 @@ use std::cell::UnsafeCell; -use std::hash::{Hash, Hasher}; -use std::ops::{Deref, DerefMut}; -use std::sync::{Arc, RwLock, RwLockReadGuard}; +use std::sync::Arc; use std::{fmt, mem, ptr}; use arcu::atomic::Arcu; use arcu::epoch_counters::GlobalEpochCounterPool; use arcu::rcu_ref::RcuRef; use arcu::Rcu; +use parking_lot::RwLock; use crate::machine::machine_indices::IndexPtr; use crate::raw_block::RawBlock; @@ -55,7 +54,7 @@ impl From>> for OffsetTableImpl< } } -impl OffsetTableImpl { +impl OffsetTableImpl { #[inline(always)] pub fn new() -> Self { Self(InnerOffsetTableImpl::Serial(SerialOffsetTable::new())) @@ -70,10 +69,17 @@ impl OffsetTableImpl { }; let serial_tbl = mem::replace(serial_tbl, empty_serial_tbl); + let num_tbl_entries = serial_tbl.block.size() / size_of::(); let block = Arcu::new(serial_tbl.block, GlobalEpochCounterPool); - let growth_lock = RwLock::new(()); - let concurrent_tbl = Arc::new(ConcurrentOffsetTable { block, growth_lock }); + let offset_locks: Vec> = + (0..num_tbl_entries).map(|_| RwLock::new(())).collect(); + + let concurrent_tbl = Arc::new(ConcurrentOffsetTable { + block, + growth_lock: RwLock::new(()), + offset_locks: RwLock::new(offset_locks), + }); self.0 = InnerOffsetTableImpl::Concurrent(concurrent_tbl.clone()); @@ -88,23 +94,48 @@ impl OffsetTableImpl { match &mut self.0 { InnerOffsetTableImpl::Serial(_serial_tbl) => Ok(()), InnerOffsetTableImpl::Concurrent(concurrent_tbl) => { - let lock_guard = concurrent_tbl.growth_lock.write().unwrap(); - let raw_block = concurrent_tbl.block.replace(RawBlock::empty_block()); - - match Arc::try_unwrap(raw_block) { - Ok(block) => { - drop(lock_guard); - self.0 = InnerOffsetTableImpl::Serial(SerialOffsetTable { block }); + let table_arc = std::mem::replace( + concurrent_tbl, + Arc::new(ConcurrentOffsetTable { + block: Arcu::new(RawBlock::empty_block(), GlobalEpochCounterPool), + growth_lock: RwLock::new(()), + offset_locks: RwLock::new(vec![]), + }), + ); + + match Arc::try_unwrap(table_arc) { + Ok(table) => { + // this was the only instance of the concurrent table, as such + // at this point no build_with/with_entry{_mut} call can be in-progress/made + + // this shouldn't be able to fail + let raw_block = + Arc::try_unwrap(table.block.replace(RawBlock::empty_block())).unwrap(); + self.0 = + InnerOffsetTableImpl::Serial(SerialOffsetTable { block: raw_block }); Ok(()) } - Err(_) => Err(()), + Err(table_arc) => { + // restore the concurrent_tbl + *concurrent_tbl = table_arc; + Err(()) + } } } } } + + #[inline] + pub fn get_entry(&self, offset: >::Offset) -> T + where + Self: OffsetTable, + T: Copy, + { + self.with_entry(offset, |value| *value) + } } -impl Default for OffsetTableImpl { +impl Default for OffsetTableImpl { fn default() -> Self { Self::new() } @@ -119,6 +150,7 @@ struct SerialOffsetTable { pub struct ConcurrentOffsetTable { block: Arcu, GlobalEpochCounterPool>, growth_lock: RwLock<()>, + offset_locks: RwLock>>, } #[derive(Debug)] @@ -132,42 +164,24 @@ impl InnerOffsetTableImpl { #[inline(always)] fn build_with(&mut self, value: T) -> usize { match self { - Self::Concurrent(concurrent_tbl) => unsafe { concurrent_tbl.build_with(value) }, + Self::Concurrent(concurrent_tbl) => concurrent_tbl.build_with(value), Self::Serial(serial_tbl) => unsafe { serial_tbl.build_with(value) }, } } #[inline(always)] - fn lookup<'a>(&'a self, offset: usize) -> TablePtr<'a, T> { + fn with_entry R>(&self, offset: usize, f: F) -> R { match self { - Self::Concurrent(concurrent_tbl) => TablePtr({ - let (rcu_ref, guard_lock) = concurrent_tbl.lookup(offset); - InnerTablePtr::Concurrent { - rcu_ref, - guard_lock, - } - }), - Self::Serial(serial_tbl) => unsafe { - TablePtr(InnerTablePtr::Serial(serial_tbl.lookup(offset))) - }, + Self::Concurrent(concurrent_tbl) => concurrent_tbl.with_entry(offset, f), + Self::Serial(serial_tbl) => f(unsafe { serial_tbl.lookup(offset) }), } } #[inline(always)] - fn lookup_mut<'a>(&'a mut self, offset: usize) -> TablePtrMut<'a, T> { + fn with_entry_mut R>(&mut self, offset: usize, f: F) -> R { match self { - InnerOffsetTableImpl::Concurrent(concurrent_tbl) => TablePtrMut({ - let (rcu_ref, guard_lock) = concurrent_tbl.lookup_mut(offset); - InnerTablePtrMut::Concurrent { - rcu_ref, - guard_lock, - } - }), - InnerOffsetTableImpl::Serial(serial_tbl) => { - TablePtrMut(InnerTablePtrMut::Serial(unsafe { - serial_tbl.lookup_mut(offset) - })) - } + Self::Concurrent(concurrent_tbl) => concurrent_tbl.with_entry_mut(offset, f), + Self::Serial(serial_tbl) => f(unsafe { serial_tbl.lookup_mut(offset) }), } } } @@ -176,8 +190,9 @@ pub trait OffsetTable { type Offset: Copy + Into; fn build_with(&mut self, value: T) -> Self::Offset; - fn lookup<'a>(&'a self, offset: Self::Offset) -> TablePtr<'a, T>; - fn lookup_mut<'a>(&'a mut self, offset: Self::Offset) -> TablePtrMut<'a, T>; + + fn with_entry R>(&self, offset: Self::Offset, f: F) -> R; + fn with_entry_mut R>(&mut self, offset: Self::Offset, f: F) -> R; } impl OffsetTable> for OffsetTableImpl> { @@ -187,12 +202,18 @@ impl OffsetTable> for OffsetTableImpl> { F64Offset(self.0.build_with(value)) } - fn lookup<'a>(&'a self, offset: F64Offset) -> TablePtr<'a, OrderedFloat> { - self.0.lookup(offset.into()) + #[inline] + fn with_entry) -> R>(&self, offset: F64Offset, f: F) -> R { + self.0.with_entry(offset.into(), f) } - fn lookup_mut<'a>(&'a mut self, offset: F64Offset) -> TablePtrMut<'a, OrderedFloat> { - self.0.lookup_mut(offset.into()) + #[inline] + fn with_entry_mut) -> R>( + &mut self, + offset: F64Offset, + f: F, + ) -> R { + self.0.with_entry_mut(offset.into(), f) } } @@ -203,12 +224,18 @@ impl OffsetTable for OffsetTableImpl { CodeIndexOffset(self.0.build_with(value)) } - fn lookup<'a>(&'a self, offset: CodeIndexOffset) -> TablePtr<'a, IndexPtr> { - self.0.lookup(offset.into()) + #[inline] + fn with_entry R>(&self, offset: CodeIndexOffset, f: F) -> R { + self.0.with_entry(offset.into(), f) } - fn lookup_mut<'a>(&'a mut self, offset: CodeIndexOffset) -> TablePtrMut<'a, IndexPtr> { - self.0.lookup_mut(offset.into()) + #[inline] + fn with_entry_mut R>( + &mut self, + offset: CodeIndexOffset, + f: F, + ) -> R { + self.0.with_entry_mut(offset.into(), f) } } @@ -251,8 +278,8 @@ impl SerialOffsetTable { impl ConcurrentOffsetTable { #[allow(clippy::missing_safety_doc)] - unsafe fn build_with(&self, value: T) -> usize { - let update_guard = self.growth_lock.write().unwrap(); + fn build_with(&self, value: T) -> usize { + let growth_lock = self.growth_lock.write(); // we don't have an index table for lookups as AtomTable does so // just get the epoch after we take the upgrade lock @@ -260,10 +287,10 @@ impl ConcurrentOffsetTable { let mut ptr; loop { - ptr = block_epoch.alloc(mem::size_of::()); + ptr = unsafe { block_epoch.alloc(mem::size_of::()) }; if ptr.is_null() { - let new_block = block_epoch.grow_new().unwrap(); + let new_block = unsafe { block_epoch.grow_new().unwrap() }; self.block.replace(new_block); block_epoch = self.block.read(); } else { @@ -271,35 +298,46 @@ impl ConcurrentOffsetTable { } } - ptr::write(ptr as *mut T, value); + let new_tbl_sz = block_epoch.size() / size_of::(); + let mut offset_locks = self.offset_locks.write(); + + offset_locks.resize_with(new_tbl_sz, || RwLock::new(())); + + unsafe { + ptr::write(ptr as *mut T, value); + } let value = ptr.addr() - block_epoch.base.addr(); // AtomTable would have to update the index table at this point // explicit drop to ensure we don't accidentally drop it early - drop(update_guard); + drop(offset_locks); + drop(growth_lock); value } - #[inline] - fn lookup<'a>(&'a self, offset: usize) -> (RcuRef, T>, RwLockReadGuard<'a, ()>) { - let growth_lock_guard = self.growth_lock.read().unwrap(); + fn with_entry R>(&self, offset: usize, f: F) -> R { + let outer_offset_lock = self.offset_locks.read(); + let inner_offset_lock = outer_offset_lock[offset / size_of::()].read(); let rcu_ref = RcuRef::try_map(self.block.read(), |raw_block| unsafe { raw_block.base.add(offset).cast::().as_ref() }) - .expect("The offset should result in a non-null pointer"); + .expect("offset valid"); - (rcu_ref, growth_lock_guard) + let result = f(&*rcu_ref); + + drop(inner_offset_lock); + drop(outer_offset_lock); + + result } - #[inline] - fn lookup_mut<'a>( - &'a self, - offset: usize, - ) -> (RcuRef, UnsafeCell>, RwLockReadGuard<'a, ()>) { - let growth_lock_guard = self.growth_lock.read().unwrap(); + fn with_entry_mut R>(&self, offset: usize, f: F) -> R { + let growth_lock = self.growth_lock.read(); + let outer_offset_lock = self.offset_locks.read(); + let inner_offset_lock = outer_offset_lock[offset / size_of::()].write(); let rcu_ref = RcuRef::try_map(self.block.read(), |raw_block| unsafe { raw_block @@ -309,9 +347,15 @@ impl ConcurrentOffsetTable { .cast::>() .as_ref() }) - .expect("The offset should result in a non-null pointer"); + .expect("offset valid"); - (rcu_ref, growth_lock_guard) + let result = f(unsafe { &mut *rcu_ref.get().as_mut().unwrap() }); + + drop(inner_offset_lock); + drop(outer_offset_lock); + drop(growth_lock); + + result } } @@ -358,64 +402,6 @@ impl CodeIndexOffset { } } -#[derive(Debug)] -pub struct TablePtr<'a, T: RawBlockTraits>(InnerTablePtr<'a, T>); - -#[derive(Debug)] -enum InnerTablePtr<'a, T: RawBlockTraits> { - Concurrent { - rcu_ref: RcuRef, T>, - #[allow(dead_code)] - guard_lock: RwLockReadGuard<'a, ()>, - }, - Serial(&'a T), -} - -impl PartialEq for TablePtr<'_, T> { - fn eq(&self, other: &TablePtr<'_, T>) -> bool { - self.deref() == other.deref() - } -} - -impl Eq for TablePtr<'_, T> {} - -impl PartialOrd for TablePtr<'_, T> { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for TablePtr<'_, T> { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - (**self).cmp(&**other) - } -} - -impl Hash for TablePtr<'_, T> { - #[inline(always)] - fn hash(&self, hasher: &mut H) { - (self as &T).hash(hasher) - } -} - -impl fmt::Display for TablePtr<'_, T> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self as &T) - } -} - -impl Deref for TablePtr<'_, T> { - type Target = T; - - #[inline] - fn deref(&self) -> &Self::Target { - match &self.0 { - InnerTablePtr::Concurrent { rcu_ref, .. } => rcu_ref, - InnerTablePtr::Serial(ref_mut) => ref_mut, - } - } -} - impl fmt::Display for CodeIndexOffset { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "CodeIndexOffset({})", self.0) @@ -434,97 +420,3 @@ impl fmt::Display for F64Offset { write!(f, "F64Offset({})", self.0) } } - -#[derive(Debug)] -pub struct TablePtrMut<'a, T: RawBlockTraits>(InnerTablePtrMut<'a, T>); - -#[derive(Debug)] -enum InnerTablePtrMut<'a, T: RawBlockTraits> { - Concurrent { - rcu_ref: RcuRef, UnsafeCell>, - #[allow(dead_code)] - guard_lock: RwLockReadGuard<'a, ()>, - }, - Serial(&'a mut T), -} - -impl PartialEq for TablePtrMut<'_, T> { - fn eq(&self, other: &TablePtrMut<'_, T>) -> bool { - self.deref() == other.deref() - } -} - -impl Eq for TablePtrMut<'_, T> {} - -impl PartialOrd for TablePtrMut<'_, T> { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for TablePtrMut<'_, T> { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - (**self).cmp(&**other) - } -} - -impl Hash for TablePtrMut<'_, T> { - #[inline(always)] - fn hash(&self, hasher: &mut H) { - (self as &T).hash(hasher) - } -} - -impl fmt::Display for TablePtrMut<'_, T> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self as &T) - } -} - -impl Deref for TablePtrMut<'_, T> { - type Target = T; - - #[inline] - fn deref(&self) -> &Self::Target { - match &self.0 { - InnerTablePtrMut::Concurrent { rcu_ref, .. } => unsafe { - rcu_ref.get().as_ref().unwrap() - }, - InnerTablePtrMut::Serial(ref_mut) => ref_mut, - } - } -} - -impl DerefMut for TablePtrMut<'_, T> { - #[inline] - fn deref_mut(&mut self) -> &mut Self::Target { - match &mut self.0 { - InnerTablePtrMut::Concurrent { rcu_ref, .. } => unsafe { - &mut *rcu_ref.get().as_mut().unwrap() - }, - InnerTablePtrMut::Serial(ref_mut) => ref_mut, - } - } -} - -impl TablePtrMut<'_, IndexPtr> { - #[inline] - pub fn set(&mut self, val: IndexPtr) { - match &mut self.0 { - InnerTablePtrMut::Concurrent { rcu_ref, .. } => unsafe { - *rcu_ref.get() = val; - }, - InnerTablePtrMut::Serial(ref_mut) => { - **ref_mut = val; - } - } - } - - #[inline] - pub fn replace(&mut self, val: IndexPtr) -> IndexPtr { - match &mut self.0 { - InnerTablePtrMut::Concurrent { rcu_ref, .. } => unsafe { rcu_ref.get().replace(val) }, - InnerTablePtrMut::Serial(ref_mut) => mem::replace(*ref_mut, val), - } - } -} diff --git a/src/parser/parser.rs b/src/parser/parser.rs index b989738d..a131ff75 100644 --- a/src/parser/parser.rs +++ b/src/parser/parser.rs @@ -962,14 +962,14 @@ impl<'a, R: CharRead> Parser<'a, R> { Token::Literal(Literal::Rational(n)) => { self.negate_number(n, negate_rat_rc, |r, _| Literal::Rational(r)) } - Token::Literal(Literal::F64Offset(n)) if self.lexer.machine_st.arena.f64_tbl.lookup(n).is_infinite() => { + Token::Literal(Literal::F64Offset(n)) if self.lexer.machine_st.arena.f64_tbl.get_entry(n).is_infinite() => { return Err(ParserError::InfiniteFloat( self.lexer.line_num, self.lexer.col_num, )); } Token::Literal(Literal::F64Offset(n)) => { - let n = *self.lexer.machine_st.arena.f64_tbl.lookup(n); + let n = self.lexer.machine_st.arena.f64_tbl.get_entry(n); self.negate_number( n,