Add benchmarks using library interface

diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml
index 9a813718550c07c59927dbabab8c1540fed9df1a..3495b6aaca14b501032ddd4b7b35a2048b309beb 100644 (file)
--- a/.github/actions/setup-rust/action.yml
+++ b/.github/actions/setup-rust/action.yml
@@ -48,7 +48,5 @@ runs:
   # Must be placed after actions/cache so its post step runs first.
   - uses: pyTooling/Actions/[email protected]
     with:
-      main: "true"
-      post: |
-        echo Post job cleanup: bash -c "cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(\" \")|.[0]]|join(\" \")'`"
-        bash -c "cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(\" \")|.[0]]|join(\" \")'`"
+      main: bash ./.github/actions/setup-rust/cleanup.sh
+      post: bash ./.github/actions/setup-rust/cleanup.sh

diff --git a/.github/actions/setup-rust/cleanup.sh b/.github/actions/setup-rust/cleanup.sh
new file mode 100755 (executable)
index 0000000..fd51bc9
--- /dev/null
+++ b/.github/actions/setup-rust/cleanup.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -e
+
+echo Cleanup workspace build artifacts and extra target output
+
+# clean just the direct members of the current workspace, with cargo metadata should generalize to all rust projects
+cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(" ")|.[0]]|join(" ")'`
+
+# remove directories in /target/ that are not named `debug` or `release`
+find ./target -maxdepth 1 -type d ! -name debug ! -name release ! -name target -exec rm -r {} \;

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 64c905ffe031a3dd4db9e9afbccf0c20ede385b5..51e97199972a5dde36a2a8d55405cf423f7ef01a 100644 (file)
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -139,6 +139,8 @@ jobs:
           cache-context: report
       - run: |
           cargo install cargo2junit --force
+          cargo install iai-callgrind-runner --force --version `cargo metadata --format-version 1 | jq -r '.resolve.nodes[].id|split(" ")|select(.[0]=="iai-callgrind")|.[1]'`
+          sudo apt install valgrind -y
 
       - name: Test and report
         run: |
@@ -156,6 +158,22 @@ jobs:
           fail_on: nothing
           comment_mode: off
 
+      - run: cargo build --bench run_iai --release
+      - run: cargo bench --bench run_iai
+      - run: cargo bench --bench run_criterion
+
+      - name: Publish criterion benchmark results
+        uses: actions/upload-artifact@v3
+        with:
+          name: bench-criterion-results
+          path: target/criterion/**/*
+
+      - name: Publish iai benchmark results
+        uses: actions/upload-artifact@v3
+        with:
+          name: bench-iai-results
+          path: target/iai/scryer-prolog/run_iai/bench_group/*
+
   # Publish binaries when building for a tag
   release:
     runs-on: ubuntu-20.04

diff --git a/Cargo.lock b/Cargo.lock
index 315f61a7340208022fe66cdd1cea5a2676c48139..8d8cba8f84f7d3888518f36cc5a4243ffd568b0b 100644 (file)
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -41,6 +41,18 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
+[[package]]
+name = "anstyle"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
+
 [[package]]
 name = "arrayvec"
 version = "0.5.2"
@@ -100,6 +112,15 @@ version = "0.21.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2"
 
+[[package]]
+name = "bincode"
+version = "1.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "bit-set"
 version = "0.5.3"
@@ -225,6 +246,12 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
 
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
 [[package]]
 name = "cc"
 version = "1.0.83"
@@ -254,6 +281,58 @@ dependencies = [
  "windows-targets",
 ]
 
+[[package]]
+name = "ciborium"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b"
+dependencies = [
+ "ciborium-io",
+ "half",
+]
+
+[[package]]
+name = "clap"
+version = "4.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b"
+dependencies = [
+ "clap_builder",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663"
+dependencies = [
+ "anstyle",
+ "clap_lex",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
+
 [[package]]
 name = "clipboard-win"
 version = "4.5.0"
@@ -320,6 +399,75 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "criterion"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
+dependencies = [
+ "anes",
+ "cast",
+ "ciborium",
+ "clap",
+ "criterion-plot",
+ "is-terminal",
+ "itertools",
+ "num-traits",
+ "once_cell",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
+dependencies = [
+ "cast",
+ "itertools",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
+dependencies = [
+ "cfg-if",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
+dependencies = [
+ "autocfg",
+ "cfg-if",
+ "crossbeam-utils",
+ "memoffset",
+ "scopeguard",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "crossterm"
 version = "0.20.0"
@@ -834,6 +982,12 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "half"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
+
 [[package]]
 name = "hashbrown"
 version = "0.12.3"
@@ -990,6 +1144,38 @@ dependencies = [
  "tokio-native-tls",
 ]
 
+[[package]]
+name = "iai-callgrind"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2679de583a9f5232b45b1f3a31b5e2f739bccfee9b7902488aca8f8c2c5482d1"
+dependencies = [
+ "bincode",
+ "iai-callgrind-macros",
+ "iai-callgrind-runner",
+]
+
+[[package]]
+name = "iai-callgrind-macros"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af5af66b85e350097b8c0f6329c6347d3323d010443475741c29a1a167f116fb"
+dependencies = [
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
+]
+
+[[package]]
+name = "iai-callgrind-runner"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12cc1093f263249d7c541c53849dac74f4ae756cd31e3cac1a12204efcec68bd"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "iana-time-zone"
 version = "0.1.57"
@@ -1048,6 +1234,17 @@ version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6"
 
+[[package]]
+name = "is-terminal"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
+dependencies = [
+ "hermit-abi",
+ "rustix",
+ "windows-sys",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -1217,6 +1414,15 @@ version = "2.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
 
+[[package]]
+name = "memoffset"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -1425,6 +1631,12 @@ version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
+[[package]]
+name = "oorandom"
+version = "11.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
+
 [[package]]
 name = "opaque-debug"
 version = "0.2.3"
@@ -1658,6 +1870,34 @@ version = "0.3.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
 
+[[package]]
+name = "plotters"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab"
+dependencies = [
+ "plotters-backend",
+]
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@@ -1697,6 +1937,30 @@ dependencies = [
  "termtree",
 ]
 
+[[package]]
+name = "proc-macro-error"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
+dependencies = [
+ "proc-macro-error-attr",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error-attr"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "version_check",
+]
+
 [[package]]
 name = "proc-macro-hack"
 version = "0.5.20+deprecated"
@@ -1767,6 +2031,26 @@ dependencies = [
  "getrandom",
 ]
 
+[[package]]
+name = "rayon"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.16"
@@ -2044,6 +2328,7 @@ dependencies = [
  "console_error_panic_hook",
  "console_log",
  "cpu-time",
+ "criterion",
  "crossterm",
  "crrl",
  "ctrlc",
@@ -2056,6 +2341,7 @@ dependencies = [
  "getrandom",
  "git-version",
  "hostname",
+ "iai-callgrind",
  "indexmap",
  "lazy_static",
  "lexical",
@@ -2498,6 +2784,16 @@ dependencies = [
  "syn 2.0.37",
 ]
 
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "tinyvec"
 version = "1.6.0"

diff --git a/Cargo.toml b/Cargo.toml
index c5e1bab748e974af4e20638d9e8bcf36b6273aee..b36e4e1380d16ab1d4c319c58b313cacab0d809a 100644 (file)
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -85,7 +85,13 @@ tokio = { version = "1.28.2", features = ["full"] }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 getrandom = { version = "0.2.10", features = ["js"] }
-tokio = { version = "1.28.2", features = ["sync", "macros", "io-util", "rt", "time"] }
+tokio = { version = "1.28.2", features = [
+    "sync",
+    "macros",
+    "io-util",
+    "rt",
+    "time",
+] }
 
 [target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies]
 console_error_panic_hook = "0.1"
@@ -93,11 +99,7 @@ console_log = "1.0"
 wasm-bindgen = "0.2.87"
 wasm-bindgen-futures = "0.4"
 serde-wasm-bindgen = "0.5"
-web-sys = { version = "0.3", features = [
-    "Document",
-    "Window",
-    "Element",
-]}
+web-sys = { version = "0.3", features = ["Document", "Window", "Element"] }
 
 [target.'cfg(target_os = "wasi")'.dependencies]
 ring-wasi = { version = "0.16.25" }
@@ -110,6 +112,16 @@ assert_cmd = "1.0.3"
 predicates-core = "1.0.2"
 maplit = "1.0.2"
 serial_test = "2.0.0"
+iai-callgrind = "0.7.3"
+criterion = "0.5.1"
 
 [patch.crates-io]
-modular-bitfield = { git = "https://github.com/mthom/modular-bitfield" }
\ No newline at end of file
+modular-bitfield = { git = "https://github.com/mthom/modular-bitfield" }
+
+[[bench]]
+name = "run_criterion"
+harness = false
+
+[[bench]]
+name = "run_iai"
+harness = false

diff --git a/benches/README.md b/benches/README.md
new file mode 100644 (file)
index 0000000..3716c9c
--- /dev/null
+++ b/benches/README.md
@@ -0,0 +1,95 @@
+# About benches
+
+The `benches` directory contains benchmarks that test scryer-prolog performance.
+
+Benchmarks are run via two harnesses:
+
+* `iai-callgrind` - this runs the benchmark with callgrind, which is able to
+  precisely track the number of instructions executed during the run. This is
+  especially helpful in a public CI runner context where neighboring VMs can
+  cause a very high wall time variance. This means that it doesn't track wall
+  time which is what we really care about, but it is a good tradeoff for CI
+  where tracking runtime is unreliable.
+* `criterion` - criterion performs statistical analysis of benchmark runs and is
+  great for benchmarking locally.
+
+Run them using the following commands:
+
+```
+cargo bench --bench run_criterion
+
+# to run iai, you need valgrind installed and to install iai-callgrind-runner
+# at the same version as is in Cargo.toml:
+cargo install iai-callgrind-runner --version 0.7.3
+
+cargo bench --bench run_iai
+```
+
+For consistency, both runners -- `run_iai.rs` and `run_criterion.rs` -- import
+the same setup code from `benches.rs`.
+
+## Setup
+
+`setup.rs` contains the setup code for the actual benchmarks, which are run
+using the `Benches` struct. `fn benches()` at the top of the file is where the
+benchmarks are defined.
+
+Benchmarks are organized around running queries against one prolog module file.
+Before any runs start, `Benches::new()` reads the module files and initializes a
+new `scryer_prolog::machine::Machine` for each file; multiple queries can be
+declared to be benchmarked in the context of that module/machine instance.
+
+Each benchmark measurement is done by running a query against the machine. In
+the case of criterion each query is run many times, in the case of iai it's run
+once.
+
+## Adding benchmarks
+
+This design is meant to suppoort defining lots of benchmarks.
+
+To add a new benchmark:
+
+* Add a new file `benches/[module].pl` that contains setup code. Import
+  libraries, define predicates, etc.
+* Add a new section in `setup.rs::benches()` that refers to it and add some
+  benchmarks.
+
+Some tips:
+
+* The goal of benchmarking is to know if a library or engine change improved
+  performance or not.
+* Once a benchmark is defined and named, don't change it's definition. If a
+  benchmark needs to change to be more useful, give the new definition a new
+  name. This will prevent charts from showing wild changes in performance just
+  because the definition changed (see previous).
+* Aim for queries to execute in about 0.1-0.5s realtime. Longer runtimes make it
+  easier for humans to see big differences, but benchmarks either run 10x slower
+  (iai) or execute repeatedly to attain statistical significance (criterion) and
+  in both cases queries that take 5+ seconds quickly become unweildly.
+* Consider that the library runtime actually parses the text output of the top
+  level. So keep the output small and don't use custom outputs or it will fail
+  to parse.
+* DO test the output of the benchmark run, we don't want to count broken
+  benchmarks.
+* Because a query may run against the same machine multiple times, don't
+  permanently mutate the state of the engine with the query since that will
+  taint subsequent runs. (Benchmarking assertz et al is desirable, but will
+  require some adjustments to how the machine is set up for runs.)
+
+## CI
+
+Both benchmark harnesses are run in `.github/workflows/ci.yaml` in the `report`
+job, and the results are published as build artifacts.
+
+A future action may consume the build artifacts and publish a report using the
+results.
+
+## Todo
+
+- [ ] Currently, the execution time to load a module is not benchmarked. It
+  would be nice to have at least one benchmark for loading a module (probably a
+  big one).
+- [ ] Adjust the benchmark execution strategy to allow queries to modify the
+  engine state (`assertz` etc).
+- [ ] Write a new action that consumes the test and benchmark results and plots
+  them over time and publishes a report (github pages?).

diff --git a/benches/edges.pl b/benches/edges.pl
new file mode 100644 (file)
index 0000000..b9529de
--- /dev/null
+++ b/benches/edges.pl
@@ -0,0 +1,130 @@
+:- use_module(library(clpb)).
+:- use_module(library(assoc)).
+:- use_module(library(lists)).
+:- use_module(library(pairs)).
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+    Contiguous United States and DC as they appear in SGB:
+    http://www-cs-faculty.stanford.edu/~uno/sgb.html
+- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
+
+edge(al, fl).
+edge(al, ga).
+edge(al, ms).
+edge(al, tn).
+edge(ar, la).
+edge(ar, mo).
+edge(ar, ms).
+edge(ar, ok).
+edge(ar, tn).
+edge(ar, tx).
+edge(az, ca).
+edge(az, nm).
+edge(az, nv).
+edge(az, ut).
+edge(ca, nv).
+edge(ca, or).
+edge(co, ks).
+edge(co, ne).
+edge(co, nm).
+edge(co, ok).
+edge(co, ut).
+edge(co, wy).
+edge(ct, ma).
+edge(ct, ny).
+edge(ct, ri).
+edge(dc, md).
+edge(dc, va).
+edge(de, md).
+edge(de, nj).
+edge(de, pa).
+edge(fl, ga).
+edge(ga, nc).
+edge(ga, sc).
+edge(ga, tn).
+edge(ia, il).
+edge(ia, mn).
+edge(ia, mo).
+edge(ia, ne).
+edge(ia, sd).
+edge(ia, wi).
+edge(id, mt).
+edge(id, nv).
+edge(id, or).
+edge(id, ut).
+edge(id, wa).
+edge(id, wy).
+edge(il, in).
+edge(il, ky).
+edge(il, mo).
+edge(il, wi).
+edge(in, ky).
+edge(in, mi).
+edge(in, oh).
+edge(ks, mo).
+edge(ks, ne).
+edge(ks, ok).
+edge(ky, mo).
+edge(ky, oh).
+edge(ky, tn).
+edge(ky, va).
+edge(ky, wv).
+edge(la, ms).
+edge(la, tx).
+edge(ma, nh).
+edge(ma, ny).
+edge(ma, ri).
+edge(ma, vt).
+edge(md, pa).
+edge(md, va).
+edge(md, wv).
+edge(me, nh).
+edge(mi, oh).
+edge(mi, wi).
+edge(mn, nd).
+edge(mn, sd).
+edge(mn, wi).
+edge(mo, ne).
+edge(mo, ok).
+edge(mo, tn).
+edge(ms, tn).
+edge(mt, nd).
+edge(mt, sd).
+edge(mt, wy).
+edge(nc, sc).
+edge(nc, tn).
+edge(nc, va).
+edge(nd, sd).
+edge(ne, sd).
+edge(ne, wy).
+edge(nh, vt).
+edge(nj, ny).
+edge(nj, pa).
+edge(nm, ok).
+edge(nm, tx).
+edge(nv, or).
+edge(nv, ut).
+edge(ny, pa).
+edge(ny, vt).
+edge(oh, pa).
+edge(oh, wv).
+edge(ok, tx).
+edge(or, wa).
+edge(pa, wv).
+edge(sd, wy).
+edge(tn, va).
+edge(ut, wy).
+edge(va, wv).
+
+independent_set(G, *(NBs)) :-
+        findall(U-V, (edge(U, V),G@<U), Edges),
+        setof(U, V^(member(U-V, Edges);member(V-U, Edges)), Nodes),
+        pairs_keys_values(Pairs, Nodes, _),
+        list_to_assoc(Pairs, Assoc),
+        maplist(not_both(Assoc), Edges, NBs).
+
+not_both(Assoc, U-V, ~BU + ~BV) :-
+        get_assoc(U, Assoc, BU),
+        get_assoc(V, Assoc, BV).
+
+independent_set_count(G, Count) :- independent_set(G, Sat), sat_count(Sat, Count).

diff --git a/benches/numlist.pl b/benches/numlist.pl
new file mode 100644 (file)
index 0000000..7458391
--- /dev/null
+++ b/benches/numlist.pl
@@ -0,0 +1,2 @@
+:- use_module(library(between)).
+run_numlist(Upper, Head) :- numlist(1, Upper, L), L = [Head|_].

diff --git a/benches/run_criterion.rs b/benches/run_criterion.rs
new file mode 100644 (file)
index 0000000..c0cdea3
--- /dev/null
+++ b/benches/run_criterion.rs
@@ -0,0 +1,14 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+
+mod setup;
+
+fn bench_criterion(c: &mut Criterion) {
+    setup::benches().run_all_criterion(c);
+}
+
+criterion_group!(
+    name = bench_group;
+    config = Criterion::default().sample_size(10);
+    targets = bench_criterion
+);
+criterion_main!(bench_group);

diff --git a/benches/run_iai.rs b/benches/run_iai.rs
new file mode 100644 (file)
index 0000000..bcb990d
--- /dev/null
+++ b/benches/run_iai.rs
@@ -0,0 +1,21 @@
+use iai_callgrind::{library_benchmark, library_benchmark_group, main};
+
+mod setup;
+
+#[library_benchmark]
+#[bench::normal(setup::benches())]
+fn bench_edges(mut b: setup::Benches) {
+    b.run_once("count_edges_short");
+}
+
+#[library_benchmark]
+#[bench::normal(setup::benches())]
+fn bench_numlist(mut b: setup::Benches) {
+    b.run_once("numlist_short");
+}
+
+library_benchmark_group!(
+    name = bench_group;
+    benchmarks = bench_edges, bench_numlist
+);
+main!(library_benchmark_groups = bench_group);

diff --git a/benches/setup.rs b/benches/setup.rs
new file mode 100644 (file)
index 0000000..2ca880c
--- /dev/null
+++ b/benches/setup.rs
@@ -0,0 +1,116 @@
+use std::{collections::BTreeMap, fs, path::Path};
+
+use criterion::{black_box, Criterion};
+
+use maplit::btreemap;
+use scryer_prolog::machine::{
+    parsed_results::{QueryMatch, QueryResolution, Value},
+    Machine,
+};
+
+pub fn benches() -> Benches {
+    Benches::new(&[
+        (
+            "benches/edges.pl", // name of the prolog module file to load
+            &[
+                (
+                    "count_edges_short",                 // name of the benchmark
+                    "independent_set_count(ky, Count).", // query to benchmark in the context of the loaded module
+                    btreemap! { "Count".to_string() => Value::try_from("2869176".to_string()).unwrap() }, // List of expected bindings
+                ),
+                (
+                    "count_edges", // multiple benchmark queries can be defined per module
+                    "independent_set_count(aa, Count).", // consider making the query adjustable to tune the runtime
+                    btreemap! { "Count".to_string() => Value::try_from("211954906".to_string()).unwrap(), },
+                ),
+            ],
+        ),
+        (
+            "benches/numlist.pl",
+            &[(
+                "numlist_short",
+                "run_numlist(1000000, Head).",
+                btreemap! { "Head".to_string() => Value::try_from("1".to_string()).unwrap()},
+            )],
+        ),
+    ])
+}
+
+pub struct Benches {
+    machines: Vec<Machine>,
+    runs: BTreeMap<String, Run>,
+}
+
+pub struct Run {
+    machine_idx: usize,
+    name: &'static str,
+    query: &'static str,
+    bindings: BTreeMap<String, Value>,
+}
+
+// Required for using a mutex. It doesn't actually send anything across threads,
+// and this is just a benchmark, so it Should Be Fine(tm). ¯\_(ツ)_/¯
+unsafe impl Send for Benches {}
+
+impl Benches {
+    #[allow(clippy::type_complexity)]
+    pub fn new(
+        benches: &[(
+            &'static str,
+            &[(&'static str, &'static str, BTreeMap<String, Value>)],
+        )],
+    ) -> Self {
+        let mut machines = vec![];
+        let mut runs = BTreeMap::new();
+
+        for b in benches {
+            let content = fs::read_to_string(b.0).unwrap();
+            let name = Path::new(b.0).file_stem().unwrap().to_str().unwrap();
+            let mut machine = Machine::new_lib();
+            machine.load_module_string(name, content);
+            machines.push(machine);
+            let idx = machines.len() - 1;
+            runs.extend(b.1.iter().cloned().map(|r| {
+                (
+                    r.0.to_string(),
+                    Run {
+                        machine_idx: idx,
+                        name: r.0,
+                        query: r.1,
+                        bindings: r.2,
+                    },
+                )
+            }));
+        }
+
+        Benches { machines, runs }
+    }
+
+    #[allow(dead_code)]
+    pub fn run_all_criterion(&mut self, c: &mut Criterion) {
+        for (_, runner) in self.runs.iter() {
+            let machine = &mut self.machines[runner.machine_idx];
+            c.bench_function(runner.name, |b| {
+                b.iter(|| {
+                    Self::run(machine, runner);
+                })
+            });
+        }
+    }
+
+    #[allow(dead_code)]
+    pub fn run_once(&mut self, name: &str) {
+        let runner = &self.runs[name];
+        let machine = &mut self.machines[runner.machine_idx];
+        Self::run(machine, runner);
+    }
+
+    fn run(machine: &mut Machine, runner: &Run) {
+        assert_eq!(
+            black_box(machine.run_query(black_box(runner.query.to_string()))),
+            Ok(QueryResolution::Matches(vec![QueryMatch::from(
+                runner.bindings.clone()
+            )]))
+        );
+    }
+}

diff --git a/src/machine/lib_machine.rs b/src/machine/lib_machine.rs
index 3a27b3d29637dfdc2cd272c37b28c43962166e2e..43aed66afb83e3e14ccdf0615c9f5415277c07ab 100644 (file)
--- a/src/machine/lib_machine.rs
+++ b/src/machine/lib_machine.rs
@@ -60,7 +60,7 @@ impl Machine {
     }
 
     pub fn run_query(&mut self, query: String) -> QueryResult {
-        println!("Query: {}", query);
+        // println!("Query: {}", query);
         // Parse the query so we can analyze and then call the term
         let mut parser = Parser::new(
             Stream::from_owned_string(query, &mut self.machine_st.arena),
@@ -192,7 +192,7 @@ impl Machine {
                 let outputter = printer.print();
 
                 let output: String = outputter.result();
-                println!("Result: {} = {}", var_key.to_string(), output);
+                // println!("Result: {} = {}", var_key.to_string(), output);
 
                 bindings.insert(var_key.to_string(), Value::try_from(output).expect("asdfs"));
             }
@@ -445,10 +445,7 @@ mod tests {
             }
 
             // Check if the block is a query
-            if block.starts_with("query") {
-                // Extract the query from the block
-                let query = &block[5..];
-
+            if let Some(query) = block.strip_prefix("query") {
                 i += 1;
                 println!("query #{}: {}", i, query);
                 // Parse and execute the query
@@ -458,10 +455,7 @@ mod tests {
 
                 // Print the result
                 println!("{:?}", result);
-            } else if block.starts_with("consult") {
-                // Extract the code from the block
-                let code = &block[7..];
-
+            } else if let Some(code) = block.strip_prefix("consult") {
                 println!("load code: {}", code);
 
                 // Load the code into the machine