From: infogulch Date: Sat, 11 Nov 2023 17:22:02 +0000 (-0600) Subject: Add benchmarks using library interface X-Git-Tag: v0.9.4~120^2~1 X-Git-Url: https://git.sagredo.dev/?a=commitdiff_plain;h=8570f119c0283fd414ed35514e9ee86e11e41cce;p=scryer-prolog.git Add benchmarks using library interface --- diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml index 9a813718..3495b6aa 100644 --- a/.github/actions/setup-rust/action.yml +++ b/.github/actions/setup-rust/action.yml @@ -48,7 +48,5 @@ runs: # Must be placed after actions/cache so its post step runs first. - uses: pyTooling/Actions/with-post-step@v0.4.6 with: - main: "true" - post: | - echo Post job cleanup: bash -c "cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(\" \")|.[0]]|join(\" \")'`" - bash -c "cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(\" \")|.[0]]|join(\" \")'`" + main: bash ./.github/actions/setup-rust/cleanup.sh + post: bash ./.github/actions/setup-rust/cleanup.sh diff --git a/.github/actions/setup-rust/cleanup.sh b/.github/actions/setup-rust/cleanup.sh new file mode 100755 index 00000000..fd51bc92 --- /dev/null +++ b/.github/actions/setup-rust/cleanup.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e + +echo Cleanup workspace build artifacts and extra target output + +# clean just the direct members of the current workspace, with cargo metadata should generalize to all rust projects +cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(" ")|.[0]]|join(" ")'` + +# remove directories in /target/ that are not named `debug` or `release` +find ./target -maxdepth 1 -type d ! -name debug ! -name release ! -name target -exec rm -r {} \; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64c905ff..51e97199 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,8 @@ jobs: cache-context: report - run: | cargo install cargo2junit --force + cargo install iai-callgrind-runner --force --version `cargo metadata --format-version 1 | jq -r '.resolve.nodes[].id|split(" ")|select(.[0]=="iai-callgrind")|.[1]'` + sudo apt install valgrind -y - name: Test and report run: | @@ -156,6 +158,22 @@ jobs: fail_on: nothing comment_mode: off + - run: cargo build --bench run_iai --release + - run: cargo bench --bench run_iai + - run: cargo bench --bench run_criterion + + - name: Publish criterion benchmark results + uses: actions/upload-artifact@v3 + with: + name: bench-criterion-results + path: target/criterion/**/* + + - name: Publish iai benchmark results + uses: actions/upload-artifact@v3 + with: + name: bench-iai-results + path: target/iai/scryer-prolog/run_iai/bench_group/* + # Publish binaries when building for a tag release: runs-on: ubuntu-20.04 diff --git a/Cargo.lock b/Cargo.lock index 315f61a7..8d8cba8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,18 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" + [[package]] name = "arrayvec" version = "0.5.2" @@ -100,6 +112,15 @@ version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -225,6 +246,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.83" @@ -254,6 +281,58 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "ciborium" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" + +[[package]] +name = "ciborium-ll" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" + [[package]] name = "clipboard-win" version = "4.5.0" @@ -320,6 +399,75 @@ dependencies = [ "libc", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossterm" version = "0.20.0" @@ -834,6 +982,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "hashbrown" version = "0.12.3" @@ -990,6 +1144,38 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "iai-callgrind" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2679de583a9f5232b45b1f3a31b5e2f739bccfee9b7902488aca8f8c2c5482d1" +dependencies = [ + "bincode", + "iai-callgrind-macros", + "iai-callgrind-runner", +] + +[[package]] +name = "iai-callgrind-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af5af66b85e350097b8c0f6329c6347d3323d010443475741c29a1a167f116fb" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "iai-callgrind-runner" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cc1093f263249d7c541c53849dac74f4ae756cd31e3cac1a12204efcec68bd" +dependencies = [ + "serde", +] + [[package]] name = "iana-time-zone" version = "0.1.57" @@ -1048,6 +1234,17 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi", + "rustix", + "windows-sys", +] + [[package]] name = "itertools" version = "0.10.5" @@ -1217,6 +1414,15 @@ version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -1425,6 +1631,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "opaque-debug" version = "0.2.3" @@ -1658,6 +1870,34 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "plotters" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" + +[[package]] +name = "plotters-svg" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1697,6 +1937,30 @@ dependencies = [ "termtree", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro-hack" version = "0.5.20+deprecated" @@ -1767,6 +2031,26 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2044,6 +2328,7 @@ dependencies = [ "console_error_panic_hook", "console_log", "cpu-time", + "criterion", "crossterm", "crrl", "ctrlc", @@ -2056,6 +2341,7 @@ dependencies = [ "getrandom", "git-version", "hostname", + "iai-callgrind", "indexmap", "lazy_static", "lexical", @@ -2498,6 +2784,16 @@ dependencies = [ "syn 2.0.37", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.6.0" diff --git a/Cargo.toml b/Cargo.toml index c5e1bab7..b36e4e13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,7 +85,13 @@ tokio = { version = "1.28.2", features = ["full"] } [target.'cfg(target_arch = "wasm32")'.dependencies] getrandom = { version = "0.2.10", features = ["js"] } -tokio = { version = "1.28.2", features = ["sync", "macros", "io-util", "rt", "time"] } +tokio = { version = "1.28.2", features = [ + "sync", + "macros", + "io-util", + "rt", + "time", +] } [target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] console_error_panic_hook = "0.1" @@ -93,11 +99,7 @@ console_log = "1.0" wasm-bindgen = "0.2.87" wasm-bindgen-futures = "0.4" serde-wasm-bindgen = "0.5" -web-sys = { version = "0.3", features = [ - "Document", - "Window", - "Element", -]} +web-sys = { version = "0.3", features = ["Document", "Window", "Element"] } [target.'cfg(target_os = "wasi")'.dependencies] ring-wasi = { version = "0.16.25" } @@ -110,6 +112,16 @@ assert_cmd = "1.0.3" predicates-core = "1.0.2" maplit = "1.0.2" serial_test = "2.0.0" +iai-callgrind = "0.7.3" +criterion = "0.5.1" [patch.crates-io] -modular-bitfield = { git = "https://github.com/mthom/modular-bitfield" } \ No newline at end of file +modular-bitfield = { git = "https://github.com/mthom/modular-bitfield" } + +[[bench]] +name = "run_criterion" +harness = false + +[[bench]] +name = "run_iai" +harness = false diff --git a/benches/README.md b/benches/README.md new file mode 100644 index 00000000..3716c9c0 --- /dev/null +++ b/benches/README.md @@ -0,0 +1,95 @@ +# About benches + +The `benches` directory contains benchmarks that test scryer-prolog performance. + +Benchmarks are run via two harnesses: + +* `iai-callgrind` - this runs the benchmark with callgrind, which is able to + precisely track the number of instructions executed during the run. This is + especially helpful in a public CI runner context where neighboring VMs can + cause a very high wall time variance. This means that it doesn't track wall + time which is what we really care about, but it is a good tradeoff for CI + where tracking runtime is unreliable. +* `criterion` - criterion performs statistical analysis of benchmark runs and is + great for benchmarking locally. + +Run them using the following commands: + +``` +cargo bench --bench run_criterion + +# to run iai, you need valgrind installed and to install iai-callgrind-runner +# at the same version as is in Cargo.toml: +cargo install iai-callgrind-runner --version 0.7.3 + +cargo bench --bench run_iai +``` + +For consistency, both runners -- `run_iai.rs` and `run_criterion.rs` -- import +the same setup code from `benches.rs`. + +## Setup + +`setup.rs` contains the setup code for the actual benchmarks, which are run +using the `Benches` struct. `fn benches()` at the top of the file is where the +benchmarks are defined. + +Benchmarks are organized around running queries against one prolog module file. +Before any runs start, `Benches::new()` reads the module files and initializes a +new `scryer_prolog::machine::Machine` for each file; multiple queries can be +declared to be benchmarked in the context of that module/machine instance. + +Each benchmark measurement is done by running a query against the machine. In +the case of criterion each query is run many times, in the case of iai it's run +once. + +## Adding benchmarks + +This design is meant to suppoort defining lots of benchmarks. + +To add a new benchmark: + +* Add a new file `benches/[module].pl` that contains setup code. Import + libraries, define predicates, etc. +* Add a new section in `setup.rs::benches()` that refers to it and add some + benchmarks. + +Some tips: + +* The goal of benchmarking is to know if a library or engine change improved + performance or not. +* Once a benchmark is defined and named, don't change it's definition. If a + benchmark needs to change to be more useful, give the new definition a new + name. This will prevent charts from showing wild changes in performance just + because the definition changed (see previous). +* Aim for queries to execute in about 0.1-0.5s realtime. Longer runtimes make it + easier for humans to see big differences, but benchmarks either run 10x slower + (iai) or execute repeatedly to attain statistical significance (criterion) and + in both cases queries that take 5+ seconds quickly become unweildly. +* Consider that the library runtime actually parses the text output of the top + level. So keep the output small and don't use custom outputs or it will fail + to parse. +* DO test the output of the benchmark run, we don't want to count broken + benchmarks. +* Because a query may run against the same machine multiple times, don't + permanently mutate the state of the engine with the query since that will + taint subsequent runs. (Benchmarking assertz et al is desirable, but will + require some adjustments to how the machine is set up for runs.) + +## CI + +Both benchmark harnesses are run in `.github/workflows/ci.yaml` in the `report` +job, and the results are published as build artifacts. + +A future action may consume the build artifacts and publish a report using the +results. + +## Todo + +- [ ] Currently, the execution time to load a module is not benchmarked. It + would be nice to have at least one benchmark for loading a module (probably a + big one). +- [ ] Adjust the benchmark execution strategy to allow queries to modify the + engine state (`assertz` etc). +- [ ] Write a new action that consumes the test and benchmark results and plots + them over time and publishes a report (github pages?). diff --git a/benches/edges.pl b/benches/edges.pl new file mode 100644 index 00000000..b9529ded --- /dev/null +++ b/benches/edges.pl @@ -0,0 +1,130 @@ +:- use_module(library(clpb)). +:- use_module(library(assoc)). +:- use_module(library(lists)). +:- use_module(library(pairs)). + +/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Contiguous United States and DC as they appear in SGB: + http://www-cs-faculty.stanford.edu/~uno/sgb.html +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ + +edge(al, fl). +edge(al, ga). +edge(al, ms). +edge(al, tn). +edge(ar, la). +edge(ar, mo). +edge(ar, ms). +edge(ar, ok). +edge(ar, tn). +edge(ar, tx). +edge(az, ca). +edge(az, nm). +edge(az, nv). +edge(az, ut). +edge(ca, nv). +edge(ca, or). +edge(co, ks). +edge(co, ne). +edge(co, nm). +edge(co, ok). +edge(co, ut). +edge(co, wy). +edge(ct, ma). +edge(ct, ny). +edge(ct, ri). +edge(dc, md). +edge(dc, va). +edge(de, md). +edge(de, nj). +edge(de, pa). +edge(fl, ga). +edge(ga, nc). +edge(ga, sc). +edge(ga, tn). +edge(ia, il). +edge(ia, mn). +edge(ia, mo). +edge(ia, ne). +edge(ia, sd). +edge(ia, wi). +edge(id, mt). +edge(id, nv). +edge(id, or). +edge(id, ut). +edge(id, wa). +edge(id, wy). +edge(il, in). +edge(il, ky). +edge(il, mo). +edge(il, wi). +edge(in, ky). +edge(in, mi). +edge(in, oh). +edge(ks, mo). +edge(ks, ne). +edge(ks, ok). +edge(ky, mo). +edge(ky, oh). +edge(ky, tn). +edge(ky, va). +edge(ky, wv). +edge(la, ms). +edge(la, tx). +edge(ma, nh). +edge(ma, ny). +edge(ma, ri). +edge(ma, vt). +edge(md, pa). +edge(md, va). +edge(md, wv). +edge(me, nh). +edge(mi, oh). +edge(mi, wi). +edge(mn, nd). +edge(mn, sd). +edge(mn, wi). +edge(mo, ne). +edge(mo, ok). +edge(mo, tn). +edge(ms, tn). +edge(mt, nd). +edge(mt, sd). +edge(mt, wy). +edge(nc, sc). +edge(nc, tn). +edge(nc, va). +edge(nd, sd). +edge(ne, sd). +edge(ne, wy). +edge(nh, vt). +edge(nj, ny). +edge(nj, pa). +edge(nm, ok). +edge(nm, tx). +edge(nv, or). +edge(nv, ut). +edge(ny, pa). +edge(ny, vt). +edge(oh, pa). +edge(oh, wv). +edge(ok, tx). +edge(or, wa). +edge(pa, wv). +edge(sd, wy). +edge(tn, va). +edge(ut, wy). +edge(va, wv). + +independent_set(G, *(NBs)) :- + findall(U-V, (edge(U, V),G@ Benches { + Benches::new(&[ + ( + "benches/edges.pl", // name of the prolog module file to load + &[ + ( + "count_edges_short", // name of the benchmark + "independent_set_count(ky, Count).", // query to benchmark in the context of the loaded module + btreemap! { "Count".to_string() => Value::try_from("2869176".to_string()).unwrap() }, // List of expected bindings + ), + ( + "count_edges", // multiple benchmark queries can be defined per module + "independent_set_count(aa, Count).", // consider making the query adjustable to tune the runtime + btreemap! { "Count".to_string() => Value::try_from("211954906".to_string()).unwrap(), }, + ), + ], + ), + ( + "benches/numlist.pl", + &[( + "numlist_short", + "run_numlist(1000000, Head).", + btreemap! { "Head".to_string() => Value::try_from("1".to_string()).unwrap()}, + )], + ), + ]) +} + +pub struct Benches { + machines: Vec, + runs: BTreeMap, +} + +pub struct Run { + machine_idx: usize, + name: &'static str, + query: &'static str, + bindings: BTreeMap, +} + +// Required for using a mutex. It doesn't actually send anything across threads, +// and this is just a benchmark, so it Should Be Fine(tm). ¯\_(ツ)_/¯ +unsafe impl Send for Benches {} + +impl Benches { + #[allow(clippy::type_complexity)] + pub fn new( + benches: &[( + &'static str, + &[(&'static str, &'static str, BTreeMap)], + )], + ) -> Self { + let mut machines = vec![]; + let mut runs = BTreeMap::new(); + + for b in benches { + let content = fs::read_to_string(b.0).unwrap(); + let name = Path::new(b.0).file_stem().unwrap().to_str().unwrap(); + let mut machine = Machine::new_lib(); + machine.load_module_string(name, content); + machines.push(machine); + let idx = machines.len() - 1; + runs.extend(b.1.iter().cloned().map(|r| { + ( + r.0.to_string(), + Run { + machine_idx: idx, + name: r.0, + query: r.1, + bindings: r.2, + }, + ) + })); + } + + Benches { machines, runs } + } + + #[allow(dead_code)] + pub fn run_all_criterion(&mut self, c: &mut Criterion) { + for (_, runner) in self.runs.iter() { + let machine = &mut self.machines[runner.machine_idx]; + c.bench_function(runner.name, |b| { + b.iter(|| { + Self::run(machine, runner); + }) + }); + } + } + + #[allow(dead_code)] + pub fn run_once(&mut self, name: &str) { + let runner = &self.runs[name]; + let machine = &mut self.machines[runner.machine_idx]; + Self::run(machine, runner); + } + + fn run(machine: &mut Machine, runner: &Run) { + assert_eq!( + black_box(machine.run_query(black_box(runner.query.to_string()))), + Ok(QueryResolution::Matches(vec![QueryMatch::from( + runner.bindings.clone() + )])) + ); + } +} diff --git a/src/machine/lib_machine.rs b/src/machine/lib_machine.rs index 3a27b3d2..43aed66a 100644 --- a/src/machine/lib_machine.rs +++ b/src/machine/lib_machine.rs @@ -60,7 +60,7 @@ impl Machine { } pub fn run_query(&mut self, query: String) -> QueryResult { - println!("Query: {}", query); + // println!("Query: {}", query); // Parse the query so we can analyze and then call the term let mut parser = Parser::new( Stream::from_owned_string(query, &mut self.machine_st.arena), @@ -192,7 +192,7 @@ impl Machine { let outputter = printer.print(); let output: String = outputter.result(); - println!("Result: {} = {}", var_key.to_string(), output); + // println!("Result: {} = {}", var_key.to_string(), output); bindings.insert(var_key.to_string(), Value::try_from(output).expect("asdfs")); } @@ -445,10 +445,7 @@ mod tests { } // Check if the block is a query - if block.starts_with("query") { - // Extract the query from the block - let query = &block[5..]; - + if let Some(query) = block.strip_prefix("query") { i += 1; println!("query #{}: {}", i, query); // Parse and execute the query @@ -458,10 +455,7 @@ mod tests { // Print the result println!("{:?}", result); - } else if block.starts_with("consult") { - // Extract the code from the block - let code = &block[7..]; - + } else if let Some(code) = block.strip_prefix("consult") { println!("load code: {}", code); // Load the code into the machine