From 6c5715b8ea0b878c0a61b8f63b715978590aabeb Mon Sep 17 00:00:00 2001 From: "Shahar \"Dawn\" Or" Date: Mon, 14 Aug 2023 22:55:39 +0700 Subject: [PATCH] doctester: repl examples --- .editorconfig | 10 + .github/workflows/ci.yml | 2 + README.md | 12 +- doctester/.gitignore | 1 + doctester/Cargo.lock | 1189 ++++++++++++++++++ doctester/Cargo.toml | 18 + doctester/rust-toolchain.toml | 2 + doctester/src/app.rs | 102 ++ doctester/src/app/state.rs | 200 +++ doctester/src/app/state/repl_state.rs | 110 ++ doctester/src/example_id.rs | 12 + doctester/src/main.rs | 84 ++ doctester/src/repl.rs | 2 + doctester/src/repl/driver.rs | 204 +++ doctester/src/repl/example.rs | 102 ++ flake.nix | 32 + source/_ext/nix_repl_workaround.py | 13 + source/conf.py | 4 + source/tutorials/first-steps/nix-language.md | 7 +- 19 files changed, 2104 insertions(+), 2 deletions(-) create mode 100644 .editorconfig create mode 100644 doctester/.gitignore create mode 100644 doctester/Cargo.lock create mode 100644 doctester/Cargo.toml create mode 100644 doctester/rust-toolchain.toml create mode 100644 doctester/src/app.rs create mode 100644 doctester/src/app/state.rs create mode 100644 doctester/src/app/state/repl_state.rs create mode 100644 doctester/src/example_id.rs create mode 100644 doctester/src/main.rs create mode 100644 doctester/src/repl.rs create mode 100644 doctester/src/repl/driver.rs create mode 100644 doctester/src/repl/example.rs create mode 100644 source/_ext/nix_repl_workaround.py diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..7a0d70711 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,10 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 + +[*.rs] +indent_style = space +indent_size = 4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f971105a..df22cb8fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,5 +21,7 @@ jobs: run: nix-build - name: Linkcheck run: nix-shell --run "make linkcheck" + - name: Run doctests + run: nix run .#doctests - name: Run code block tests run: nix-shell --run "./run_code_block_tests.sh" diff --git a/README.md b/README.md index 04a533fd0..ae10a7e29 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,22 @@ Official documentation for getting things done with Nix. -## Contributing +## Development daemon Run `./live` and open a browser at . As you make changes your browser should auto-reload within a few seconds. +## Guidelines + For contents and style see [contribution guide](CONTRIBUTING.md). For syntax see [RST/Sphinx Cheatsheet](https://sphinx-tutorial.readthedocs.io/cheatsheet/). + +## Doctests + +Must be run in the repository root. + +- Single run: `nix run .#doctests` +- Daemon: `nix run .#watch-doctests` + diff --git a/doctester/.gitignore b/doctester/.gitignore new file mode 100644 index 000000000..ea8c4bf7f --- /dev/null +++ b/doctester/.gitignore @@ -0,0 +1 @@ +/target diff --git a/doctester/Cargo.lock b/doctester/Cargo.lock new file mode 100644 index 000000000..9ff0b9611 --- /dev/null +++ b/doctester/Cargo.lock @@ -0,0 +1,1189 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" + +[[package]] +name = "anstyle-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" +dependencies = [ + "backtrace", +] + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + +[[package]] +name = "camino" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "4.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c27cdf28c0f604ba3f512b0c9a409f8de8513e4816705deb0498b627e7c3a3fd" +dependencies = [ + "clap_builder", + "clap_derive", + "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08a9f1ab5e9f01a9b81f202e8562eb9a10de70abf9eaeac1be465c28b75aa4aa" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", + "terminal_size", +] + +[[package]] +name = "clap_derive" +version = "4.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "comrak" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "482aa5695bca086022be453c700a40c02893f1ba7098a2c88351de55341ae894" +dependencies = [ + "clap", + "entities", + "memchr", + "once_cell", + "regex", + "shell-words", + "slug", + "syntect", + "typed-arena", + "unicode_categories", + "xdg", +] + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "derive_more" +version = "1.0.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d79dfbcc1f34f3b3a0ce7574276f6f198acb811d70dd19d9dcbfe6263a83d983" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "395aee42a456ecfd4c7034be5011e1a98edcbab2611867c8988a0f40d0bb242a" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "deunicode" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "850878694b7933ca4c9569d30a34b55031b9b139ee1fc7b94a527c4ef960d690" + +[[package]] +name = "doctester" +version = "0.1.0" +dependencies = [ + "anyhow", + "camino", + "clap", + "comrak", + "derive_more", + "futures", + "glob", + "indoc", + "itertools", + "pty-process", + "strip-ansi-escapes", + "tokio", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "entities" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5320ae4c3782150d900b79807611a59a99fc9a1d61d686faafc24b93fc8d7ca" + +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fancy-regex" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6b8560a05112eb52f04b00e5d3790c0dd75d9d980eb8a122fb23b92a623ccf" +dependencies = [ + "bit-set", + "regex", +] + +[[package]] +name = "flate2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "indoc" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c785eefb63ebd0e33416dfcb8d6da0bf27ce752843a45632a67bf10d4d4b5c4" + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi", + "rustix 0.38.8", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "line-wrap" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9" +dependencies = [ + "safemem", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "linux-raw-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "onig" +version = "6.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" +dependencies = [ + "bitflags 1.3.2", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "plist" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdc0001cfea3db57a2e24bc0d818e9e20e554b5f97fabb9bc231dc240269ae06" +dependencies = [ + "base64", + "indexmap", + "line-wrap", + "quick-xml", + "serde", + "time", +] + +[[package]] +name = "proc-macro2" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pty-process" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8749b545e244c90bf74a5767764cc2194f1888bb42f84015486a64c82bea5cc0" +dependencies = [ + "libc", + "rustix 0.38.8", + "tokio", +] + +[[package]] +name = "quick-xml" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b9228215d82c7b61490fec1de287136b5de6f5700f6e58ea9ad61a7964ca51" +dependencies = [ + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax 0.7.4", +] + +[[package]] +name = "regex-automata" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.7.4", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustix" +version = "0.37.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys 0.3.8", + "windows-sys", +] + +[[package]] +name = "rustix" +version = "0.38.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +dependencies = [ + "bitflags 2.3.3", + "errno", + "itoa", + "libc", + "linux-raw-sys 0.4.3", + "windows-sys", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "safemem" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.175" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b" + +[[package]] +name = "serde_derive" +version = "1.0.175" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "shell-words" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + +[[package]] +name = "slug" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3bc762e6a4b6c6fcaade73e77f9ebc6991b676f88bb2358bddb56560f073373" +dependencies = [ + "deunicode", +] + +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "strip-ansi-escapes" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "011cbb39cf7c1f62871aea3cc46e5817b0937b49e9447370c93cacbe93a766d8" +dependencies = [ + "vte", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "2.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syntect" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6c454c27d9d7d9a84c7803aaa3c50cd088d2906fe3c6e42da3209aa623576a8" +dependencies = [ + "bincode", + "bitflags 1.3.2", + "fancy-regex", + "flate2", + "fnv", + "lazy_static", + "once_cell", + "onig", + "plist", + "regex-syntax 0.6.29", + "serde", + "serde_derive", + "serde_json", + "thiserror", + "walkdir", + "yaml-rust", +] + +[[package]] +name = "terminal_size" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e6bf6f19e9f8ed8d4048dc22981458ebcf406d67e94cd422e5ecd73d63b3237" +dependencies = [ + "rustix 0.37.23", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + +[[package]] +name = "time-macros" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96ba15a897f3c86766b757e5ac7221554c6750054d74d5b28844fce5fb36a6c4" +dependencies = [ + "time-core", +] + +[[package]] +name = "tokio" +version = "1.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" +dependencies = [ + "autocfg", + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + +[[package]] +name = "unicode-ident" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "vte" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cbce692ab4ca2f1f3047fcf732430249c0e971bfdd2b234cf2c47ad93af5983" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d257817081c7dffcdbab24b9e62d2def62e2ff7d00b1c20062551e6cccc145ff" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "walkdir" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "xdg" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213b7324336b53d2414b2db8537e56544d981803139155afa84f76eeebb7a546" + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] diff --git a/doctester/Cargo.toml b/doctester/Cargo.toml new file mode 100644 index 000000000..d16da8196 --- /dev/null +++ b/doctester/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "doctester" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = { version = "1.0.72", features = ["backtrace"] } +camino = "1.1.6" +clap = "4.3.21" +comrak = "0.18.0" +derive_more = { version = "1.0.0-beta.2", features = ["deref", "display", "into_iterator", "constructor"] } +futures = "0.3.28" +glob = "0.3.1" +indoc = "2.0.3" +itertools = "0.11.0" +pty-process = { version = "0.4.0", features = ["async"] } +strip-ansi-escapes = "0.1.1" +tokio = { version = "1.29.1", features = ["macros", "rt-multi-thread", "io-util"] } diff --git a/doctester/rust-toolchain.toml b/doctester/rust-toolchain.toml new file mode 100644 index 000000000..292fe499e --- /dev/null +++ b/doctester/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "stable" diff --git a/doctester/src/app.rs b/doctester/src/app.rs new file mode 100644 index 000000000..228c89677 --- /dev/null +++ b/doctester/src/app.rs @@ -0,0 +1,102 @@ +pub(super) mod state; + +use futures::{FutureExt, SinkExt, StreamExt}; + +use crate::repl::{ + driver::{ReplCommand, ReplEvent}, + example::ReplExample, +}; + +use self::state::State; + +pub(crate) struct Inputs { + pub(crate) repl_examples: Vec, + pub(crate) repl_events: futures::stream::LocalBoxStream<'static, ReplEvent>, +} + +pub(crate) struct Outputs { + pub(crate) execution_handle: futures::future::LocalBoxFuture<'static, ()>, + pub(crate) repl_commands: futures::stream::LocalBoxStream<'static, ReplCommand>, + pub(crate) done: futures::future::LocalBoxFuture<'static, anyhow::Result<()>>, + pub(crate) eprintln_strings: futures::stream::LocalBoxStream<'static, String>, +} + +#[derive(Debug)] +enum OutputEvent { + Done(anyhow::Result<()>), + ReplCommand(ReplCommand), + Eprintln(String), +} + +#[derive(Debug)] +enum InputEvent { + ReplExample(ReplExample), + ReplEvent(ReplEvent), +} + +pub(crate) fn app(inputs: Inputs) -> Outputs { + let Inputs { + repl_examples, + repl_events, + } = inputs; + + let repl_examples = futures::stream::iter(repl_examples).map(InputEvent::ReplExample); + let repl_events = repl_events.map(InputEvent::ReplEvent); + + let input_events = + futures::stream::select_all([repl_examples.boxed_local(), repl_events.boxed_local()]); + + let output_events = input_events + .scan(State::default(), |state, event| { + let output = match event { + InputEvent::ReplExample(repl_example) => state.repl_example(repl_example), + InputEvent::ReplEvent(repl_event) => state.repl_event(repl_event), + }; + + let output = match output { + Ok(output) => output, + Err(error) => vec![OutputEvent::Done(Err(error))], + }; + + futures::future::ready(Some(output)) + }) + .flat_map(futures::stream::iter); + + let (eprintln_sender, eprintln_strings) = futures::channel::mpsc::unbounded::(); + let (repl_commands_sender, repl_commands) = futures::channel::mpsc::unbounded::(); + let (done_sender, done) = futures::channel::mpsc::unbounded::>(); + + let execution_handle = output_events.for_each(move |output_event| match output_event { + OutputEvent::Done(done) => { + let mut sender = done_sender.clone(); + async move { + sender.send(done).await.unwrap(); + } + .boxed_local() + } + OutputEvent::ReplCommand(repl_command) => { + let mut sender = repl_commands_sender.clone(); + async move { + sender.send(repl_command).await.unwrap(); + } + .boxed_local() + } + OutputEvent::Eprintln(string) => { + let mut sender = eprintln_sender.clone(); + async move { + sender.send(string).await.unwrap(); + } + .boxed_local() + } + }); + + Outputs { + eprintln_strings: eprintln_strings.boxed_local(), + repl_commands: repl_commands.boxed_local(), + done: done + .into_future() + .map(|(next_item, _tail)| next_item.unwrap()) + .boxed_local(), + execution_handle: execution_handle.boxed_local(), + } +} diff --git a/doctester/src/app/state.rs b/doctester/src/app/state.rs new file mode 100644 index 000000000..b289bbec0 --- /dev/null +++ b/doctester/src/app/state.rs @@ -0,0 +1,200 @@ +pub(crate) mod repl_state; + +use crate::{ + example_id::ExampleId, + repl::{ + driver::{ReplCommand, ReplEvent, ReplQuery}, + example::ReplExample, + }, +}; + +use self::repl_state::{ + ReplSession, ReplSessionExpecting, ReplSessionLive, ReplSessionState, ReplState, +}; + +use super::OutputEvent; + +#[derive(Default, Debug)] +pub(super) struct State { + repl: ReplState, +} + +impl State { + pub(super) fn repl_example( + &mut self, + repl_example: ReplExample, + ) -> anyhow::Result> { + let id = repl_example.id.clone(); + + self.repl + .insert(id.clone(), ReplSession::new(repl_example))?; + + Ok(vec![OutputEvent::ReplCommand(ReplCommand::Spawn(id))]) + } + + pub(super) fn repl_event(&mut self, repl_event: ReplEvent) -> anyhow::Result> { + match repl_event { + ReplEvent::Spawn(spawn) => self.repl_event_spawn(spawn), + ReplEvent::Query(id, query, result) => self.repl_event_query(id, query, result), + ReplEvent::Kill(id) => self.repl_event_kill(id), + ReplEvent::Read(id, result) => self.repl_event_read(id, result), + } + } + + fn repl_event_spawn( + &mut self, + spawn: Result, + ) -> anyhow::Result> { + let id = spawn?; + + let session = self.repl.get_mut(&id)?; + + if let ReplSessionState::Live(_) = &session.state { + return Err(anyhow::anyhow!("spawned session {session:?} already live")); + } + + let session_live = ReplSessionLive::new(session.example.entries.clone()); + session.state = ReplSessionState::Live(session_live); + Ok(vec![]) + } + + fn repl_event_query( + &self, + _id: ExampleId, + _query: ReplQuery, + result: anyhow::Result<()>, + ) -> anyhow::Result> { + result?; + // TODO possibly store this fact + Ok(vec![]) + } + + fn repl_event_kill( + &mut self, + result: anyhow::Result, + ) -> anyhow::Result> { + let id = result?; + self.repl.remove(&id)?; + + let events = if self.repl.is_empty() { + vec![OutputEvent::Done(Ok(()))] + } else { + vec![] + }; + + Ok(events) + } + + fn repl_event_read( + &mut self, + id: ExampleId, + result: std::io::Result, + ) -> anyhow::Result> { + let session_live = self.repl.get_mut(&id)?; + let session_live = session_live.state.live_mut()?; + let ch = result?; + + let output = match &mut session_live.expecting { + ReplSessionExpecting::Nothing => anyhow::bail!("not expecting, got {:?}", ch as char), + ReplSessionExpecting::Prompt(acc) => { + acc.push(ch.try_into()?); + let string = String::from_utf8(strip_ansi_escapes::strip(acc)?)?; + + if string == "nix-repl> " { + session_live.expecting = ReplSessionExpecting::Nothing; + self.next_query(&id)? + } else { + vec![] + } + } + ReplSessionExpecting::Echo { + acc, + last_query: expected, + expected_result, + } => { + acc.push(ch.try_into()?); + if !acc.ends_with('\n') { + vec![] + } else if Self::sanitize(acc)? == expected.as_str() { + session_live.expecting = ReplSessionExpecting::Result { + acc: String::new(), + expected_result: expected_result.clone(), + }; + vec![] + } else { + anyhow::bail!("actual: {acc:?}, expected: {expected:?}"); + } + } + ReplSessionExpecting::Result { + acc, + expected_result, + } => 'arm: { + acc.push(ch.try_into()?); + + let Some(stripped_crlf_once) = acc.strip_suffix("\r\n") else { + break 'arm vec![]; + }; + + if !stripped_crlf_once.ends_with("\r\n") { + break 'arm vec![]; + } + + let sanitized = Self::sanitize(stripped_crlf_once)?; + + if sanitized != expected_result.as_str() { + anyhow::bail!(indoc::formatdoc! {" + {id} + actual (sanitized): {sanitized:?} + expected : {expected_result}" + }) + } + + session_live.expecting = ReplSessionExpecting::Prompt(String::new()); + vec![] + } + }; + + Ok(output) + } + + fn next_query(&mut self, id: &ExampleId) -> anyhow::Result> { + let session = self.repl.get_mut(id)?; + + let ReplSessionState::Live(session_live) = &mut session.state else { + anyhow::bail!("expected session {id} to be live"); + }; + + let Some(entry) = session_live.next() else { + return self.session_end(id); + }; + + session_live.expecting = ReplSessionExpecting::Echo { + acc: String::new(), + last_query: entry.query.clone(), + expected_result: entry.expected_result, + }; + + Ok(vec![OutputEvent::ReplCommand(ReplCommand::Query( + id.clone(), + entry.query.clone(), + ))]) + } + + fn session_end(&mut self, id: &ExampleId) -> anyhow::Result> { + let session = self.repl.get_mut(id)?; + session.state = ReplSessionState::Killing; + Ok(vec![ + OutputEvent::ReplCommand(ReplCommand::Kill(id.clone())), + OutputEvent::Eprintln(format!("PASS: {id}")), + ]) + } + + fn sanitize(s: &str) -> anyhow::Result { + let ansi_stripped = strip_ansi_escapes::strip(s)?; + let string = String::from_utf8(ansi_stripped)? + .chars() + .filter(|ch| ch != &'\r') + .collect(); + Ok(string) + } +} diff --git a/doctester/src/app/state/repl_state.rs b/doctester/src/app/state/repl_state.rs new file mode 100644 index 000000000..ee5215b9d --- /dev/null +++ b/doctester/src/app/state/repl_state.rs @@ -0,0 +1,110 @@ +use crate::{ + example_id::ExampleId, + repl::{ + driver::ReplQuery, + example::{ReplEntry, ReplExample, ReplExampleEntries}, + }, +}; + +#[derive(Debug, Default)] +pub(crate) struct ReplState(std::collections::BTreeMap); + +impl ReplState { + pub(crate) fn insert(&mut self, id: ExampleId, session: ReplSession) -> anyhow::Result<()> { + if self.0.insert(id.clone(), session).is_some() { + anyhow::bail!("duplicate session id {id:?}"); + }; + Ok(()) + } + + pub(crate) fn get_mut(&mut self, id: &ExampleId) -> anyhow::Result<&mut ReplSession> { + self.0 + .get_mut(id) + .ok_or_else(|| anyhow::anyhow!("repl session not found {id:?}")) + } + + pub(crate) fn remove(&mut self, id: &ExampleId) -> anyhow::Result { + self.0 + .remove(id) + .ok_or_else(|| anyhow::anyhow!("repl session not found {id:?}")) + } + + pub(crate) fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +#[derive(Debug)] +pub(crate) struct ReplSession { + pub(crate) example: ReplExample, + pub(crate) state: ReplSessionState, +} + +impl ReplSession { + pub(crate) fn new(repl_example: ReplExample) -> Self { + Self { + example: repl_example, + state: Default::default(), + } + } +} + +#[derive(Debug, Default)] +pub(crate) enum ReplSessionState { + #[default] + Pending, + Live(ReplSessionLive), + Killing, +} + +impl ReplSessionState { + pub(crate) fn live_mut(&mut self) -> anyhow::Result<&mut ReplSessionLive> { + let Self::Live(live) = self else { + anyhow::bail!("session not live"); + }; + + Ok(live) + } +} + +#[derive(Debug)] +pub(crate) struct ReplSessionLive { + pub(crate) iterator: std::vec::IntoIter, + pub(crate) expecting: ReplSessionExpecting, +} + +#[derive(Debug)] +pub(crate) enum ReplSessionExpecting { + Nothing, + Prompt(String), + Echo { + acc: String, + last_query: ReplQuery, + expected_result: ExpectedResult, + }, + Result { + acc: String, + expected_result: ExpectedResult, + }, +} + +impl ReplSessionLive { + pub(crate) fn new(entries: ReplExampleEntries) -> Self { + Self { + iterator: entries.into_iter(), + expecting: ReplSessionExpecting::Prompt(String::new()), + } + } +} + +impl Iterator for ReplSessionLive { + type Item = ReplEntry; + + fn next(&mut self) -> Option { + let entry = self.iterator.next()?; + Some(entry) + } +} + +#[derive(Debug, Clone, derive_more::Deref, derive_more::Display)] +pub(crate) struct ExpectedResult(pub(crate) String); diff --git a/doctester/src/example_id.rs b/doctester/src/example_id.rs new file mode 100644 index 000000000..04e9d195d --- /dev/null +++ b/doctester/src/example_id.rs @@ -0,0 +1,12 @@ +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, derive_more::Display, derive_more::Constructor)] +#[display("{}:{}", source_path, line)] +pub(crate) struct ExampleId { + source_path: camino::Utf8PathBuf, + line: usize, +} + +impl std::fmt::Debug for ExampleId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "ExampleId({self})") + } +} diff --git a/doctester/src/main.rs b/doctester/src/main.rs new file mode 100644 index 000000000..9e7b394e9 --- /dev/null +++ b/doctester/src/main.rs @@ -0,0 +1,84 @@ +pub(crate) mod app; +pub(crate) mod example_id; +pub(crate) mod repl; + +use clap::Parser; +use futures::{FutureExt, StreamExt}; +use itertools::Itertools; + +use crate::{ + app::{Inputs, Outputs}, + repl::driver::ReplDriver, +}; + +#[derive(Debug, clap::Parser)] +#[command()] +struct Cli { + /// Path to a `nix` executable + nix_path: camino::Utf8PathBuf, + /// `glob` crate pattern of filespaths to extract doctests from + sources: String, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let cli = Cli::parse(); + let repl_examples = repl::example::obtain(&cli.sources)?; + if repl_examples.is_empty() { + anyhow::bail!("could not find any REPL examples"); + } + let (repl_driver, repl_events) = ReplDriver::new(cli.nix_path); + + let inputs = Inputs { + repl_examples, + repl_events: repl_events.boxed_local(), + }; + + let outputs = app::app(inputs); + + let Outputs { + repl_commands, + done, + execution_handle, + eprintln_strings, + } = outputs; + + let eprintln_task = eprintln_strings.for_each(|string| async move { + eprintln!("{string}"); + }); + let repl_task = repl_driver.init(repl_commands); + + tokio::select! { + _ = execution_handle.fuse() => unreachable!(), + _ = eprintln_task.fuse() => unreachable!(), + _ = repl_task.fuse() => unreachable!(), + done = done.fuse() => done, + } +} + +#[derive(Debug, Clone, derive_more::Display)] +#[display("{}", _0)] +struct Eprintln(String); + +#[derive(Debug, derive_more::Deref)] +struct PtyLine(String); + +impl std::str::FromStr for PtyLine { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + s.chars() + .with_position() + .try_for_each(|(position, character)| { + use itertools::Position::*; + match (position, character) { + (Last | Only, '\n') => Ok(()), + (Last | Only, _) => Err(anyhow::anyhow!("does not end with LF {s:?}")), + (_, '\n') => Err(anyhow::anyhow!("LF before end {s:?}")), + _ => Ok(()), + } + })?; + + Ok(Self(s.to_string())) + } +} diff --git a/doctester/src/repl.rs b/doctester/src/repl.rs new file mode 100644 index 000000000..7431016ca --- /dev/null +++ b/doctester/src/repl.rs @@ -0,0 +1,2 @@ +pub(crate) mod driver; +pub(crate) mod example; diff --git a/doctester/src/repl/driver.rs b/doctester/src/repl/driver.rs new file mode 100644 index 000000000..e27f03061 --- /dev/null +++ b/doctester/src/repl/driver.rs @@ -0,0 +1,204 @@ +use futures::{FutureExt, SinkExt, StreamExt}; +use itertools::Itertools; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +use crate::example_id::ExampleId; + +#[derive(Debug, Clone, derive_more::Deref)] +pub(crate) struct LFLine(String); + +impl std::str::FromStr for LFLine { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + s.chars() + .with_position() + .try_for_each(|(position, character)| { + use itertools::Position::*; + match (position, character) { + (Last | Only, '\n') => Ok(()), + (Last | Only, _) => Err(anyhow::anyhow!("does not end with LF {s:?}")), + (_, '\r') => Err(anyhow::anyhow!("found CR {s:?}")), + (_, '\n') => Err(anyhow::anyhow!("newline before end {s:?}")), + (_, _) => Ok(()), + } + })?; + Ok(Self(s.to_string())) + } +} + +#[derive(Debug, Clone, derive_more::Deref)] +pub(crate) struct ReplQuery(LFLine); + +impl TryFrom for ReplQuery { + type Error = anyhow::Error; + + fn try_from(line: LFLine) -> Result { + let Option::Some(("", query)) = line.split_once("nix-repl> ") else { + return Err(anyhow::anyhow!("missing prompt {line:?}")); +}; + Ok(Self(query.parse().unwrap())) + } +} + +#[derive(Debug)] +pub(crate) enum ReplCommand { + Spawn(ExampleId), + Query(ExampleId, ReplQuery), + Kill(ExampleId), +} + +#[derive(Debug)] +pub(crate) enum ReplEvent { + Spawn(pty_process::Result), + Query(ExampleId, ReplQuery, anyhow::Result<()>), + Kill(anyhow::Result), + Read(ExampleId, std::io::Result), +} + +pub(crate) struct ReplDriver { + sessions: std::collections::BTreeMap, + sender: futures::channel::mpsc::UnboundedSender, + nix_path: camino::Utf8PathBuf, +} + +impl ReplDriver { + pub(crate) fn new( + nix_path: camino::Utf8PathBuf, + ) -> (Self, futures::stream::LocalBoxStream<'static, ReplEvent>) { + let (sender, receiver) = futures::channel::mpsc::unbounded::(); + let driver = Self { + sessions: Default::default(), + sender, + nix_path, + }; + (driver, receiver.boxed_local()) + } + + pub(crate) fn init( + mut self, + mut commands: futures::stream::LocalBoxStream<'static, ReplCommand>, + ) -> futures::future::LocalBoxFuture<'static, ()> { + async move { + loop { + let command = futures::poll!(&mut commands.next()); + if let std::task::Poll::Ready(Some(command)) = command { + self.command(command).await; + } + + for (id, (pty, _child)) in self.sessions.iter_mut() { + let byte = futures::poll!(std::pin::pin!(pty.read_u8())); + let std::task::Poll::Ready(byte) = byte else { + continue; + }; + + match byte { + Ok(byte) => { + self.sender + .send(ReplEvent::Read(id.clone(), Ok(byte))) + .await + .unwrap(); + } + Err(error) => { + self.sender + .send(ReplEvent::Read(id.clone(), Err(error))) + .await + .unwrap(); + } + } + } + + tokio::task::yield_now().await; + } + } + .boxed_local() + } + + async fn command(&mut self, repl_command: ReplCommand) { + match repl_command { + ReplCommand::Spawn(id) => self.spawn(id).await, + ReplCommand::Query(id, query) => self.query(id, query).await, + ReplCommand::Kill(id) => self.kill(id).await, + } + } + + async fn spawn(&mut self, id: ExampleId) { + let pty = match pty_process::Pty::new() { + Ok(pty) => pty, + Err(error) => { + self.sender + .send(ReplEvent::Spawn(Err(error))) + .await + .unwrap(); + return; + } + }; + + let pts = match pty.pts() { + Ok(pts) => pts, + Err(error) => { + self.sender + .send(ReplEvent::Spawn(Err(error))) + .await + .unwrap(); + return; + } + }; + + let child = pty_process::Command::new(&self.nix_path) + .args(["repl", "--quiet"]) + .spawn(&pts); + + let child = match child { + Err(error) => { + self.sender + .send(ReplEvent::Spawn(Err(error))) + .await + .unwrap(); + return; + } + Ok(child) => child, + }; + + self.sessions.insert(id.clone(), (pty, child)); + self.sender.send(ReplEvent::Spawn(Ok(id))).await.unwrap(); + } + + async fn query(&mut self, id: ExampleId, query: ReplQuery) { + let (pty, _child) = match self.sessions.get_mut(&id) { + Some(pty) => pty, + None => { + let error = anyhow::anyhow!("no pty for {id:?}"); + self.sender + .send(ReplEvent::Query(id, query, Err(error))) + .await + .unwrap(); + return; + } + }; + + let write = pty.write_all(query.as_bytes()).await; + if let Err(error) = write { + let error = anyhow::anyhow!("failed to query {error}"); + self.sender + .send(ReplEvent::Query(id, query, Err(error))) + .await + .unwrap(); + return; + } + + self.sender + .send(ReplEvent::Query(id, query, Ok(()))) + .await + .unwrap(); + } + + async fn kill(&mut self, id: ExampleId) { + let Some(session) = self.sessions.remove(&id) else { + self.sender.send(ReplEvent::Kill(Err(anyhow::anyhow!("no session {id:?} to kill")))).await.unwrap(); + return; +}; + drop(session); + self.sender.send(ReplEvent::Kill(Ok(id))).await.unwrap(); + } +} diff --git a/doctester/src/repl/example.rs b/doctester/src/repl/example.rs new file mode 100644 index 000000000..31949fec5 --- /dev/null +++ b/doctester/src/repl/example.rs @@ -0,0 +1,102 @@ +use itertools::Itertools; + +use crate::{app::state::repl_state::ExpectedResult, example_id::ExampleId}; + +use super::driver::{LFLine, ReplQuery}; + +#[derive(Debug, Clone)] +pub(crate) struct ReplExample { + pub(crate) id: ExampleId, + pub(crate) entries: ReplExampleEntries, +} + +impl ReplExample { + pub(crate) fn try_new( + source_path: camino::Utf8PathBuf, + line: usize, + contents: String, + ) -> anyhow::Result { + let id = ExampleId::new(source_path, line); + + Ok(Self { + id, + entries: contents.parse()?, + }) + } +} + +#[derive(Debug, Clone, derive_more::IntoIterator)] +pub(crate) struct ReplExampleEntries(Vec); + +impl std::str::FromStr for ReplExampleEntries { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let entries = s + .split_inclusive('\n') + .map(|line| LFLine::from_str(line).unwrap()) + .filter(|line| **line != "\n") + .tuples::<(_, _)>() + .map(ReplEntry::try_from) + .collect::>>()?; + + Ok(Self(entries)) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct ReplEntry { + pub(crate) query: ReplQuery, + pub(crate) expected_result: ExpectedResult, +} + +impl TryFrom<(LFLine, LFLine)> for ReplEntry { + type Error = anyhow::Error; + + fn try_from((query, response): (LFLine, LFLine)) -> Result { + Ok(Self { + query: query.try_into()?, + expected_result: ExpectedResult(response.as_str().to_owned()), + }) + } +} + +const NIX_REPL_LANG_TAG: &str = "nix-repl"; + +pub(crate) fn obtain(glob: &str) -> anyhow::Result> { + glob::glob(glob)? + .map(|path| { + let path = camino::Utf8PathBuf::try_from(path?)?; + let contents = std::fs::read_to_string(path.clone())?; + anyhow::Ok((path, contents)) + }) + .collect::, _>>()? + .into_iter() + .flat_map(|(path, contents)| { + let arena = comrak::Arena::new(); + let ast = comrak::parse_document(&arena, &contents, &comrak::ComrakOptions::default()); + ast.traverse() + .filter_map(move |node_edge| match node_edge { + comrak::arena_tree::NodeEdge::Start(node) => { + let ast = node.data.borrow().clone(); + Some((path.clone(), ast)) + } + comrak::arena_tree::NodeEdge::End(_) => None, + }) + .collect::>() + }) + .filter_map(|(path, ast)| { + if let comrak::nodes::NodeValue::CodeBlock(code_block) = ast.value { + let comrak::nodes::NodeCodeBlock { info, literal, .. } = code_block; + if let Some(NIX_REPL_LANG_TAG) = info.split_ascii_whitespace().next() { + Some((path, ast.sourcepos.start.line, literal.clone())) + } else { + None + } + } else { + None + } + }) + .map(|(path, line, contents)| ReplExample::try_new(path, line, contents)) + .try_collect() +} diff --git a/flake.nix b/flake.nix index 523152fd8..69efc131c 100644 --- a/flake.nix +++ b/flake.nix @@ -49,8 +49,40 @@ } ); }; + doctesterPkg = pkgs.rustPlatform.buildRustPackage { + pname = "doctester"; + version = "0.1.0"; + src = ./doctester; + cargoLock = { + lockFile = ./doctester/Cargo.lock; + }; + doCheck = false; + buildType = "debug"; + }; + contentPath = "source"; + doctester = pkgs.writeShellScriptBin "doctester" '' + ${doctesterPkg}/bin/doctester ${pkgs.nix}/bin/nix '${contentPath}/**/*.md' + ''; + doctests-watcher = pkgs.writeShellScriptBin "doctests-watcher" '' + ${pkgs.watchexec}/bin/watchexec \ + --watch=${contentPath}\ + --restart\ + --shell=none\ + --emit-events-to=none\ + --no-meta\ + --print-events\ + '${doctester}/bin/doctester' + ''; in rec { + apps.doctests = { + type = "app"; + program = "${doctester}/bin/doctester"; + }; + apps.watch-doctests = { + type = "app"; + program = "${doctests-watcher}/bin/doctests-watcher"; + }; packages = flake-utils.lib.flattenTree { nix-dev-pyenv = pkgs.poetry2nix.mkPoetryEnv { projectDir = self; diff --git a/source/_ext/nix_repl_workaround.py b/source/_ext/nix_repl_workaround.py new file mode 100644 index 000000000..5c7bb1f51 --- /dev/null +++ b/source/_ext/nix_repl_workaround.py @@ -0,0 +1,13 @@ +from pygments.lexer import RegexLexer +from pygments.token import Text + +class NixReplLexer(RegexLexer): + name = 'Nix REPL' + aliases = ['nix-repl'] + filenames = [] + + tokens = { + 'root': [ + (r'.+', Text), + ], + } diff --git a/source/conf.py b/source/conf.py index b6507c03c..4262906a6 100644 --- a/source/conf.py +++ b/source/conf.py @@ -127,6 +127,10 @@ # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" +from nix_repl_workaround import NixReplLexer +from sphinx.highlighting import lexers +lexers['nix-repl'] = NixReplLexer() + # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] diff --git a/source/tutorials/first-steps/nix-language.md b/source/tutorials/first-steps/nix-language.md index 9fdd7f966..a00f97a2d 100644 --- a/source/tutorials/first-steps/nix-language.md +++ b/source/tutorials/first-steps/nix-language.md @@ -114,8 +114,13 @@ Use [`nix repl`] to evaluate Nix expressions interactively (by typing them on th ```shell-session $ nix repl Welcome to Nix 2.13.3. Type :? for help. +``` + +And you have a REPL: +```nix-repl nix-repl> 1 + 2 + 3 ``` @@ -127,7 +132,7 @@ If your output does not match the example, try prepending `:p` to the input expr Example: -```shell-session +```nix-repl nix-repl> { a.b.c = 1; } { a = { ... }; }