diff --git a/README.md b/README.md index f50459c..093a1be 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,23 @@ # proc -Maybe the easiest way to run child processes in Deno ever. +Introducing `proc`, a powerful functional extension for `AsyncIterable` in Deno. +`proc` supports managing external processes, provides extensions for concurrent +programming, and works seamlessly with `Deno` IO streams. -## The Old API +## The New Stuff + +Documentation is at +[https://j50n.github.io/deno-proc/](https://j50n.github.io/deno-proc/). + +To use the new API: + +- http://deno.land/x/proc/mod3.ts + +The new API is stabilizing and the code is working. New features and tests are +being added regularly. Changes to existing features are settling down. The +documentation is still a work in progress. + +## The Old Stuff The documentation for the legacy API is available at [Legacy Documentation](./legacy/README.md). @@ -12,27 +27,6 @@ TO use the old API: - https://deno.land/x/proc/mod.ts - https://deno.land/x/proc/mod1.ts -When the new API is ready, there will be a 1.0 release and `mod.ts` will be -switched to the new API. The old API will continue to be maintained at `mod1.ts` -for some time after the Deno 2.0 release. - The old API was built on `Deno.run()` which is now deprecated and scheduled to be removed in Deno 2.0. There were other reasons to start over from scratch, but this was the primary reason everything changed. - -## The New API - -The documentation is available at -[https://j50n.github.io/deno-proc/](https://j50n.github.io/deno-proc/). - -To use the new API: - -- http://deno.land/x/proc/mod3.ts - -The new API is stabilizing and the code is working. New tests are being added -regularly. Obviously the documentation is a work in progress. Refactors are -likely before the 1.0.0 release. - -The new API is a nice way to work with `AsyncIterable` in general. It also -blends in a really nice way to work with child processes where things like -errors _just work_ and you don't have to worry about resource leaks. diff --git a/site/book.toml b/site/book.toml index de44d99..26ecb45 100644 --- a/site/book.toml +++ b/site/book.toml @@ -7,4 +7,8 @@ title = "Proc" [preprocessor.graphviz] command = "mdbook-graphviz" +output-to-file = false + +[preprocessor.gitversion] +command = "./gitversion.ts" output-to-file = false \ No newline at end of file diff --git a/site/gitversion.ts b/site/gitversion.ts new file mode 100755 index 0000000..b4087a5 --- /dev/null +++ b/site/gitversion.ts @@ -0,0 +1,52 @@ +#!/usr/bin/env -S deno run --allow-run --allow-read + +/** + * Convert `{{gitversion}}` in the content to the latest `git` tag. + * + * My very first `mdbook` plugin. + */ + +import { run, toLines } from "../mod3.ts"; +import { enumerate } from "../src/enumerable.ts"; + +console.error(Deno.args); + +type Section = { + Chapter: { + content: string; + }; +}; + +function isSection(obj: unknown): obj is Section { + return obj != null && typeof obj === "object" && "Chapter" in obj && + obj.Chapter != null; +} + +type CONTEXT = unknown; +type BOOK = { + sections: Section[]; +}; +type Input = [CONTEXT, BOOK]; + +if (Deno.args[0] === "supports") { + Deno.exit(0); +} else { + const [_context, book]: Input = JSON.parse( + (await enumerate(Deno.stdin.readable).transform(toLines).collect()) + .join("\n"), + ); + + const gitversion = (await run("git", "describe", "--tags").lines.first) + .split("-")[0]; + + for (const section of book.sections) { + if (isSection(section)) { + section.Chapter.content = section.Chapter.content.replaceAll( + "{{gitversion}}", + gitversion, + ); + } + } + + console.log(JSON.stringify(book)); +} diff --git a/site/src/introduction.md b/site/src/introduction.md index 4fb8077..93d99ef 100644 --- a/site/src/introduction.md +++ b/site/src/introduction.md @@ -1,10 +1,119 @@ -# Introduction +# `proc {{gitversion}}` -## Concurrent Total Used Storage for S3 Buckets +`proc` is a powerful functional extension for `AsyncIterable` in Deno. It +supports managing external processes, provides extensions for concurrent +programming, and works seamlessly with `Deno` IO streams. With `proc`, writing +shell-style solutions in Deno is painless. -`proc` supports controlled concurrent operations. This is the feature missing from `Promise.all()`. +## Import -If you have to work with S3 buckets, you know it is time consuming to determine how much storage space you are using/paying for, and where you are using the most storage. `proc` makes it possible to run `ls --summarize` with parallelism matching the number of CPU cores available (or whatever concurrency you specify). +Import using this path (note the use of `mod3.ts` rather than `mod.ts`). + +```typescript +import { + Cmd, + enumerate, + read, + run, +} from "https://deno.land/x/proc@{{gitversion}}/mod3.ts"; +``` + +## Examples + +These examples show some of the things `proc` can do. + +### Running a Process + +List the file names in the current directory (`-1` puts each on its own line), +capture as lines, and collect the names into an array. + +```typescript +const filesAndFolders = run("ls", "-1", "-a", ".").lines.collect(); +``` + +### Chaining Processes Together + +Read "War and Peace" in from a compressed file. Uncompress the file. `grep` out +empty lines. Print it. + +```typescript +const warandpeace = resolve("./warandpeace.txt.gz"); + +read(warandpeace) + .run("gunzip"). + .run("grep", "-v", "^$") + .lines + .forEach((line) => console.log(line)); +``` + +This is equivalent to: + +```sh +cat ./warandpeace.txt.gz | gunzip | grep -v '^$' +``` + +### Working with Multiple Copies of the Same Data + +Read "War and Peace" and uncompress. Convert to lower case. Split into words. +Split into `A` and `B` enumerations. + +With `A`, sort the words, find the unique words, and count them. + +With `B`, just count the total number of words. + +```typescript +const warandpeace = resolve("./warandpeace.txt.gz"); + +const [wordsA, wordsB] = read(warandpeace) + .run("gunzip").lines + .map((line) => line.toLocaleLowerCase()) + .run("grep", "-oE", "(\\w|')+") + .tee(); + +const [uniqueWords, totalWords] = await Promise.all([ + wordsA.run("sort").run("uniq").run("wc", "-l").lines + .map((n) => parseInt(n, 10)) + .first, + wordsB.run("wc", "-l").lines + .map((n) => parseInt(n, 10)) + .first, +]); + +console.log(`Total: ${totalWords.toLocaleString()}`); +console.log(`Unique: ${uniqueWords.toLocaleString()}`); +``` + +This is (_almost_) equivalent to: + +```sh +# Count unique words +cat ./warandpeace.txt.gz \ + | gunzip \ + | tr '[:upper:]' '[:lower:]' \ + | grep -oE "(\\w|')+" \ + | sort \ + | uniq \ + | wc -l + +# Count all words +cat ./warandpeace.txt.gz \ + | gunzip \ + | tr '[:upper:]' '[:lower:]' \ + | grep -oE "(\\w|')+" \ + | wc -l +``` + +### Concurrent Total Used Storage for S3 Buckets + +`proc` supports concurrent operations with controlled (limited) concurrency. +This is a way to run child processes in parallel without swamping your server. + +If you have to work with S3 buckets, you know it is time consuming to determine +how much storage space you are using/paying for, and where you are using the +most storage. `proc` makes it possible to run `ls --summarize` with parallelism +matching the number of CPU cores available (or whatever concurrency you +specify). The specific methods that support concurrent operations are +`.concurrentMap()` and `.concurrentUnorderedMap()`. To list the `s3` buckets in your AWS account from terminal: @@ -61,7 +170,7 @@ enumerate(buckets).concurrentUnorderedMap( ) ``` -Use `nice` because this will eat your server otherwise. The method +Use `nice` because _this will eat your server otherwise._ The method `.concurrentUnorderedMap()` will, by default, run one process for each CPU available concurrently until all work is done. diff --git a/site/src/utility.md b/site/src/utility.md index b4e2f88..66bb73f 100644 --- a/site/src/utility.md +++ b/site/src/utility.md @@ -58,7 +58,7 @@ for (const i of range({ from: 10, to: 0, step: -1 })) { ### Example 4 -Prinst the numbers from `0` to `99`. +Prints the numbers from `0` to `99`. ```typescript for (const i of range({ to: 10 })) { diff --git a/tests/enumerable/showcase.test.ts b/tests/enumerable/showcase.test.ts index f928b71..abfd858 100644 --- a/tests/enumerable/showcase.test.ts +++ b/tests/enumerable/showcase.test.ts @@ -3,27 +3,39 @@ import { assertEquals } from "../deps/asserts.ts"; import { fromFileUrl } from "../deps/path.ts"; Deno.test({ - name: "I can count the words in a file, shorthand version.", + name: + "I can count the words in a file two different ways (tee), shorthand version.", async fn() { const file = await Deno.open( fromFileUrl(import.meta.resolve("./warandpeace.txt.gz")), ); - const buffer = true; - - const count = await enumerate(file.readable) - .run("gunzip") - .run("grep", "-oE", "(\\w|')+").lines + const [words1, words2] = enumerate(file.readable) + .run("gunzip").lines .map((line) => line.toLocaleLowerCase()) - .run({ buffer }, "sort") - .run("uniq") - .lines - .reduce(0, (c, _item) => c + 1); + .run({ buffer: true }, "grep", "-oE", "(\\w|')+") + .tee(); + + const [uniqueWords, totalWords] = await Promise.all([ + words1.run("sort").run("uniq").run("wc", "-l").lines.map((n) => + parseInt(n, 10) + ).first, + words2.run("wc", "-l").lines.map((n) => parseInt(n, 10)).first, + ]); + + console.log(`Total: ${totalWords.toLocaleString()}`); + console.log(`Unique: ${uniqueWords.toLocaleString()}`); + + assertEquals( + uniqueWords, + 17_557, + "Unique words.", + ); assertEquals( - count, - 17557, - "The words I count match the value I believe I should get.", + totalWords, + 572_642, + "Total words.", ); }, });