Add more tests and streaming interface (#3)

* Add more tests

* Enforce rustfmt in CI

* more tests

* more tests

* delete all comments

* make the url cleaning configurable

* dont run handlers twice

* add fuzzing target

* kinda fix

* improve performance, add streaming api

* use simd accelerated utf8 validation (~1.5-4.5% improvement), gate behind feature-flag

.github/workflows/rust.yml

@@ -9,6 +9,16 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
+  formatting:
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: "stable"
+          components: rustfmt
+      - run: cargo fmt --check
+
   check:
     runs-on: "ubuntu-latest"
     steps:

Cargo.toml

@@ -20,11 +20,22 @@ harness = false
 
 [dependencies]
 ahash = "0.8.6"
+bytecount = "0.6.7"
 lol_html = "1.2.0"
 once_cell = "1.18.0"
+simdutf8 = { version = "0.1.4", optional = true }
 slab = "0.4.9"
+thiserror = "1.0.50"
+
+[features]
+default = ["simd"]
+simd = [
+    "bytecount/runtime-dispatch-simd",
+    "dep:simdutf8",
+    "simdutf8/aarch64_neon",
+]
 
 [dev-dependencies]
 ammonia = "3.3.0"
 criterion = "0.5.1"
-insta = "1.34.0"
+insta = { version = "1.34.0", features = ["glob"] }

fuzz/.gitignore

@@ -0,0 +1,4 @@
+target
+corpus
+artifacts
+coverage