This repository was archived by the owner on Aug 7, 2026. It is now read-only.
Repository navigation
Expand file tree
/
Copy pathlib.rs
More file actions
804 lines (725 loc) · 28.5 KB
/
Copy pathlib.rs
File metadata and controls
804 lines (725 loc) · 28.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
use wasm_bindgen::prelude::*;
use blusc::api::{
blosc2_cbuffer_sizes as blusc_cbuffer_sizes, blosc2_compress_ctx, blosc2_create_cctx,
Blosc2Cparams, BLOSC2_CPARAMS_DEFAULTS,
};
use blusc::convenience::blosc1_decompress as blusc_decompress;
use blusc::internal::constants::*;
fn cname_to_compcode(cname: &str) -> u8 {
match cname {
"lz4" => BLOSC_LZ4,
"lz4hc" => BLOSC_LZ4HC,
"snappy" => BLOSC_SNAPPY,
"zlib" => BLOSC_ZLIB,
"zstd" => BLOSC_ZSTD,
_ => BLOSC_BLOSCLZ,
}
}
#[wasm_bindgen]
pub fn blosc_compress(
data: &[u8],
cname: &str,
clevel: u8,
shuffle: i32,
typesize: i32,
blocksize: i32,
) -> Vec<u8> {
let compcode = cname_to_compcode(cname);
let mut filters = [BLOSC_NOFILTER; BLOSC2_MAX_FILTERS as usize];
match shuffle {
s if s == BLOSC_SHUFFLE as i32 => filters[5] = BLOSC_SHUFFLE,
s if s == BLOSC_BITSHUFFLE as i32 => filters[5] = BLOSC_BITSHUFFLE,
_ => {}
}
let cparams = Blosc2Cparams {
compcode,
clevel,
typesize,
blocksize,
filters,
splitmode: BLOSC_FORWARD_COMPAT_SPLIT as i32,
nthreads: 1,
..BLOSC2_CPARAMS_DEFAULTS
};
let ctx = blosc2_create_cctx(cparams);
let max_out = data.len() + BLOSC2_MAX_OVERHEAD;
let mut dest = vec![0u8; max_out];
let n = blosc2_compress_ctx(&ctx, data, &mut dest);
if n <= 0 {
return vec![];
}
dest.truncate(n as usize);
dest
}
/// A Blosc header is untrusted input: `blusc` reads the buffer sizes straight out
/// of it and then indexes the source with them, so a corrupt chunk can panic
/// inside the decompressor. A panic traps the whole WASM instance and takes every
/// later call down with it, so implausible headers are rejected here instead and
/// `src/blosc.ts` turns the empty result into a thrown Error.
fn blosc_header_is_sane(data: &[u8]) -> bool {
if data.len() < BLOSC_MIN_HEADER_LENGTH {
return false;
}
let version = data[0];
if version == 0 || version > BLOSC2_VERSION_FORMAT_STABLE {
return false;
}
let header_len = if version == BLOSC2_VERSION_FORMAT_STABLE
|| version == BLOSC2_VERSION_FORMAT_BETA1
|| version == BLOSC2_VERSION_FORMAT_ALPHA
{
BLOSC_EXTENDED_HEADER_LENGTH
} else {
BLOSC_MIN_HEADER_LENGTH
};
if data.len() < header_len {
return false;
}
let (nbytes, cbytes, _) = blusc_cbuffer_sizes(data);
// The frame records its own compressed length; it cannot exceed what we hold.
if cbytes < header_len || cbytes > data.len() {
return false;
}
// A header claiming more than Blosc allows only gets us an allocation
// failure, which aborts the instance just like a panic does.
if nbytes > BLOSC2_MAX_BUFFERSIZE {
return false;
}
// An uncompressed (memcpy'd) frame stores nbytes verbatim after the header.
// This is the bound blusc itself omits before copying out of the source.
if data[2] & BLOSC_MEMCPYED != 0 && header_len.saturating_add(nbytes) > data.len() {
return false;
}
true
}
#[wasm_bindgen]
pub fn blosc_decompress(data: &[u8]) -> Vec<u8> {
if !blosc_header_is_sane(data) {
return vec![];
}
match blusc_decompress(data) {
Ok(result) => result,
Err(_) => vec![],
}
}
#[wasm_bindgen]
pub fn blosc_cbuffer_sizes(data: &[u8]) -> Vec<u32> {
let (nb, cb, bs) = blusc_cbuffer_sizes(data);
vec![nb as u32, cb as u32, bs as u32]
}
/// The uncompressed size a well-formed Blosc frame declares, or `undefined` if
/// the header is not one we would decode.
///
/// `blosc_decompress` signals failure with an empty `Vec`, and a frame that
/// really does hold zero bytes decodes to exactly the same thing —
/// `blosc_compress(&[])` emits a valid 32-byte frame declaring `nbytes = 0`.
/// `src/blosc.ts` uses this to tell them apart. Reading `blosc_cbuffer_sizes`
/// directly would not do: it reports zeroes for garbage too, which is the
/// answer that makes a corrupt chunk look like an empty one.
#[wasm_bindgen]
pub fn blosc_declared_size(data: &[u8]) -> Option<usize> {
if !blosc_header_is_sane(data) {
return None;
}
let (nbytes, _, _) = blusc_cbuffer_sizes(data);
Some(nbytes)
}
// Standalone LZ4 codec — numcodecs format: 4-byte LE original size + LZ4 block
#[wasm_bindgen]
pub fn lz4_compress(data: &[u8], _acceleration: i32) -> Vec<u8> {
let compressed = lz4_flex::block::compress(data);
let orig_size = data.len() as u32;
let mut result = Vec::with_capacity(4 + compressed.len());
result.extend_from_slice(&orig_size.to_le_bytes());
result.extend_from_slice(&compressed);
result
}
/// numcodecs' ceiling on a single buffer, mirrored in `src/lz4.ts`.
const MAX_DECODED_SIZE: usize = 0x7e00_0000;
/// Max output bytes an LZ4 block can produce per input byte: each match-length
/// extension byte adds 255 to the copy. A constant buffer measures 255.0:1.
const LZ4_MAX_EXPANSION: usize = 256;
/// The size in an LZ4 header is untrusted, and `lz4_flex` zero-fills a buffer
/// that large before it reads a single byte of the block — so a corrupt header
/// asking for 4 GB aborts the WASM instance rather than failing cleanly. Reject
/// sizes no LZ4 block could actually expand to.
fn lz4_declared_size_is_plausible(orig_size: usize, block_len: usize) -> bool {
orig_size <= MAX_DECODED_SIZE
&& orig_size
<= block_len
.saturating_mul(LZ4_MAX_EXPANSION)
.saturating_add(1024)
}
/// A zeroed buffer of `len` bytes, or `None` if the allocator cannot supply one.
///
/// The plausibility guards above reject sizes no real block could produce, but
/// what they cannot bound is how much memory this instance still has. An
/// infallible `vec![0; n]` turns that shortfall into `handle_alloc_error`, which
/// aborts the module exactly like a panic and takes every later decode with it —
/// so the reservation is made fallibly and a shortfall becomes an ordinary
/// decode failure.
fn try_zeroed(len: usize) -> Option<Vec<u8>> {
let mut buf = Vec::new();
buf.try_reserve_exact(len).ok()?;
buf.resize(len, 0);
Some(buf)
}
#[wasm_bindgen]
pub fn lz4_decompress(data: &[u8]) -> Vec<u8> {
if data.len() < 4 {
return vec![];
}
let orig_size = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize;
let block = &data[4..];
if !lz4_declared_size_is_plausible(orig_size, block.len()) {
return vec![];
}
let Some(mut out) = try_zeroed(orig_size) else {
return vec![];
};
match lz4_flex::block::decompress_into(block, &mut out) {
// A block that decodes short of its declared size is corrupt; returning
// the zero-padded remainder would hand back silent garbage.
Ok(n) if n == orig_size => out,
_ => vec![],
}
}
// Standalone Zstd codec — standard Zstd frame format
#[wasm_bindgen]
pub fn zstd_compress(data: &[u8], level: i32) -> Vec<u8> {
zstd::bulk::compress(data, level).unwrap_or_default()
}
/// Max output bytes a zstd frame can produce per input byte: a 4-byte RLE block
/// regenerates a full 128 KB block. A constant buffer measures ~32500:1.
const ZSTD_MAX_EXPANSION: usize = 131_072 / 4;
/// Same hazard as the LZ4 header, one size class up: `zstd::bulk::decompress`
/// reserves the capacity we hand it up front, and that capacity comes straight
/// out of the frame's declared content size.
fn zstd_declared_size_is_plausible(declared: usize, frame_len: usize) -> bool {
declared <= MAX_DECODED_SIZE
&& declared
<= frame_len
.saturating_mul(ZSTD_MAX_EXPANSION)
.saturating_add(131_072)
}
#[wasm_bindgen]
pub fn zstd_decompress(data: &[u8]) -> Vec<u8> {
// Frames written by `zstd_compress` always declare their content size; the
// multiple-of-input fallback only covers frames from other producers that
// omit it, and will fail cleanly if it guesses low.
let capacity = zstd_frame_content_size(data).unwrap_or_else(|| data.len().saturating_mul(4));
if !zstd_declared_size_is_plausible(capacity, data.len()) {
return vec![];
}
// `zstd::bulk::decompress` would reserve `capacity` infallibly; reserve it
// ourselves so a request this instance cannot satisfy fails as a decode
// error rather than aborting the module. See `try_zeroed`.
let mut out = Vec::new();
if out.try_reserve_exact(capacity).is_err() {
return vec![];
}
match zstd::bulk::Decompressor::new() {
Ok(mut decoder) => match decoder.decompress_to_buffer(data, &mut out) {
Ok(_) => out,
Err(_) => vec![],
},
Err(_) => vec![],
}
}
/// The content size a zstd frame declares in its header, or `undefined` when it
/// declares none (or is not a zstd frame at all).
///
/// `zstd_decompress` reports failure the same way every decoder here does — an
/// empty `Vec` — which on its own cannot be told apart from a frame that really
/// does hold nothing. `src/zstd.ts` uses this to make that call before deciding
/// whether to throw. Exposed for that reason, not as part of the codec API.
#[wasm_bindgen]
pub fn zstd_declared_content_size(data: &[u8]) -> Option<usize> {
zstd_frame_content_size(data)
}
fn zstd_frame_content_size(data: &[u8]) -> Option<usize> {
if data.len() < 5 {
return None;
}
let magic = u32::from_le_bytes([data[0], data[1], data[2], data[3]]);
if magic != 0xFD2FB528 {
return None;
}
let descriptor = data[4];
let fcs_flag = (descriptor >> 6) & 0x3;
let single_segment = (descriptor >> 5) & 0x1;
let dict_id_flag = descriptor & 0x3;
let content_checksum = (descriptor >> 2) & 0x1;
let _ = content_checksum;
let mut offset = 5;
// Window descriptor (absent if single_segment)
if single_segment == 0 {
offset += 1;
}
// Dictionary ID
let dict_id_size = match dict_id_flag {
0 => 0,
1 => 1,
2 => 2,
3 => 4,
_ => return None,
};
offset += dict_id_size;
// Frame content size
match fcs_flag {
0 if single_segment == 1 => {
if offset >= data.len() {
return None;
}
Some(data[offset] as usize)
}
1 => {
if offset + 2 > data.len() {
return None;
}
let val = u16::from_le_bytes([data[offset], data[offset + 1]]);
Some(val as usize + 256)
}
2 => {
if offset + 4 > data.len() {
return None;
}
let val = u32::from_le_bytes([
data[offset],
data[offset + 1],
data[offset + 2],
data[offset + 3],
]);
Some(val as usize)
}
3 => {
if offset + 8 > data.len() {
return None;
}
let val = u64::from_le_bytes([
data[offset],
data[offset + 1],
data[offset + 2],
data[offset + 3],
data[offset + 4],
data[offset + 5],
data[offset + 6],
data[offset + 7],
]);
// `usize` is 32-bit on wasm32, so a declared size past 4 GiB would
// wrap to a small plausible-looking one rather than being rejected.
usize::try_from(val).ok()
}
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
const COMPRESSORS: [&str; 6] = ["blosclz", "lz4", "lz4hc", "snappy", "zlib", "zstd"];
/// Mildly compressible bytes, so a round trip exercises a real codec path
/// rather than an all-zero fast path.
fn sample() -> Vec<u8> {
(0..8192u32)
.flat_map(|i| (i as u16).to_le_bytes())
.collect()
}
// The TypeScript layer always passes typesize 1; see `src/blosc.ts`.
fn compress_blosc(data: &[u8], cname: &str, clevel: u8, shuffle: i32) -> Vec<u8> {
blosc_compress(data, cname, clevel, shuffle, 1, 0)
}
#[test]
fn cname_maps_to_blosc_compcode() {
assert_eq!(cname_to_compcode("lz4"), BLOSC_LZ4);
assert_eq!(cname_to_compcode("lz4hc"), BLOSC_LZ4HC);
assert_eq!(cname_to_compcode("snappy"), BLOSC_SNAPPY);
assert_eq!(cname_to_compcode("zlib"), BLOSC_ZLIB);
assert_eq!(cname_to_compcode("zstd"), BLOSC_ZSTD);
assert_eq!(cname_to_compcode("blosclz"), BLOSC_BLOSCLZ);
// numcodecs validates the name in JS; an unknown one still must not panic.
assert_eq!(cname_to_compcode("not-a-compressor"), BLOSC_BLOSCLZ);
}
#[test]
fn blosc_round_trips_every_compressor() {
let data = sample();
for cname in COMPRESSORS {
let compressed = compress_blosc(&data, cname, 5, BLOSC_SHUFFLE as i32);
assert!(!compressed.is_empty(), "{cname} produced no output");
assert_eq!(blosc_decompress(&compressed), data, "{cname} round trip");
}
}
#[test]
fn blosc_round_trips_every_shuffle_mode() {
let data = sample();
let modes = [
BLOSC_NOSHUFFLE as i32,
BLOSC_SHUFFLE as i32,
BLOSC_BITSHUFFLE as i32,
];
for shuffle in modes {
let compressed = compress_blosc(&data, "lz4", 5, shuffle);
assert!(
!compressed.is_empty(),
"shuffle {shuffle} produced no output"
);
assert_eq!(blosc_decompress(&compressed), data, "shuffle {shuffle}");
}
}
#[test]
fn blosc_round_trips_at_clevel_zero() {
// clevel 0 stores the buffer uncompressed but still emits a Blosc frame.
let data = sample();
let compressed = compress_blosc(&data, "lz4", 0, BLOSC_NOSHUFFLE as i32);
assert!(!compressed.is_empty());
assert_eq!(blosc_decompress(&compressed), data);
}
#[test]
fn blosc_round_trips_an_empty_buffer() {
let compressed = compress_blosc(&[], "lz4", 5, BLOSC_SHUFFLE as i32);
// An empty input may legitimately yield no output; what must not happen
// is a panic across the wasm_bindgen boundary.
assert!(blosc_decompress(&compressed).is_empty());
}
#[test]
fn blosc_header_is_self_describing() {
// Blosc1-compatible framing: the header carries the sizes back out, which
// is what lets Python numcodecs read a chunk we wrote.
let data = sample();
let compressed = compress_blosc(&data, "zstd", 5, BLOSC_SHUFFLE as i32);
let sizes = blosc_cbuffer_sizes(&compressed);
assert_eq!(sizes.len(), 3);
assert_eq!(sizes[0] as usize, data.len(), "nbytes");
assert_eq!(sizes[1] as usize, compressed.len(), "cbytes");
assert!(sizes[2] > 0, "blocksize");
}
#[test]
fn blosc_decompress_returns_empty_on_garbage() {
// The error convention across the boundary is an empty Vec, never a panic;
// src/blosc.ts turns that into a thrown Error. A panic here would trap the
// WASM instance, so a single corrupt Zarr chunk would break every later read.
assert!(blosc_decompress(&[]).is_empty(), "empty");
assert!(
blosc_decompress(&[0u8; 3]).is_empty(),
"shorter than a header"
);
assert!(blosc_decompress(&[0u8; 64]).is_empty(), "zeroed header");
assert!(blosc_decompress(&[0xff; 64]).is_empty(), "all-ones header");
assert!(
blosc_decompress(&[0x02; 64]).is_empty(),
"plausible version byte"
);
}
#[test]
fn blosc_decompress_rejects_a_lying_header() {
let data = sample();
let compressed = compress_blosc(&data, "lz4", 5, BLOSC_SHUFFLE as i32);
// cbytes larger than the buffer we were actually handed.
let mut overlong = compressed.clone();
overlong[12..16].copy_from_slice(&(compressed.len() as u32 + 1).to_le_bytes());
assert!(
blosc_decompress(&overlong).is_empty(),
"cbytes past the buffer"
);
// A memcpy'd frame promising far more payload than it carries. This is the
// bound blusc skips before copying, and the shape that used to panic.
let mut memcpyed = compressed.clone();
memcpyed[2] |= BLOSC_MEMCPYED;
memcpyed[4..8].copy_from_slice(&u32::MAX.to_le_bytes());
assert!(
blosc_decompress(&memcpyed).is_empty(),
"memcpy past the buffer"
);
// Truncation must not be mistaken for a decodable frame.
assert!(
blosc_decompress(&compressed[..compressed.len() / 2]).is_empty(),
"truncated"
);
}
#[test]
fn blosc_header_check_accepts_what_we_emit() {
// The guard must not reject our own frames, in any of the shapes we emit:
// compressed, stored at clevel 0 (which sets BLOSC_MEMCPYED), and empty.
let data = sample();
for cname in COMPRESSORS {
let compressed = compress_blosc(&data, cname, 5, BLOSC_SHUFFLE as i32);
assert!(blosc_header_is_sane(&compressed), "{cname}");
}
let stored = compress_blosc(&data, "lz4", 0, BLOSC_NOSHUFFLE as i32);
assert!(blosc_header_is_sane(&stored), "clevel 0");
assert_ne!(stored[2] & BLOSC_MEMCPYED, 0, "clevel 0 should be memcpy'd");
}
#[test]
fn lz4_uses_numcodecs_framing() {
// 4-byte little-endian original size, then a raw LZ4 block. Changing this
// silently would break interchange with Python numcodecs.
let data = sample();
let compressed = lz4_compress(&data, 1);
assert!(compressed.len() > 4);
assert_eq!(&compressed[..4], &(data.len() as u32).to_le_bytes());
let block = lz4_flex::block::decompress(&compressed[4..], data.len()).unwrap();
assert_eq!(block, data);
}
#[test]
fn lz4_round_trips() {
let data = sample();
assert_eq!(lz4_decompress(&lz4_compress(&data, 1)), data);
}
#[test]
fn lz4_round_trips_an_empty_buffer() {
let compressed = lz4_compress(&[], 1);
assert_eq!(&compressed[..4], &0u32.to_le_bytes());
assert!(lz4_decompress(&compressed).is_empty());
}
#[test]
fn lz4_decompress_returns_empty_on_garbage() {
assert!(lz4_decompress(&[]).is_empty(), "no header");
assert!(lz4_decompress(&[0, 0, 0]).is_empty(), "truncated header");
// Well-formed header, unreadable block.
let mut bad = 32u32.to_le_bytes().to_vec();
bad.extend_from_slice(&[0xff; 16]);
assert!(lz4_decompress(&bad).is_empty(), "corrupt block");
}
#[test]
fn lz4_rejects_an_implausible_declared_size() {
// A declared size is a request to zero-fill that many bytes before the
// block is even read, so it has to be bounded by what the block could
// possibly expand to. 4 GB from an 8-byte chunk is an OOM, not an error.
assert!(
!lz4_declared_size_is_plausible(u32::MAX as usize, 8),
"u32::MAX"
);
assert!(
!lz4_declared_size_is_plausible(MAX_DECODED_SIZE + 1, usize::MAX),
"over cap"
);
assert!(!lz4_declared_size_is_plausible(16 << 20, 16), "past 255:1");
let mut hostile = u32::MAX.to_le_bytes().to_vec();
hostile.extend_from_slice(&[0u8; 8]);
assert!(lz4_decompress(&hostile).is_empty());
}
#[test]
fn lz4_accepts_the_maximum_real_expansion() {
// The guard must not reject genuinely well-compressed data. A constant
// buffer sits right on the format's 255:1 ceiling, so if anything real
// trips this bound, it is this.
let data = vec![0u8; 1 << 20];
let compressed = lz4_compress(&data, 1);
let ratio = data.len() / (compressed.len() - 4);
assert!(ratio >= 250, "expected a near-ceiling ratio, got {ratio}:1");
assert!(lz4_declared_size_is_plausible(
data.len(),
compressed.len() - 4
));
assert_eq!(lz4_decompress(&compressed), data);
}
#[test]
fn zstd_emits_a_standard_frame() {
let data = sample();
let compressed = zstd_compress(&data, 1);
assert!(compressed.len() > 4);
assert_eq!(
u32::from_le_bytes([compressed[0], compressed[1], compressed[2], compressed[3]]),
0xFD2FB528,
"zstd frame magic"
);
}
#[test]
fn zstd_round_trips_across_levels() {
let data = sample();
for level in [1, 10, 22] {
let compressed = zstd_compress(&data, level);
assert!(!compressed.is_empty(), "level {level} produced no output");
assert_eq!(zstd_decompress(&compressed), data, "level {level}");
}
}
#[test]
fn zstd_round_trips_highly_compressible_data() {
// Guards the `data.len() * 4` capacity fallback in zstd_decompress: this
// buffer compresses far past 4:1, so decoding only fits if the frame's
// declared content size is used instead.
let data = vec![7u8; 1 << 16];
assert_eq!(zstd_decompress(&zstd_compress(&data, 1)), data);
}
#[test]
fn zstd_decompress_returns_empty_on_garbage() {
assert!(zstd_decompress(&[]).is_empty());
assert!(zstd_decompress(&[0xff; 64]).is_empty());
}
#[test]
fn zstd_rejects_an_implausible_declared_size() {
assert!(
!zstd_declared_size_is_plausible(usize::MAX, 64),
"usize::MAX"
);
assert!(
!zstd_declared_size_is_plausible(MAX_DECODED_SIZE + 1, usize::MAX),
"over cap"
);
assert!(
!zstd_declared_size_is_plausible(1 << 30, 16),
"past 32768:1"
);
// A frame header is enough to trigger the allocation; the block never
// has to be readable, so the guard has to run before decompression.
let mut hostile = frame_header(0xE0, &[0u8; 8]);
hostile[5..13].copy_from_slice(&u64::MAX.to_le_bytes());
assert!(zstd_decompress(&hostile).is_empty());
}
#[test]
fn zstd_accepts_the_maximum_real_expansion() {
// 16 MB of one byte lands near the format's 32768:1 RLE ceiling, so this
// is the shape most likely to be rejected by an over-tight bound.
let data = vec![0u8; 16 << 20];
let compressed = zstd_compress(&data, 1);
let ratio = data.len() / compressed.len();
assert!(
ratio >= 30_000,
"expected a near-ceiling ratio, got {ratio}:1"
);
assert!(zstd_declared_size_is_plausible(
data.len(),
compressed.len()
));
assert_eq!(zstd_decompress(&compressed), data);
}
/// Builds a synthetic zstd frame header: magic, frame descriptor, then the
/// trailing bytes the descriptor says to expect.
fn frame_header(descriptor: u8, rest: &[u8]) -> Vec<u8> {
let mut frame = vec![0x28, 0xB5, 0x2F, 0xFD, descriptor];
frame.extend_from_slice(rest);
frame
}
#[test]
fn frame_content_size_rejects_non_zstd_input() {
assert_eq!(zstd_frame_content_size(&[]), None, "empty");
assert_eq!(
zstd_frame_content_size(&[0x28, 0xB5, 0x2F, 0xFD]),
None,
"no descriptor"
);
assert_eq!(zstd_frame_content_size(&[0u8; 16]), None, "wrong magic");
}
#[test]
fn frame_content_size_reads_single_byte_field() {
// fcs_flag 0 + single_segment: one byte of size, no window descriptor.
assert_eq!(
zstd_frame_content_size(&frame_header(0x20, &[0x42])),
Some(0x42)
);
// fcs_flag 0 without single_segment means the field is absent entirely.
assert_eq!(zstd_frame_content_size(&frame_header(0x00, &[0x42])), None);
}
#[test]
fn frame_content_size_reads_two_byte_field() {
// fcs_flag 1: u16 biased by 256, and a window descriptor byte to skip.
let frame = frame_header(0x40, &[0x00, 0x10, 0x00]);
assert_eq!(zstd_frame_content_size(&frame), Some(16 + 256));
}
#[test]
fn frame_content_size_skips_the_dictionary_id() {
// fcs_flag 2 + single_segment + 1-byte dict id.
let frame = frame_header(0xA1, &[0x07, 0x00, 0x10, 0x00, 0x00]);
assert_eq!(zstd_frame_content_size(&frame), Some(0x1000));
// fcs_flag 3 + single_segment + 4-byte dict id.
let mut rest = vec![0xDE, 0xAD, 0xBE, 0xEF];
rest.extend_from_slice(&1_000_000u64.to_le_bytes());
assert_eq!(
zstd_frame_content_size(&frame_header(0xE3, &rest)),
Some(1_000_000)
);
}
#[test]
fn frame_content_size_does_not_wrap_a_64_bit_field() {
// `usize` is 32-bit on wasm32, where an `as usize` cast would turn a
// declared size past 4 GiB into a small, plausible-looking one.
let frame = frame_header(0xE0, &u64::MAX.to_le_bytes());
assert_eq!(
zstd_frame_content_size(&frame),
usize::try_from(u64::MAX).ok()
);
}
#[test]
fn frame_content_size_rejects_a_truncated_field() {
assert_eq!(
zstd_frame_content_size(&frame_header(0x20, &[])),
None,
"1-byte"
);
assert_eq!(
zstd_frame_content_size(&frame_header(0x40, &[0x00, 0x10])),
None,
"2-byte"
);
assert_eq!(
zstd_frame_content_size(&frame_header(0x80, &[0x00, 0x10])),
None,
"4-byte"
);
assert_eq!(
zstd_frame_content_size(&frame_header(0xE0, &[0x00; 4])),
None,
"8-byte"
);
}
#[test]
fn frame_content_size_matches_a_real_frame() {
let data = sample();
let compressed = zstd_compress(&data, 1);
assert_eq!(zstd_frame_content_size(&compressed), Some(data.len()));
}
#[test]
fn blosc_declared_size_separates_an_empty_frame_from_a_bad_one() {
// The empty frame is a real frame: 32 bytes of header declaring nothing.
// Without this distinction src/blosc.ts reports it as a decode failure.
let empty = compress_blosc(&[], "lz4", 5, BLOSC_SHUFFLE as i32);
assert!(!empty.is_empty(), "empty input still yields a frame");
assert_eq!(blosc_declared_size(&empty), Some(0));
assert!(blosc_decompress(&empty).is_empty());
let data = sample();
let compressed = compress_blosc(&data, "lz4", 5, BLOSC_SHUFFLE as i32);
assert_eq!(blosc_declared_size(&compressed), Some(data.len()));
// Garbage must not read as "a frame holding zero bytes" — which is
// exactly what blosc_cbuffer_sizes reports for a zeroed buffer, and why
// the raw sizes cannot be used as the empty-vs-failed signal.
assert_eq!(blusc_cbuffer_sizes(&[0u8; 64]).0, 0, "the trap");
assert_eq!(blosc_declared_size(&[0u8; 64]), None);
assert_eq!(blosc_declared_size(&[0xff; 64]), None);
assert_eq!(blosc_declared_size(&[]), None);
assert_eq!(
blosc_declared_size(&compressed[..compressed.len() / 2]),
None,
"truncated"
);
}
#[test]
fn declared_content_size_is_visible_to_the_wrapper() {
// src/zstd.ts needs this to tell a frame that holds nothing from one that
// failed to decode — both come back as an empty Vec.
let data = sample();
assert_eq!(
zstd_declared_content_size(&zstd_compress(&data, 1)),
Some(data.len())
);
assert_eq!(zstd_declared_content_size(&zstd_compress(&[], 1)), Some(0));
assert_eq!(zstd_declared_content_size(&[0xff; 64]), None, "not a frame");
}
#[test]
fn lz4_rejects_a_block_that_stops_short_of_its_declared_size() {
// Decoding into a preallocated buffer means a block that ends early
// leaves the tail zeroed. Handing that back would be silent corruption,
// so a short decode is a failure like any other.
let data = sample();
let compressed = lz4_compress(&data, 1);
let truncated = &compressed[..compressed.len() - 8];
assert!(lz4_decompress(truncated).is_empty());
}
#[test]
fn try_zeroed_reports_a_reservation_it_cannot_make() {
// The plausibility guards bound what a header may claim; this bounds what
// the allocator can actually supply. Without it the shortfall is an abort,
// which on wasm32 takes the whole instance down.
assert_eq!(try_zeroed(0).map(|b| b.len()), Some(0));
assert_eq!(try_zeroed(64).map(|b| b.len()), Some(64));
assert!(try_zeroed(usize::MAX).is_none());
}
}