Merge pull request #39 from AndersTrier/AndersTrier/Relax_shard_size_…

…requirements

Relax shard size requirements

README.md

@@ -69,7 +69,7 @@ $ cargo bench main
 ## Simple usage
 
 1. Divide data into equal-sized original shards.
-   Shard size must be multiple of 64 bytes.
+   Shard size must be even (`shard.len() % 2 == 0`).
 2. Decide how many recovery shards you want.
 3. Generate recovery shards with [`reed_solomon_simd::encode`].
 4. When some original shards get lost, restore them with [`reed_solomon_simd::decode`].

src/decoder_result.rs

@@ -103,15 +103,11 @@ mod tests {
     use super::*;
     use crate::{test_util, ReedSolomonDecoder, ReedSolomonEncoder};
 
-    #[test]
-    // DecoderResult::restored_original
-    // DecoderResult::restored_original_iter
-    // RestoredOriginal
-    fn decoder_result() {
-        let original = test_util::generate_original(3, 1024, 0);
+    fn simple_roundtrip(shard_size: usize) {
+        let original = test_util::generate_original(3, shard_size, 0);
 
-        let mut encoder = ReedSolomonEncoder::new(3, 2, 1024).unwrap();
-        let mut decoder = ReedSolomonDecoder::new(3, 2, 1024).unwrap();
+        let mut encoder = ReedSolomonEncoder::new(3, 2, shard_size).unwrap();
+        let mut decoder = ReedSolomonDecoder::new(3, 2, shard_size).unwrap();
 
         for original in &original {
             encoder.add_original_shard(original).unwrap();
@@ -120,6 +116,8 @@ mod tests {
         let result = encoder.encode().unwrap();
         let recovery: Vec<_> = result.recovery_iter().collect();
 
+        assert!(recovery.iter().all(|slice| slice.len() == shard_size));
+
         decoder.add_original_shard(1, &original[1]).unwrap();
         decoder.add_recovery_shard(0, recovery[0]).unwrap();
         decoder.add_recovery_shard(1, recovery[1]).unwrap();
@@ -137,4 +135,19 @@ mod tests {
         assert_eq!(iter.next(), None);
         assert_eq!(iter.next(), None);
     }
+
+    #[test]
+    // DecoderResult::restored_original
+    // DecoderResult::restored_original_iter
+    // RestoredOriginal
+    fn decoder_result() {
+        simple_roundtrip(1024);
+    }
+
+    #[test]
+    fn shard_size_not_divisible_by_64() {
+        for shard_size in [2, 4, 6, 30, 32, 34, 62, 64, 66, 126, 128, 130] {
+            simple_roundtrip(shard_size);
+        }
+    }
 }

src/engine/shards.rs

@@ -1,4 +1,4 @@
-use std::ops::{Bound, Index, IndexMut, RangeBounds};
+use std::ops::{Bound, Index, IndexMut, Range, RangeBounds};
 
 // ======================================================================
 // Shards - CRATE
@@ -32,6 +32,44 @@ impl Shards {
         self.data
             .resize(self.shard_count * self.shard_len_64, [0; 64]);
     }
+
+    pub(crate) fn insert(&mut self, index: usize, shard: &[u8]) {
+        debug_assert_eq!(shard.len() % 2, 0);
+
+        let whole_chunk_count = shard.len() / 64;
+        let tail_len = shard.len() % 64;
+
+        let (src_chunks, src_tail) = shard.split_at(shard.len() - tail_len);
+
+        let dst = &mut self[index];
+        dst[..whole_chunk_count]
+            .as_flattened_mut()
+            .copy_from_slice(src_chunks);
+
+        // Last chunk is special if shard.len() % 64 != 0.
+        // See src/algorithm.md for an explanation.
+        if tail_len > 0 {
+            let (src_lo, src_hi) = src_tail.split_at(tail_len / 2);
+            let (dst_lo, dst_hi) = dst[whole_chunk_count].split_at_mut(32);
+            dst_lo[..src_lo.len()].copy_from_slice(src_lo);
+            dst_hi[..src_hi.len()].copy_from_slice(src_hi);
+        }
+    }
+
+    // Undoes the encoding of the last chunk for the given range of shards
+    pub(crate) fn undo_last_chunk_encoding(&mut self, shard_bytes: usize, range: Range<usize>) {
+        let whole_chunk_count = shard_bytes / 64;
+        let tail_len = shard_bytes % 64;
+
+        if tail_len == 0 {
+            return;
+        };
+
+        for idx in range {
+            let last_chunk = &mut self[idx][whole_chunk_count];
+            last_chunk.copy_within(32..32 + tail_len / 2, tail_len / 2);
+        }
+    }
 }
 
 // ======================================================================

src/lib.rs

@@ -75,7 +75,7 @@ pub enum Error {
     },
 
     /// Given or inferred shard size is invalid:
-    /// Size must be non-zero and multiple of 64 bytes.
+    /// Size must be non-zero and even.
     ///
     /// - Shard size is given explicitly to encoders/decoders
     ///   and inferred for [`reed_solomon_simd::encode`]
@@ -171,7 +171,7 @@ impl fmt::Display for Error {
             Error::InvalidShardSize { shard_bytes } => {
                 write!(
                     f,
-                    "invalid shard size: {} bytes (must non-zero and multiple of 64)",
+                    "invalid shard size: {} bytes (must non-zero and multiple of 2)",
                     shard_bytes
                 )
             }

src/rate.rs

@@ -98,7 +98,7 @@ pub trait Rate<E: Engine> {
                 original_count,
                 recovery_count,
             })
-        } else if shard_bytes == 0 || shard_bytes & 63 != 0 {
+        } else if shard_bytes == 0 || shard_bytes & 1 != 0 {
             Err(Error::InvalidShardSize { shard_bytes })
         } else {
             Ok(())

src/rate/decoder_work.rs

@@ -80,9 +80,8 @@ impl DecoderWork {
                 got: original_shard.len(),
             })
         } else {
-            self.shards[pos]
-                .as_flattened_mut()
-                .copy_from_slice(original_shard);
+            self.shards.insert(pos, original_shard);
+
             self.original_received_count += 1;
             self.received.set(pos, true);
             Ok(())
@@ -110,9 +109,8 @@ impl DecoderWork {
                 got: recovery_shard.len(),
             })
         } else {
-            self.shards[pos]
-                .as_flattened_mut()
-                .copy_from_slice(recovery_shard);
+            self.shards.insert(pos, recovery_shard);
+
             self.recovery_received_count += 1;
             self.received.set(pos, true);
             Ok(())
@@ -156,7 +154,7 @@ impl DecoderWork {
         recovery_base_pos: usize,
         work_count: usize,
     ) {
-        assert!(shard_bytes % 64 == 0);
+        assert!(shard_bytes % 2 == 0);
 
         self.original_count = original_count;
         self.recovery_count = recovery_count;
@@ -178,7 +176,7 @@ impl DecoderWork {
             self.received.grow(max_received_pos);
         }
 
-        self.shards.resize(work_count, shard_bytes / 64);
+        self.shards.resize(work_count, shard_bytes.div_ceil(64));
     }
 
     pub(crate) fn reset_received(&mut self) {
@@ -192,9 +190,16 @@ impl DecoderWork {
         let pos = self.original_base_pos + index;
 
         if index < self.original_count && !self.received[pos] {
-            Some(self.shards[pos].as_flattened())
+            Some(&self.shards[pos].as_flattened()[..self.shard_bytes])
         } else {
             None
         }
     }
+
+    pub(crate) fn undo_last_chunk_encoding(&mut self) {
+        self.shards.undo_last_chunk_encoding(
+            self.shard_bytes,
+            self.original_base_pos..self.original_base_pos + self.original_count,
+        );
+    }
 }

src/rate/encoder_work.rs

@@ -12,7 +12,8 @@ use crate::{
 pub struct EncoderWork {
     original_count: usize,
     recovery_count: usize,
-    shard_bytes: usize,
+
+    pub(crate) shard_bytes: usize,
 
     original_received_count: usize,
     shards: Shards,
@@ -62,9 +63,9 @@ impl EncoderWork {
                 got: original_shard.len(),
             })
         } else {
-            self.shards[self.original_received_count]
-                .as_flattened_mut()
-                .copy_from_slice(original_shard);
+            self.shards
+                .insert(self.original_received_count, original_shard);
+
             self.original_received_count += 1;
             Ok(())
         }
@@ -88,7 +89,7 @@ impl EncoderWork {
     // This must only be called by `EncoderResult`.
     pub(crate) fn recovery(&self, index: usize) -> Option<&[u8]> {
         if index < self.recovery_count {
-            Some(self.shards[index].as_flattened())
+            Some(&self.shards[index].as_flattened()[..self.shard_bytes])
         } else {
             None
         }
@@ -101,17 +102,22 @@ impl EncoderWork {
         shard_bytes: usize,
         work_count: usize,
     ) {
-        assert!(shard_bytes % 64 == 0);
+        assert!(shard_bytes % 2 == 0);
 
         self.original_count = original_count;
         self.recovery_count = recovery_count;
         self.shard_bytes = shard_bytes;
 
         self.original_received_count = 0;
-        self.shards.resize(work_count, shard_bytes / 64);
+        self.shards.resize(work_count, shard_bytes.div_ceil(64));
     }
 
     pub(crate) fn reset_received(&mut self) {
         self.original_received_count = 0;
     }
+
+    pub(crate) fn undo_last_chunk_encoding(&mut self) {
+        self.shards
+            .undo_last_chunk_encoding(self.shard_bytes, 0..self.recovery_count);
+    }
 }

src/rate/rate_high.rs

@@ -77,6 +77,10 @@ impl<E: Engine> RateEncoder<E> for HighRateEncoder<E> {
 
         engine.fft(&mut work, 0, chunk_size, recovery_count, 0);
 
+        // UNDO LAST CHUNK ENCODING
+
+        self.work.undo_last_chunk_encoding();
+
         // DONE
 
         Ok(EncoderResult::new(&mut self.work))
@@ -241,6 +245,10 @@ impl<E: Engine> RateDecoder<E> for HighRateDecoder<E> {
             }
         }
 
+        // UNDO LAST CHUNK ENCODING
+
+        self.work.undo_last_chunk_encoding();
+
         // DONE
 
         Ok(DecoderResult::new(&mut self.work))
@@ -396,6 +404,20 @@ mod tests {
         );
     }
 
+    #[test]
+    fn roundtrip_34000_2000_shard_size_8() {
+        roundtrip_single!(
+            HighRate,
+            34000,
+            2000,
+            8,
+            test_util::HIGH_34000_2000_123_8,
+            &[0..32000],
+            &[0..2000],
+            123
+        );
+    }
+
     // ============================================================
     // ROUNDTRIPS - TWO ROUNDS
 

src/rate/rate_low.rs

@@ -77,6 +77,10 @@ impl<E: Engine> RateEncoder<E> for LowRateEncoder<E> {
             engine::fft_skew_end(engine, &mut work, chunk_start, chunk_size, last_count);
         }
 
+        // UNDO LAST CHUNK ENCODING
+
+        self.work.undo_last_chunk_encoding();
+
         // DONE
 
         Ok(EncoderResult::new(&mut self.work))
@@ -241,6 +245,10 @@ impl<E: Engine> RateDecoder<E> for LowRateDecoder<E> {
             }
         }
 
+        // UNDO LAST CHUNK ENCODING
+
+        self.work.undo_last_chunk_encoding();
+
         // DONE
 
         Ok(DecoderResult::new(&mut self.work))
@@ -396,6 +404,20 @@ mod tests {
         );
     }
 
+    #[test]
+    fn roundtrip_2000_34000_shard_size_8() {
+        roundtrip_single!(
+            LowRate,
+            2000,
+            34000,
+            8,
+            test_util::LOW_2000_34000_123_8,
+            &[0..2000],
+            &[0..32000],
+            123
+        );
+    }
+
     // ============================================================
     // ROUNDTRIPS - TWO ROUNDS
 

src/test_util.rs

@@ -807,6 +807,10 @@ pub(crate) const HIGH_3000_30000_14: &str =
 pub(crate) const HIGH_60000_3000_12: &str =
     "88e68e1d86a0fc168a549e195845d20b49ff85734db20d560c36ff2e14f78676";
 
+// 34000 original ; 2000 recovery ; 123 seed ; shard_bytes = 8
+pub(crate) const HIGH_34000_2000_123_8: &str =
+    "8bd33dbe0189b5bffcb843fd93fd8c85daada2533cc7df0c352773e846b701f5";
+
 // ==================================================
 // LOW RATE
 
@@ -832,3 +836,7 @@ pub(crate) const LOW_3000_60000_13: &str =
 // NOTE: Chunk size is 4096, with partial chunk at end.
 pub(crate) const LOW_30000_3000_15: &str =
     "202f99a2ade121d2404e967d5c04ff390f7a147070a2dcbe71dcf3baeafdf93a";
+
+// 2000 original ; 34000 recovery ; 123 seed ; shard_bytes = 8
+pub(crate) const LOW_2000_34000_123_8: &str =
+    "9bd2da4d03580d3e2471c60a49595b209a6f9a5f1d504d0c4bd017b953efdd99";