Implement streaming Dory

jolt-core/Cargo.toml

@@ -87,7 +87,10 @@ rand_distr = "0.4.3"
 anyhow = "1.0.97"
 derive_more = { version = "2.0.1", features = ["from"] }
 paste = "1.0.15"
+# dory = { git = "https://github.com/spaceandtimefdn/sxt-dory", branch = "dev/twist-shout" }
 dory = { git = "https://github.com/spaceandtimefdn/sxt-dory", branch = "feat/return_row_commitments" }
+# dory = { git = "https://github.com/GaloisInc/sxt-dory", branch = "feat/return_row_commitments_streaming" }
+# dory = { path = "../../sxt-dory" }
 lazy_static = "1.5.0"
 
 [dev-dependencies]

jolt-core/src/benches/bench.rs

@@ -104,7 +104,7 @@ where
 }
 
 fn fibonacci() -> Vec<(tracing::Span, Box<dyn FnOnce()>)> {
-    prove_example("fibonacci-guest", postcard::to_stdvec(&400000u32).unwrap())
+    prove_example("fibonacci-guest", postcard::to_stdvec(&400u32).unwrap())
 }
 
 fn sha2() -> Vec<(tracing::Span, Box<dyn FnOnce()>)> {
@@ -129,7 +129,7 @@ fn prove_example(
     let mut tasks = Vec::new();
     let mut program = host::Program::new(example_name);
     let (bytecode, init_memory_state, _) = program.decode();
-    let (_, _, program_io) = program.trace(&serialized_input);
+    let (_, _, _, program_io) = program.trace(&serialized_input);
 
     let task = move || {
         let preprocessing = JoltRV32IM::prover_preprocess(

jolt-core/src/host/mod.rs

@@ -19,6 +19,7 @@ use rayon::prelude::*;
 use tracer::{
     emulator::memory::Memory,
     instruction::{RV32IMCycle, RV32IMInstruction, VirtualInstructionSequence},
+    LazyTraceIterator,
 };
 
 use crate::field::JoltField;
@@ -202,7 +203,10 @@ impl Program {
 
     // TODO(moodlezoup): Make this generic over InstructionSet
     #[tracing::instrument(skip_all, name = "Program::trace")]
-    pub fn trace(&mut self, inputs: &[u8]) -> (Vec<RV32IMCycle>, Memory, JoltDevice) {
+    pub fn trace(
+        &mut self,
+        inputs: &[u8],
+    ) -> (LazyTraceIterator, Vec<RV32IMCycle>, Memory, JoltDevice) {
         self.build(DEFAULT_TARGET_DIR);
         let elf = self.elf.as_ref().unwrap();
         let mut elf_file =
@@ -221,12 +225,14 @@ impl Program {
             max_output_size: self.max_output_size,
             bytecode_size: Some(bytecode_size),
         };
-        tracer::trace(elf_contents, inputs, &memory_config)
+        let l = tracer::trace_lazy(elf_contents.clone(), inputs, &memory_config);
+        let r = tracer::trace(elf_contents, inputs, &memory_config);
+        (l, r.0, r.1, r.2)
     }
 
     pub fn trace_analyze<F: JoltField>(mut self, inputs: &[u8]) -> ProgramSummary {
         let (bytecode, init_memory_state, _) = self.decode();
-        let (trace, _, io_device) = self.trace(inputs);
+        let (_, trace, _, io_device) = self.trace(inputs);
 
         ProgramSummary {
             trace,

jolt-core/src/poly/commitment/commitment_scheme.rs

@@ -2,7 +2,13 @@ use ark_serialize::{CanonicalDeserialize, CanonicalSerialize};
 use std::borrow::Borrow;
 use std::fmt::Debug;
 
+use crate::poly::commitment::dory::JoltG1Wrapper;
+use crate::poly::compact_polynomial::StreamingCompactWitness;
+use crate::poly::dense_mlpoly::StreamingDenseWitness;
+use crate::poly::multilinear_polynomial::Multilinear;
+use crate::poly::one_hot_polynomial::StreamingOneHotWitness;
 use crate::utils::transcript::Transcript;
+use crate::zkvm::witness::CommittedPolynomial;
 use crate::{
     field::JoltField,
     poly::multilinear_polynomial::MultilinearPolynomial,
@@ -27,7 +33,7 @@ pub trait CommitmentScheme: Clone + Sync + Send + 'static {
     /// A hint that helps the prover compute an opening proof. Typically some byproduct of
     /// the commitment computation, e.g. for Dory the Pedersen commitments to the rows can be
     /// used as a hint for the opening proof.
-    type OpeningProofHint: Sync + Send + Clone + Debug;
+    type OpeningProofHint: Sync + Send + Clone + Debug + PartialEq;
 
     /// Generates the prover setup for this PCS. `max_num_vars` is the maximum number of
     /// variables of any polynomial that will be committed using this setup.
@@ -123,10 +129,41 @@ pub trait CommitmentScheme: Clone + Sync + Send + 'static {
     fn protocol_name() -> &'static [u8];
 }
 
-pub trait StreamingCommitmentScheme: CommitmentScheme {
-    type State<'a>; // : Clone + Debug;
+pub trait StreamingCommitmentScheme_: CommitmentScheme {
+    type State<'a>: Sync;
+    type ChunkState: Send;
 
-    fn initialize<'a>(size: usize, setup: &'a Self::ProverSetup) -> Self::State<'a>;
-    fn process<'a>(state: Self::State<'a>, eval: Self::Field) -> Self::State<'a>;
-    fn finalize<'a>(state: Self::State<'a>) -> Self::Commitment;
+    fn initialize<'a>(poly: Multilinear, size: usize, setup: &'a Self::ProverSetup) -> Self::State<'a>;
+    fn process<'a>(poly: Multilinear, state: Self::State<'a>, eval: Self::Field) -> Self::State<'a>;
+    fn process_chunk<'a, T>(state: &Self::State<'a>, chunk: &[T]) -> Self::ChunkState where Self: StreamingProcessChunk<T>;
+    fn finalize<'a>(state: Self::State<'a>, chunks: &[Self::ChunkState]) -> (Self::Commitment, Self::OpeningProofHint);
 }
+
+/// This trait abstraction allows us to use the optimized version of MSMs for small scalars.
+// JP: Does `VariableBaseMSM` make sense instead of this?
+pub trait StreamingProcessChunk<T>: StreamingCommitmentScheme_ {
+    fn process_chunk_t<'a>(s: &Self::State<'a>, chunk: &[T]) -> Self::ChunkState;
+}
+
+pub trait StreamingCommitmentScheme :
+    StreamingCommitmentScheme_ +
+      StreamingProcessChunk<StreamingDenseWitness<<Self as CommitmentScheme>::Field>>
+    + StreamingProcessChunk<StreamingCompactWitness<u8, <Self as CommitmentScheme>::Field>>
+    + StreamingProcessChunk<StreamingCompactWitness<u16, <Self as CommitmentScheme>::Field>>
+    + StreamingProcessChunk<StreamingCompactWitness<u32, <Self as CommitmentScheme>::Field>>
+    + StreamingProcessChunk<StreamingCompactWitness<u64, <Self as CommitmentScheme>::Field>>
+    + StreamingProcessChunk<StreamingCompactWitness<i64, <Self as CommitmentScheme>::Field>>
+    + StreamingProcessChunk<StreamingOneHotWitness<<Self as CommitmentScheme>::Field>>
+{}
+
+impl<PCS> StreamingCommitmentScheme for PCS
+where
+    PCS: StreamingCommitmentScheme_ +
+      StreamingProcessChunk<StreamingDenseWitness<<Self as CommitmentScheme>::Field>>
+    + StreamingProcessChunk<StreamingCompactWitness<u8, <Self as CommitmentScheme>::Field>>
+    + StreamingProcessChunk<StreamingCompactWitness<u16, <Self as CommitmentScheme>::Field>>
+    + StreamingProcessChunk<StreamingCompactWitness<u32, <Self as CommitmentScheme>::Field>>
+    + StreamingProcessChunk<StreamingCompactWitness<u64, <Self as CommitmentScheme>::Field>>
+    + StreamingProcessChunk<StreamingCompactWitness<i64, <Self as CommitmentScheme>::Field>>
+    + StreamingProcessChunk<StreamingOneHotWitness<<Self as CommitmentScheme>::Field>>,
+{}