Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rust 1.74 #16416

Draft
wants to merge 5 commits into
base: dw/use-latest-master-proof-systems
Choose a base branch
from
Draft

Rust 1.74 #16416

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2ec9256
rust-impure shell loads with 1.74
martyall Dec 12, 2024
1b667e5
remove refs to 1.72, change to 1.74
martyall Dec 13, 2024
d0b632d
update proof systems
martyall Dec 13, 2024
a4a992f
formatting
martyall Dec 13, 2024
308fa27
don't panic if you need to build wasm bindings
martyall Dec 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions buildkite/src/Constants/ContainerImages.dhall
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@
-- NOTE: minaToolchainBookworm is also used for building Ubuntu Jammy packages in CI
{ toolchainBase = "codaprotocol/ci-toolchain-base:v3"
, minaToolchainBullseye =
"gcr.io/o1labs-192920/mina-toolchain@sha256:fee11e64a54fd8f026c4632fed7b7b9835b8262a037cdb156deb61d3d0aac8b2"
"gcr.io/o1labs-192920/mina-toolchain@sha256:8dc3a721b2be119db98a45970c21249f7ed9a95a75f2b330d7c0f37cdafcc99a"
, minaToolchainBookworm =
"gcr.io/o1labs-192920/mina-toolchain@sha256:fee11e64a54fd8f026c4632fed7b7b9835b8262a037cdb156deb61d3d0aac8b2"
"gcr.io/o1labs-192920/mina-toolchain@sha256:8dc3a721b2be119db98a45970c21249f7ed9a95a75f2b330d7c0f37cdafcc99a"
, minaToolchain =
"gcr.io/o1labs-192920/mina-toolchain@sha256:fee11e64a54fd8f026c4632fed7b7b9835b8262a037cdb156deb61d3d0aac8b2"
"gcr.io/o1labs-192920/mina-toolchain@sha256:8dc3a721b2be119db98a45970c21249f7ed9a95a75f2b330d7c0f37cdafcc99a"
, elixirToolchain = "elixir:1.10-alpine"
, nodeToolchain = "node:14.13.1-stretch-slim"
, ubuntu2004 = "ubuntu:20.04"
Expand Down
7 changes: 1 addition & 6 deletions dockerfiles/stages/1-build-deps
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,7 @@ ARG GO_CAPNP_VERSION=v3.0.0-alpha.5
# This should stay in line with:
# - src/lib/crypto/kimchi_bindings/stubs/rust-toolchain.toml
# - src/lib/crypto/proof-systems/rust-toolchain.toml
ARG RUST_VERSION=1.72
# Nightly Rust Version used for WebAssembly builds
# - src/lib/crypto/kimchi_bindings/wasm/rust-toolchain.toml
ARG RUST_NIGHTLY=2023-09-01
ARG RUST_VERSION=1.74
# wasm-pack version
ARG WASM_PACK_VERSION=v0.12.1

Expand Down Expand Up @@ -97,7 +94,6 @@ RUN curl -sL \
RUN curl -s "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" | tar -xz -C /usr/lib/

# --- Rust install via rustup-init to a given RUST_VERSION
# --- Additionally, install RUST_NIGHTLY via rustup
# For more about rustup-init see: https://github.com/rust-lang/rustup/blob/master/README.md
# As opposed to introducing another shell script here (that mostly just determines the platform)
# we just download the binary for the only platform we care about in this docker environment
Expand All @@ -106,7 +102,6 @@ RUN curl --proto '=https' --tlsv1.2 -sSf -o /tmp/rustup-init \
https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init \
&& chmod +x /tmp/rustup-init \
&& /tmp/rustup-init -y --default-toolchain "${RUST_VERSION}" --profile minimal --component rust-src --target wasm32-unknown-unknown \
&& $HOME/.cargo/bin/rustup toolchain install "nightly-${RUST_NIGHTLY}" --profile minimal --component rust-src --target wasm32-unknown-unknown --no-self-update \
&& rm /tmp/rustup-init
USER root

Expand Down
6 changes: 2 additions & 4 deletions nix/rust.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,7 @@ let
# override stdenv.targetPlatform here, if neccesary
};
toolchainHashes = {
"1.72" = "sha256-dxE7lmCFWlq0nl/wKcmYvpP9zqQbBitAQgZ1zx9Ooik=";
"nightly-2023-09-01" =
"sha256-zek9JAnRaoX8V0U2Y5ssXVe9tvoQ0ERGXfUCUGYdrMA=";
"1.74" = "sha256-PjvuouwTsYfNKW5Vi5Ye7y+lL7SsWGBxCtBOOm2z14c=";
# copy the placeholder line with the correct toolchain name when adding a new toolchain
# That is,
# 1. Put the correct version name;
Expand All @@ -19,7 +17,7 @@ let
# error: hash mismatch in fixed-output derivation '/nix/store/XXXXX'
# specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
# got: sha256-Q9UgzzvxLi4x9aWUJTn+/5EXekC98ODRU1TwhUs9RnY=
"placeholder" = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
# "placeholder" = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
};
# rust-toolchain.toml -> { rustc, cargo, rust-analyzer, ... }
rustChannelFromToolchainFileOf = file:
Expand Down
2 changes: 1 addition & 1 deletion src/lib/crypto/kimchi_bindings/js/node_js/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ set -euo pipefail
if [[ -z "${PLONK_WASM_NODEJS-}" ]]; then
export RUSTFLAGS="-C target-feature=+atomics,+bulk-memory,+mutable-globals -C link-arg=--no-check-features -C link-arg=--max-memory=4294967296"
# The version should stay in line with the one in kimchi_bindings/wasm/rust-toolchain.toml
rustup run nightly-2023-09-01 wasm-pack build --target nodejs --out-dir ../js/node_js ../../wasm -- -Z build-std=panic_abort,std --features nodejs
rustup run 1.74 wasm-pack build --target nodejs --out-dir ../js/node_js ../../wasm -- -Z build-std=panic_abort,std --features nodejs
martyall marked this conversation as resolved.
Show resolved Hide resolved
else
cp "$PLONK_WASM_NODEJS"/* -R .
fi
2 changes: 1 addition & 1 deletion src/lib/crypto/kimchi_bindings/js/web/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ set -euo pipefail
if [[ -z "${PLONK_WASM_WEB-}" ]]; then
export RUSTFLAGS="-C target-feature=+atomics,+bulk-memory,+mutable-globals -C link-arg=--no-check-features -C link-arg=--max-memory=4294967296"
# The version should stay in line with the one in kimchi_bindings/wasm/rust-toolchain.toml
rustup run nightly-2023-09-01 wasm-pack build --target web --out-dir ../js/web ../../wasm -- -Z build-std=panic_abort,std
rustup run 1.74 wasm-pack build --target web --out-dir ../js/web ../../wasm -- -Z build-std=panic_abort,std
martyall marked this conversation as resolved.
Show resolved Hide resolved
else
cp "$PLONK_WASM_WEB"/* -R .
fi
2 changes: 1 addition & 1 deletion src/lib/crypto/kimchi_bindings/stubs/rust-toolchain.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,4 @@
# 4. figure out the hashes of the (now obsolete) docker images used in CI rules that are failing, grep for these hashes and replace them with the new hashes

[toolchain]
channel = "1.72"
channel = "1.74"
15 changes: 8 additions & 7 deletions src/lib/crypto/kimchi_bindings/stubs/src/caml/caml_pointer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
macro_rules! impl_caml_pointer {
($name: ident => $typ: ty) => {
#[derive(std::fmt::Debug, Clone, ::ocaml_gen::CustomType)]
pub struct $name(pub ::std::rc::Rc<$typ>);
pub struct $name(pub ::std::rc::Rc<std::cell::UnsafeCell<$typ>>);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note as it is important for the Caml binding: UnsafeCell is implemented as:

#[lang = "unsafe_cell"]
#[stable(feature = "rust1", since = "1.0.0")]
#[repr(transparent)]
#[rustc_pub_transparent]
pub struct UnsafeCell<T: ?Sized> {
    value: T,
}

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess we're adding a new indirection? We might lose some performances?

Copy link
Member Author

@martyall martyall Dec 13, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note as it is important for the Caml binding: UnsafeCell is implemented as:

The nightly tests were passing with this change, does this mean that that the ocaml bindings lib was able to resolve this on its own or is it just coincidence because maybe the runtime rep is the same

Copy link
Member Author

@martyall martyall Dec 15, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess we're adding a new indirection? We might lose some performances?

My intuition is that this UnsafeCell should be optimized away. We can discuss


impl $name {
extern "C" fn caml_pointer_finalize(v: ocaml::Raw) {
Expand Down Expand Up @@ -32,15 +32,17 @@ macro_rules! impl_caml_pointer {

impl $name {
pub fn create(x: $typ) -> $name {
$name(::std::rc::Rc::new(x))
$name(::std::rc::Rc::new(std::cell::UnsafeCell::new(x)))
}
}

impl ::std::ops::Deref for $name {
type Target = $typ;

fn deref(&self) -> &Self::Target {
&*self.0
unsafe {
martyall marked this conversation as resolved.
Show resolved Hide resolved
&*self.0.get()
}
}
}

Expand All @@ -49,16 +51,15 @@ macro_rules! impl_caml_pointer {
unsafe {
// Wholely unsafe, Batman!
// We would use [`get_mut_unchecked`] here, but it is nightly-only.
// Instead, we get coerce our constant pointer to a mutable
// pointer, in the knowledge that
// Instead, we use UnsafeCell in the knowledge that
// * all of our mutations called from OCaml are blocking, so
// we won't have multiple live mutable references live
// we won't have multiple live mutable references
// simultaneously, and
// * the underlying pointer is in the correct state to be
// mutable, since we can call [`get_mut_unchecked`] in
// nightly, or can call [`get_mut`] and unwrap if this is
// the only live reference.
&mut *(((&*self.0) as *const Self::Target) as *mut Self::Target)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

1.74 throws a compiler error here. There might be some other way to get it to stop complaining, but I think this is equivalent without the illegal cast

&mut *self.0.get()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note: implementation of get.

    #[inline(always)]
    #[stable(feature = "rust1", since = "1.0.0")]
    #[rustc_const_stable(feature = "const_unsafecell_get", since = "1.32.0")]
    #[rustc_never_returns_null_ptr]
    pub const fn get(&self) -> *mut T {
        // We can just cast the pointer from `UnsafeCell<T>` to `T` because of
        // #[repr(transparent)]. This exploits std's special status, there is
        // no guarantee for user code that this will work in future versions of the compiler!
        self as *const UnsafeCell<T> as *const T as *mut T
    }

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: cargo fmt reformats this line. It seems we do not run in the Mina CI clippy nor fmt.

Copy link
Member Author

@martyall martyall Dec 13, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note: implementation of get.

This makes me think someone was trying to get around using UnsafeCell when they wrote this the first time. I would hope that this would be optimized away

}
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_proof.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ pub fn caml_pasta_fp_plonk_proof_create(
.collect()
};

let witness: Vec<Vec<_>> = witness.iter().map(|x| (*x.0).clone()).collect();
let witness: Vec<Vec<_>> = witness.iter().map(|x| (**x).clone()).collect();
let witness: [Vec<_>; COLUMNS] = witness
.try_into()
.map_err(|_| ocaml::Error::Message("the witness should be a column of 15 vectors"))?;
Expand Down Expand Up @@ -140,7 +140,7 @@ pub fn caml_pasta_fp_plonk_proof_create_and_verify(
.collect()
};

let witness: Vec<Vec<_>> = witness.iter().map(|x| (*x.0).clone()).collect();
let witness: Vec<Vec<_>> = witness.iter().map(|x| (**x).clone()).collect();
let witness: [Vec<_>; COLUMNS] = witness
.try_into()
.map_err(|_| ocaml::Error::Message("the witness should be a column of 15 vectors"))?;
Expand Down
2 changes: 1 addition & 1 deletion src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_proof.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ pub fn caml_pasta_fq_plonk_proof_create(
.collect()
};

let witness: Vec<Vec<_>> = witness.iter().map(|x| (*x.0).clone()).collect();
let witness: Vec<Vec<_>> = witness.iter().map(|x| (**x).clone()).collect();
let witness: [Vec<_>; COLUMNS] = witness
.try_into()
.expect("the witness should be a column of 15 vectors");
Expand Down
2 changes: 1 addition & 1 deletion src/lib/crypto/kimchi_bindings/wasm/rust-toolchain.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@
# This should stay in line with the versions in
# - kimchi_bindings/js/node_js/build.sh
# - kimchi_bindings/js/web/build.sh
channel = "nightly-2023-09-01" # roughly matches 1.72
channel = "1.74"
1 change: 0 additions & 1 deletion src/lib/crypto/kimchi_bindings/wasm/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
#![feature(get_mut_unchecked)]
//! The Marlin_plonk_stubs crate exports some functionalities
//! and structures from the following the Rust crates to OCaml:
//!
Expand Down
2 changes: 1 addition & 1 deletion src/lib/crypto/proof-systems
Open in desktop
Submodule proof-systems updated 6 files
+1 −1 .github/workflows/benches.yml
+2 −2 .github/workflows/ci.yml
+1 −1 .github/workflows/gh-page.yml
+2 −4 Makefile
+0 −1 rust-toolchain
+3 −0 rust-toolchain.toml