Failure building wheel for tokenizers on M1

[mich889]

I'm getting the error
warning: variable does not need to be mutable --> tokenizers-lib/src/models/unigram/model.rs:265:21 | 265 | let mut target_node = &mut best_path_ends_at[key_pos]; | ----^^^^^^^^^^^ | | | help: remove thismut`
|
= note: #[warn(unused_mut)] on by default

  warning: variable does not need to be mutable
     --> tokenizers-lib/src/models/unigram/model.rs:282:21
      |
  282 |                 let mut target_node = &mut best_path_ends_at[starts_at + mblen];
      |                     ----^^^^^^^^^^^
      |                     |
      |                     help: remove this `mut`
  
  warning: variable does not need to be mutable
     --> tokenizers-lib/src/pre_tokenizers/byte_level.rs:200:59
      |
  200 |     encoding.process_tokens_with_offsets_mut(|(i, (token, mut offsets))| {
      |                                                           ----^^^^^^^
      |                                                           |
      |                                                           help: remove this `mut`
  
  error: casting `&T` to `&mut T` is undefined behavior, even if the reference is unused, consider instead using an `UnsafeCell`
     --> tokenizers-lib/src/models/bpe/trainer.rs:526:47
      |
  522 |                     let w = &words[*i] as *const _ as *mut _;
      |                             -------------------------------- casting happend here
  ...
  526 |                         let word: &mut Word = &mut (*w);
      |                                               ^^^^^^^^^
      |
      = note: for more information, visit <https://doc.rust-lang.org/book/ch15-05-interior-mutability.html>
      = note: `#[deny(invalid_reference_casting)]` on by default
  
  warning: `tokenizers` (lib) generated 3 warnings
  error: could not compile `tokenizers` (lib) due to 1 previous error; 3 warnings emitted
  
  Caused by:
    process didn't exit successfully: `rustc --crate-name tokenizers --edition=2018 tokenizers-lib/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C opt-level=3 -C embed-bitcode=no --cfg 'feature="cached-path"' --cfg 'feature="clap"' --cfg 'feature="cli"' --cfg 'feature="default"' --cfg 'feature="http"' --cfg 'feature="indicatif"' --cfg 'feature="progressbar"' --cfg 'feature="reqwest"' --check-cfg 'cfg(docsrs)' --check-cfg 'cfg(feature, values("cached-path", "clap", "cli", "default", "http", "indicatif", "progressbar", "reqwest"))' -C metadata=2b393932a93a4b93 -C extra-filename=-2b393932a93a4b93 --out-dir /private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps -C strip=debuginfo -L dependency=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps --extern aho_corasick=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libaho_corasick-abfd66d8fb48e753.rmeta --extern cached_path=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libcached_path-ff07d745fdf84349.rmeta --extern clap=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libclap-ccc9412e5a6e60ff.rmeta --extern derive_builder=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libderive_builder-162e50e631efd6c3.dylib --extern dirs=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libdirs-c7dfc1edc7d3d5a6.rmeta --extern esaxx_rs=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libesaxx_rs-1f0326f77855f96a.rmeta --extern indicatif=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libindicatif-14648757c526be56.rmeta --extern itertools=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libitertools-ca50b4fe4d804307.rmeta --extern lazy_static=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/liblazy_static-848418946463bcd0.rmeta --extern log=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/liblog-9f7dc97ea6c27b16.rmeta --extern macro_rules_attribute=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libmacro_rules_attribute-d9d4d04f0bcace75.rmeta --extern onig=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libonig-4210ea5403b4637f.rmeta --extern paste=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libpaste-7b5aa6905de1efae.dylib --extern rand=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/librand-07d5d0fb6479f0de.rmeta --extern rayon=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/librayon-bba08fd61352717b.rmeta --extern rayon_cond=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/librayon_cond-011045f9e0ba22f6.rmeta --extern regex=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libregex-b35a8552ba164435.rmeta --extern regex_syntax=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libregex_syntax-938f212692cb4838.rmeta --extern reqwest=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libreqwest-6028dd27fe0377c7.rmeta --extern serde=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libserde-57f61cd852669902.rmeta --extern serde_json=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libserde_json-e06fc0eda2f5e0ae.rmeta --extern spm_precompiled=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libspm_precompiled-cc81af876596fd1f.rmeta --extern thiserror=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libthiserror-6f6e3badb41d6aca.rmeta --extern unicode_normalization_alignments=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libunicode_normalization_alignments-3efb55fa7e6b2b68.rmeta --extern unicode_segmentation=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libunicode_segmentation-c2a9572c9052c980.rmeta --extern unicode_categories=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/deps/libunicode_categories-5be271239ddd500b.rmeta -L native=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/build/bzip2-sys-a9e0035c73169edb/out/lib -L native=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/build/zstd-sys-7906aac0e014c7db/out -L native=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/build/esaxx-rs-ac7d31710543b10a/out -L native=/private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-install-mb4zkt42/tokenizers_21bda27978314ff9987f1b5d6e028c48/target/release/build/onig_sys-fd605a3bc229564e/out (exit status: 1)
  error: cargo rustc --lib --message-format=json-render-diagnostics --manifest-path Cargo.toml --release -v --features pyo3/extension-module --crate-type cdylib -- -C 'link-args=-undefined dynamic_lookup -Wl,-install_name,@rpath/tokenizers.cpython-310-darwin.so' failed with code 101
  [end of output]

note: This error originates from a subprocess, and is likely not a problem with pip.
ERROR: Failed building wheel for tokenizers
Building wheel for clip (setup.py) ... done
Created wheel for clip: filename=clip-1.0-py3-none-any.whl size=1369491 sha256=ed48be402599ee663737cc2429dddbf2261bd8aa9e46778860c0fa0cd8505d5f
Stored in directory: /private/var/folders/4s/66z0r0jx3_v86ztl39l2c07m0000gn/T/pip-ephem-wheel-cache-uw3drn9_/wheels/da/2b/4c/d6691fa9597aac8bb85d2ac13b112deb897d5b50f5ad9a37e4
Successfully built clip
Failed to build tokenizers
ERROR: ERROR: Failed to build installable wheels for some pyproject.toml based projects (tokenizers)`

but when i run pip install tokenizers it seems to be working fine with no errors? My python version is Python 3.10.14, my rustc version is rustc 1.80.1

[laohuangsk8]

try tokenizers==0.19.1
and then transformers==4.44.2