Skip to content

feat: add MLX builds for LFM2.5 and privacy-filter models #1266

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
NorbertKlockiewicz merged 2 commits into from
Jun 22, 2026
Merged

feat: add MLX builds for LFM2.5 and privacy-filter models #1266

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dad59c8
feat: add MLX builds for LFM2.5 and privacy-filter models
NorbertKlockiewicz Jun 18, 2026
2b4dce6
refactor: address review on vision dtype conversion
NorbertKlockiewicz Jun 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions packages/react-native-executorch/common/runner/encoders/vision_encoder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include <rnexecutorch/Error.h>
#include <rnexecutorch/data_processing/ImageProcessing.h>
#include <runner/constants.h>
#include <runner/util.h>

#include <executorch/extension/tensor/tensor.h>
#include <opencv2/opencv.hpp>
Expand Down Expand Up @@ -124,8 +125,13 @@ Result<EValue> VisionEncoder::encode(const MultimodalInput &input) {
sizes.insert(sizes.begin(), 1);
}

auto vision_meta = ET_UNWRAP(module_->method_meta(kVisionEncoderMethod));
const auto want_dtype =
ET_UNWRAP(vision_meta.input_tensor_meta(0)).scalar_type();

auto image_tensor = ::executorch::extension::from_blob(
chw.data(), sizes, ::executorch::aten::ScalarType::Float);
image_tensor = ET_UNWRAP(convert_from_float(image_tensor, want_dtype));

auto result = ET_UNWRAP(module_->execute(kVisionEncoderMethod, image_tensor));
auto out_tensor = result[0].toTensor();
Expand Down
16 changes: 16 additions & 0 deletions packages/react-native-executorch/common/runner/multimodal_prefiller.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,22 @@ bool MultimodalPrefiller::get_enable_dynamic_shape() const {
for (size_t i = 0; i < visual_elems; ++i) {
dst_f[i] = static_cast<float>(src[i]);
}
} else if (vision_dtype == ::executorch::aten::ScalarType::Float &&
embeds_dtype == ::executorch::aten::ScalarType::BFloat16) {
// Hybrid VLM: fp32 vision encoder (e.g. XNNPACK) + bf16 decoder embeds.
const float *src = vision_tensor.const_data_ptr<float>();
auto *dst_b = reinterpret_cast<::executorch::aten::BFloat16 *>(dst);
for (size_t i = 0; i < visual_elems; ++i) {
dst_b[i] = ::executorch::aten::BFloat16(src[i]);
}
} else if (vision_dtype == ::executorch::aten::ScalarType::BFloat16 &&
embeds_dtype == ::executorch::aten::ScalarType::Float) {
const auto *src =
vision_tensor.const_data_ptr<::executorch::aten::BFloat16>();
auto *dst_f = reinterpret_cast<float *>(dst);
for (size_t i = 0; i < visual_elems; ++i) {
dst_f[i] = static_cast<float>(src[i]);
}
} else {
ET_CHECK_OR_RETURN_ERROR(
false, InvalidState,
Expand Down
32 changes: 32 additions & 0 deletions packages/react-native-executorch/common/runner/util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,38 @@ convert_to_bfloat16(const ::executorch::extension::TensorPtr &src_tensor) {
return bf16_tensor;
}

/**
* Convert a Float tensor to `dtype` (Float passthrough, BFloat16, or Half).
* Used to match an exported method's declared input dtype when preprocessing
* produces fp32 data. Returns InvalidArgument for unsupported targets.
*/
inline ::executorch::runtime::Result<::executorch::extension::TensorPtr>
convert_from_float(const ::executorch::extension::TensorPtr &src_tensor,
::executorch::aten::ScalarType dtype) {
using ::executorch::aten::ScalarType;
if (dtype == ScalarType::Float) {
return src_tensor;
}
if (dtype == ScalarType::BFloat16) {
return convert_to_bfloat16(src_tensor);
}
ET_CHECK_OR_RETURN_ERROR(src_tensor->scalar_type() == ScalarType::Float,
InvalidArgument,
"convert_from_float only supports a Float source");
ET_CHECK_OR_RETURN_ERROR(dtype == ScalarType::Half, InvalidArgument,
"Unsupported target dtype %hhd for pixel conversion",
static_cast<int8_t>(dtype));
const auto num_elements = static_cast<size_t>(src_tensor->numel());
const float *float_data = src_tensor->const_data_ptr<float>();
auto half_tensor =
::executorch::extension::empty_like(src_tensor, ScalarType::Half);
auto *half_data = half_tensor->mutable_data_ptr<::executorch::aten::Half>();
for (size_t i = 0; i < num_elements; ++i) {
half_data[i] = ::executorch::aten::Half(float_data[i]);
Comment thread
msluszniak marked this conversation as resolved.
Outdated
}
return half_tensor;
}

} // namespace llm
} // namespace extension
} // namespace executorch
Expand Down
75 changes: 66 additions & 9 deletions packages/react-native-executorch/src/constants/modelRegistry.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,64 @@ const GEMMA4_E2B_MM_VARIANTS = {
},
};

const LFM2_5_350M_VARIANTS = {
mlx: { base: { ...M.LFM2_5_350M, modelSource: M.LFM2_5_350M_MLX_MODEL } },
xnnpack: { base: M.LFM2_5_350M, quant: M.LFM2_5_350M_QUANTIZED },
};

const LFM2_5_1_2B_INSTRUCT_VARIANTS = {
mlx: {
base: {
...M.LFM2_5_1_2B_INSTRUCT,
modelSource: M.LFM2_5_1_2B_INSTRUCT_MLX_MODEL,
},
},
xnnpack: {
base: M.LFM2_5_1_2B_INSTRUCT,
quant: M.LFM2_5_1_2B_INSTRUCT_QUANTIZED,
},
};

const LFM2_5_VL_1_6B_VARIANTS = {
mlx: {
base: {
...M.LFM2_5_VL_1_6B_QUANTIZED,
modelSource: M.LFM2_5_VL_1_6B_MLX_MODEL,
},
},
xnnpack: { base: M.LFM2_5_VL_1_6B_QUANTIZED },
};

const LFM2_5_VL_450M_VARIANTS = {
mlx: {
base: {
...M.LFM2_5_VL_450M_QUANTIZED,
modelSource: M.LFM2_5_VL_450M_MLX_MODEL,
},
},
xnnpack: { base: M.LFM2_5_VL_450M_QUANTIZED },
};

const PRIVACY_FILTER_OPENAI_VARIANTS = {
mlx: {
base: {
...M.PRIVACY_FILTER_OPENAI,
modelSource: M.PRIVACY_FILTER_OPENAI_MLX_MODEL,
},
},
xnnpack: { base: M.PRIVACY_FILTER_OPENAI },
};

const PRIVACY_FILTER_NEMOTRON_VARIANTS = {
mlx: {
base: {
...M.PRIVACY_FILTER_NEMOTRON,
modelSource: M.PRIVACY_FILTER_NEMOTRON_MLX_MODEL,
},
},
xnnpack: { base: M.PRIVACY_FILTER_NEMOTRON },
};

const EFFICIENTNET_V2_S_VARIANTS = {
xnnpack: {
base: {
Expand Down Expand Up @@ -594,20 +652,19 @@ export const models = {
smollm2_1_360m: pair(M.SMOLLM2_1_360M, M.SMOLLM2_1_360M_QUANTIZED),
smollm2_1_1_7b: pair(M.SMOLLM2_1_1_7B, M.SMOLLM2_1_1_7B_QUANTIZED),
phi_4_mini_4b: pair(M.PHI_4_MINI_4B, M.PHI_4_MINI_4B_QUANTIZED),
lfm2_5_350m: pair(M.LFM2_5_350M, M.LFM2_5_350M_QUANTIZED),
lfm2_5_1_2b_instruct: pair(
M.LFM2_5_1_2B_INSTRUCT,
M.LFM2_5_1_2B_INSTRUCT_QUANTIZED
),
lfm2_5_350m: variant(LFM2_5_350M_VARIANTS, { ios: 'mlx' }),
lfm2_5_1_2b_instruct: variant(LFM2_5_1_2B_INSTRUCT_VARIANTS, {
ios: 'mlx',
}),
bielik_v3_0_1_5b: pair(M.BIELIK_V3_0_1_5B, M.BIELIK_V3_0_1_5B_QUANTIZED),
gemma4_e2b: variant(GEMMA4_E2B_VARIANTS, {
ios: 'mlx',
android: 'vulkan',
}),
// Multimodal LLMs — same hook/module as plain LLMs, listed here so users
// pick a model by capability ("LLM") rather than by modality.
lfm2_5_vl_1_6b: base(M.LFM2_5_VL_1_6B_QUANTIZED),
lfm2_5_vl_450m: base(M.LFM2_5_VL_450M_QUANTIZED),
lfm2_5_vl_1_6b: variant(LFM2_5_VL_1_6B_VARIANTS, { ios: 'mlx' }),
lfm2_5_vl_450m: variant(LFM2_5_VL_450M_VARIANTS, { ios: 'mlx' }),
gemma4_e2b_multimodal: variant(GEMMA4_E2B_MM_VARIANTS, {
ios: 'mlx',
android: 'vulkan',
Expand All @@ -617,8 +674,8 @@ export const models = {
efficientnet_v2_s: variant(EFFICIENTNET_V2_S_VARIANTS),
},
privacy_filter: {
openai: base(M.PRIVACY_FILTER_OPENAI),
nemotron: base(M.PRIVACY_FILTER_NEMOTRON),
openai: variant(PRIVACY_FILTER_OPENAI_VARIANTS, { ios: 'mlx' }),
nemotron: variant(PRIVACY_FILTER_NEMOTRON_VARIANTS, { ios: 'mlx' }),
},
object_detection: {
ssdlite_320_mobilenet_v3_large: variant(
Expand Down
8 changes: 8 additions & 0 deletions packages/react-native-executorch/src/constants/modelUrls.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,7 @@ export const PHI_4_MINI_4B_QUANTIZED = {
// LFM2.5-1.2B-Instruct
const LFM2_5_1_2B_INSTRUCT_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/1_2b/xnnpack/lfm_2_5_1_2b_xnnpack_fp16.pte`;
const LFM2_5_1_2B_INSTRUCT_QUANTIZED_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/1_2b/xnnpack/lfm_2_5_1_2b_xnnpack_8da4w.pte`;
export const LFM2_5_1_2B_INSTRUCT_MLX_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/1_2b/mlx/lfm_2_5_1_2b_mlx_int4.pte`;
const LFM2_5_1_2B_TOKENIZER = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/1_2b/tokenizer.json`;
const LFM2_5_1_2B_TOKENIZER_CONFIG = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/1_2b/tokenizer_config.json`;

Expand All @@ -476,6 +477,7 @@ export const LFM2_5_1_2B_INSTRUCT_QUANTIZED = {
// LFM2.5-350M
const LFM2_5_350M_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/350m/xnnpack/lfm_2_5_350m_xnnpack_fp16.pte`;
const LFM2_5_350M_QUANTIZED_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/350m/xnnpack/lfm_2_5_350m_xnnpack_8da4w.pte`;
export const LFM2_5_350M_MLX_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/350m/mlx/lfm_2_5_350m_mlx_int4.pte`;
const LFM2_5_350M_TOKENIZER = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/350m/tokenizer.json`;
const LFM2_5_350M_TOKENIZER_CONFIG = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/350m/tokenizer_config.json`;

Expand Down Expand Up @@ -527,11 +529,13 @@ export const BIELIK_V3_0_1_5B_QUANTIZED = {

// LFM2.5-VL-1.6B
const LFM2_VL_1_6B_QUANTIZED_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_1_6b/xnnpack/lfm_2_5_vl_1_6b_xnnpack_8da4w.pte`;
export const LFM2_5_VL_1_6B_MLX_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int4.pte`;
const LFM2_VL_1_6B_TOKENIZER = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_1_6b/tokenizer.json`;
const LFM2_VL_1_6B_TOKENIZER_CONFIG = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_1_6b/tokenizer_config.json`;

// LFM2.5-VL-450M
const LFM2_VL_450M_QUANTIZED_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_450m/xnnpack/lfm_2_5_vl_450m_xnnpack_8da4w.pte`;
export const LFM2_5_VL_450M_MLX_MODEL = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_450m/mlx/lfm_2_5_vl_450m_mlx_int4.pte`;
const LFM2_VL_450M_TOKENIZER = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_450m/tokenizer.json`;
const LFM2_VL_450M_TOKENIZER_CONFIG = `${URL_PREFIX}-lfm-2.5/${PREVIOUS_VERSION_TAG}/vl_450m/tokenizer_config.json`;

Expand Down Expand Up @@ -1281,6 +1285,8 @@ export const PRIVACY_FILTER_OPENAI = {
tokenizerSource: `${URL_PREFIX}-privacy-filter-openai/${PREVIOUS_VERSION_TAG}/tokenizer.json`,
} as const;

export const PRIVACY_FILTER_OPENAI_MLX_MODEL = `${URL_PREFIX}-privacy-filter-openai/${PREVIOUS_VERSION_TAG}/mlx/privacy_filter_openai_mlx_int4.pte`;

/**
* OpenMed/privacy-filter-nemotron — extended PII detector with 55 entity
* types (adds medical, financial, identity, technical, demographic, etc.).
Expand All @@ -1293,6 +1299,8 @@ export const PRIVACY_FILTER_NEMOTRON = {
tokenizerSource: `${URL_PREFIX}-privacy-filter-nemotron/${PREVIOUS_VERSION_TAG}/tokenizer.json`,
} as const;

export const PRIVACY_FILTER_NEMOTRON_MLX_MODEL = `${URL_PREFIX}-privacy-filter-nemotron/${PREVIOUS_VERSION_TAG}/mlx/privacy_filter_nemotron_mlx_int8.pte`;

// Image generation

/**
Expand Down
Loading