diff --git a/examples/flowgrpo_trainer/minimax_h3/README.md b/examples/flowgrpo_trainer/minimax_h3/README.md index 5b606c503..8162f155e 100644 --- a/examples/flowgrpo_trainer/minimax_h3/README.md +++ b/examples/flowgrpo_trainer/minimax_h3/README.md @@ -1,6 +1,6 @@ # MiniMax H3 T2VA and FL2VA FlowGRPO -Last updated: 08/28/2026 +Last updated: 08/31/2026 These recipes train `MiniMaxAI/MiniMax-H3` LoRA adapters with FlowGRPO for text-to-audio-video (T2VA) and first-frame image-to-audio-video (FL2VA) @@ -177,6 +177,26 @@ FL2VA reuses the same CPS FlowGRPO configuration as T2VA. The first-frame condition rows are held fixed across the reverse-SDE window and re-injected after every transition, so only the target video/audio rows are scored. +### NVIDIA GPU (V1 sync) + +The V1 recipes use TransferQueue and ReplayBuffer synchronously. They preserve +the corresponding V0 model, LoRA, reward, pipeline, and CPS FlowGRPO settings; +they only select `main_diffusion_v1`, `trainer.use_v1=true`, and +`trainer.v1.trainer_mode=sync`: + +```bash +# T2VA +bash examples/flowgrpo_trainer/minimax_h3/run_minimax_h3_t2va_lora_v1.sh + +# FL2VA +bash examples/flowgrpo_trainer/minimax_h3/run_minimax_h3_fl2va_lora_v1.sh +``` + +These recipes use the same GPU topology as their V0 counterparts. A +`separate_async` MiniMax H3 recipe is not provided: it requires dedicated +actor/rollout pools and checkpoint-engine synchronization rather than a +mechanical V1 entrypoint switch. + ### Ascend NPU ```bash diff --git a/examples/flowgrpo_trainer/minimax_h3/run_minimax_h3_fl2va_lora_v1.sh b/examples/flowgrpo_trainer/minimax_h3/run_minimax_h3_fl2va_lora_v1.sh new file mode 100755 index 000000000..8f6377484 --- /dev/null +++ b/examples/flowgrpo_trainer/minimax_h3/run_minimax_h3_fl2va_lora_v1.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# MiniMax H3 FL2VA (first-frame conditioned) LoRA FlowGRPO (V1 trainer: TransferQueue + ReplayBuffer + sync mode). +# +# This is the V1 sync counterpart of run_minimax_h3_fl2va_lora.sh. It uses +# `verl_omni.trainer.main_diffusion_v1` with `trainer.use_v1=true` and +# `trainer.v1.trainer_mode=sync`. TransferQueue is enabled by the runner; +# model, LoRA, reward, pipeline, and CPS FlowGRPO knobs match the V0 recipe. +set -x + +export WANDB_MODE=${WANDB_MODE:-online} + +WORKSPACE=${WORKSPACE:-$HOME} +MODEL_PATH=${MODEL_PATH:-$WORKSPACE/models/MiniMax-H3/FL2VA} +DATA_DIR=${DATA_DIR:-$WORKSPACE/data/fl2va/verl_omni} +CLAP_MODEL_PATH=${CLAP_MODEL_PATH:-laion/larger_clap_general} +IMAGEBIND_MODEL_PATH=${IMAGEBIND_MODEL_PATH:-.checkpoints/imagebind_huge.pth} +ACTOR_CONFIG_PATH=${ACTOR_CONFIG_PATH:-$(dirname "$MODEL_PATH")/transformer} +NUM_GPUS=${NUM_GPUS:-8} +ROLLOUT_TP=${ROLLOUT_TP:-2} +TEXT_ENCODER_TP=${TEXT_ENCODER_TP:-$ROLLOUT_TP} +REWARD_DEVICE=${REWARD_DEVICE:-cuda} +REWARD_NUM_WORKERS=${REWARD_NUM_WORKERS:-1} +TOTAL_TRAINING_STEPS=${TOTAL_TRAINING_STEPS:-100} +ASPECT_RATIO=${ASPECT_RATIO:-16:9} +HEIGHT=${HEIGHT:-256} +WIDTH=${WIDTH:-384} +NUM_FRAMES=${NUM_FRAMES:-121} +INFER_STEPS=${INFER_STEPS:-10} +VAL_HEIGHT=${VAL_HEIGHT:-512} +VAL_WIDTH=${VAL_WIDTH:-768} + +train_path=$DATA_DIR/train.parquet +test_path=$DATA_DIR/test.parquet + +script_path=$(readlink -f "$0") +script_name=$(basename "$script_path" .sh) +repo_root=$(dirname "$script_path") +while [[ "$repo_root" != "/" && ! -f "$repo_root/LICENSE" ]]; do + repo_root=$(dirname "$repo_root") +done +if [[ ! -f "$repo_root/LICENSE" ]]; then + echo "Unable to locate repo root from $script_path: no LICENSE found" >&2 + exit 1 +fi + +output_dir=${OUTPUT_DIR:-$repo_root/outputs/$script_name} +checkpoint_dir=$output_dir/checkpoints +run_timestamp=$(date +"%Y%m%d_%H%M") +log_file=$output_dir/logs/$run_timestamp/${NODE_RANK:-0}.log +mkdir -p "$checkpoint_dir" "$(dirname "$log_file")" +exec > >(tee -a "$log_file") 2>&1 + +h3_lora_targets="['to_q','to_k','to_v','to_out.0','ff.net.0.proj','ff.net.2']" + +python3 -m verl_omni.trainer.main_diffusion_v1 \ + data.train_files=$train_path \ + data.val_files=$test_path \ + data.train_batch_size=32 \ + data.val_max_samples=128 \ + data.max_prompt_length=1024 \ + data.truncation=error \ + data.seed=42 \ + algorithm.adv_estimator=flow_grpo \ + algorithm.global_std=True \ + actor_rollout_ref.model.path=$MODEL_PATH \ + actor_rollout_ref.model.config_path=$ACTOR_CONFIG_PATH \ + actor_rollout_ref.model.algorithm=flow_grpo \ + actor_rollout_ref.model.transformer_subfolder=transformer \ + actor_rollout_ref.model.attn_backend=_flash_3_varlen_hub \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.model.lora_rank=64 \ + actor_rollout_ref.model.lora_alpha=128 \ + actor_rollout_ref.model.target_modules="$h3_lora_targets" \ + actor_rollout_ref.model.fsdp_layer_prefixes="['transformer_blocks.','token_refiner.refiner_blocks.']" \ + '+actor_rollout_ref.actor.fsdp_config.wrap_policy.transformer_layer_cls_to_wrap=[MiniMaxH3TransformerBlock,MiniMaxH3TokenRefinerBlock]' \ + actor_rollout_ref.actor.strategy=fsdp2 \ + actor_rollout_ref.actor.optim.lr=3e-4 \ + actor_rollout_ref.actor.optim.weight_decay=0.0001 \ + actor_rollout_ref.actor.ppo_mini_batch_size=16 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.fsdp_config.model_dtype=bfloat16 \ + actor_rollout_ref.actor.fsdp_config.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm_omni \ + actor_rollout_ref.rollout.rollout_attn_backend=FLASH_ATTN_3_HUB \ + actor_rollout_ref.rollout.tensor_model_parallel_size=$ROLLOUT_TP \ + +actor_rollout_ref.rollout.engine_kwargs.vllm_omni.text_encoder_tp_size=$TEXT_ENCODER_TP \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.rollout.seed=42 \ + actor_rollout_ref.rollout.agent.num_workers=$((NUM_GPUS / ROLLOUT_TP)) \ + actor_rollout_ref.rollout.agent.default_agent_loop=minimax_h3_diffusion_single_turn_agent \ + actor_rollout_ref.rollout.max_prompt_embed_length=1024 \ + actor_rollout_ref.rollout.load_format=safetensors \ + actor_rollout_ref.rollout.calculate_log_probs=True \ + actor_rollout_ref.rollout.pipeline.height=$HEIGHT \ + actor_rollout_ref.rollout.pipeline.width=$WIDTH \ + actor_rollout_ref.rollout.pipeline.aspect_ratio=${ASPECT_RATIO} \ + actor_rollout_ref.rollout.pipeline.num_frames=$NUM_FRAMES \ + actor_rollout_ref.rollout.pipeline.frame_rate=24 \ + actor_rollout_ref.rollout.pipeline.num_inference_steps=$INFER_STEPS \ + actor_rollout_ref.rollout.pipeline.true_cfg_scale=1.0 \ + actor_rollout_ref.rollout.pipeline.max_sequence_length=1024 \ + +actor_rollout_ref.rollout.pipeline.output_type=np \ + actor_rollout_ref.rollout.val_kwargs.pipeline.height=$VAL_HEIGHT \ + actor_rollout_ref.rollout.val_kwargs.pipeline.width=$VAL_WIDTH \ + actor_rollout_ref.rollout.val_kwargs.pipeline.num_frames=$NUM_FRAMES \ + actor_rollout_ref.rollout.val_kwargs.pipeline.frame_rate=24.0 \ + actor_rollout_ref.rollout.val_kwargs.pipeline.num_inference_steps=40 \ + actor_rollout_ref.rollout.val_kwargs.pipeline.true_cfg_scale=1.0 \ + +actor_rollout_ref.rollout.val_kwargs.pipeline.output_type=pt \ + actor_rollout_ref.rollout.algo.noise_level=0.8 \ + actor_rollout_ref.rollout.algo.sde_type=cps \ + actor_rollout_ref.rollout.algo.sde_window_range='[0,8]' \ + actor_rollout_ref.rollout.algo.sde_window_size=3 \ + actor_rollout_ref.rollout.algo.sde_contiguous=True \ + actor_rollout_ref.rollout.algo.sde_window_seed=42 \ + reward.num_workers=$REWARD_NUM_WORKERS \ + reward.reward_model.enable=False \ + reward.custom_reward_function.path=pkg://verl_omni.reward_loop.reward_manager.multi \ + reward.custom_reward_function.name=_multi_reward_placeholder \ + reward.reward_manager.name=MultiVisualRewardManager \ + reward.reward_manager.module.path=pkg://verl_omni.reward_loop.reward_manager \ + "+reward.reward_functions.clap.path=$repo_root/verl_omni/utils/reward_score/clap.py" \ + '+reward.reward_functions.clap.name=compute_score' \ + '+reward.reward_functions.clap.weight=1.0' \ + '+reward.reward_functions.clap.required=true' \ + "+reward.reward_functions.clap.device=$REWARD_DEVICE:0" \ + "+reward.reward_functions.clap.model_name_or_path=$CLAP_MODEL_PATH" \ + "+reward.reward_functions.imagebind.path=$repo_root/verl_omni/utils/reward_score/imagebind.py" \ + '+reward.reward_functions.imagebind.name=compute_score' \ + '+reward.reward_functions.imagebind.weight=1.0' \ + '+reward.reward_functions.imagebind.required=true' \ + "+reward.reward_functions.imagebind.device=$REWARD_DEVICE:1" \ + "+reward.reward_functions.imagebind.model_name_or_path=$IMAGEBIND_MODEL_PATH" \ + '+reward.reward_functions.imagebind.mode=audio_video' \ + reward.aggregation=weighted_sum \ + trainer.logger='["console","wandb"]' \ + trainer.project_name=flow_grpo \ + trainer.experiment_name=minimax_h3_fl2va_lora_v1_gpu \ + trainer.default_local_dir=$checkpoint_dir \ + trainer.validation_data_dir=$output_dir/validation_data \ + trainer.rollout_data_dir=$output_dir/rollout_data \ + trainer.rollout_data_save_freq=10 \ + trainer.log_val_generations=8 \ + trainer.video_fps=24 \ + trainer.val_before_train=True \ + trainer.n_gpus_per_node=$NUM_GPUS \ + trainer.nnodes=1 \ + trainer.save_freq=10 \ + trainer.max_actor_ckpt_to_keep=1 \ + trainer.test_freq=10 \ + trainer.total_epochs=15 \ + trainer.total_training_steps=$TOTAL_TRAINING_STEPS \ + trainer.use_v1=true \ + trainer.v1.trainer_mode=sync "$@" diff --git a/examples/flowgrpo_trainer/minimax_h3/run_minimax_h3_t2va_lora_v1.sh b/examples/flowgrpo_trainer/minimax_h3/run_minimax_h3_t2va_lora_v1.sh new file mode 100644 index 000000000..d575eceed --- /dev/null +++ b/examples/flowgrpo_trainer/minimax_h3/run_minimax_h3_t2va_lora_v1.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# MiniMax H3 T2VA LoRA FlowGRPO (V1 trainer: TransferQueue + ReplayBuffer + sync mode). +# +# This is the V1 sync counterpart of run_minimax_h3_t2va_lora.sh. It uses +# `verl_omni.trainer.main_diffusion_v1` with `trainer.use_v1=true` and +# `trainer.v1.trainer_mode=sync`. TransferQueue is enabled by the runner; +# model, LoRA, reward, pipeline, and CPS FlowGRPO knobs match the V0 recipe. +set -x + +export WANDB_MODE=${WANDB_MODE:-online} + +WORKSPACE=${WORKSPACE:-$HOME} +MODEL_PATH=${MODEL_PATH:-$WORKSPACE/models/MiniMax-H3/FL2VA} +DATA_DIR=${DATA_DIR:-$WORKSPACE/data/vid_prompt/verl_omni} +CLAP_MODEL_PATH=${CLAP_MODEL_PATH:-laion/larger_clap_general} +IMAGEBIND_MODEL_PATH=${IMAGEBIND_MODEL_PATH:-.checkpoints/imagebind_huge.pth} +ACTOR_CONFIG_PATH=${ACTOR_CONFIG_PATH:-$(dirname "$MODEL_PATH")/transformer} +NUM_GPUS=${NUM_GPUS:-8} +ROLLOUT_TP=${ROLLOUT_TP:-2} +TEXT_ENCODER_TP=${TEXT_ENCODER_TP:-$ROLLOUT_TP} +REWARD_DEVICE=${REWARD_DEVICE:-cuda} +REWARD_NUM_WORKERS=${REWARD_NUM_WORKERS:-1} +TOTAL_TRAINING_STEPS=${TOTAL_TRAINING_STEPS:-100} +ASPECT_RATIO=${ASPECT_RATIO:-16:9} +HEIGHT=${HEIGHT:-256} +WIDTH=${WIDTH:-384} +NUM_FRAMES=${NUM_FRAMES:-121} +INFER_STEPS=${INFER_STEPS:-10} +VAL_HEIGHT=${VAL_HEIGHT:-512} +VAL_WIDTH=${VAL_WIDTH:-768} + +train_path=$DATA_DIR/train.parquet +test_path=$DATA_DIR/test.parquet + +script_path=$(readlink -f "$0") +script_name=$(basename "$script_path" .sh) +repo_root=$(dirname "$script_path") +while [[ "$repo_root" != "/" && ! -f "$repo_root/LICENSE" ]]; do + repo_root=$(dirname "$repo_root") +done +if [[ ! -f "$repo_root/LICENSE" ]]; then + echo "Unable to locate repo root from $script_path: no LICENSE found" >&2 + exit 1 +fi + +output_dir=${OUTPUT_DIR:-$repo_root/outputs/$script_name} +checkpoint_dir=$output_dir/checkpoints +run_timestamp=$(date +"%Y%m%d_%H%M") +log_file=$output_dir/logs/$run_timestamp/${NODE_RANK:-0}.log +mkdir -p "$checkpoint_dir" "$(dirname "$log_file")" +exec > >(tee -a "$log_file") 2>&1 + +h3_lora_targets="['to_q','to_k','to_v','to_out.0','ff.net.0.proj','ff.net.2']" + +python3 -m verl_omni.trainer.main_diffusion_v1 \ + data.train_files=$train_path \ + data.val_files=$test_path \ + data.train_batch_size=32 \ + data.val_max_samples=128 \ + data.max_prompt_length=1024 \ + data.truncation=error \ + data.seed=42 \ + algorithm.adv_estimator=flow_grpo \ + algorithm.global_std=True \ + actor_rollout_ref.model.path=$MODEL_PATH \ + actor_rollout_ref.model.config_path=$ACTOR_CONFIG_PATH \ + actor_rollout_ref.model.algorithm=flow_grpo \ + actor_rollout_ref.model.transformer_subfolder=transformer \ + actor_rollout_ref.model.attn_backend=_flash_3_varlen_hub \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.model.lora_rank=64 \ + actor_rollout_ref.model.lora_alpha=128 \ + actor_rollout_ref.model.target_modules="$h3_lora_targets" \ + actor_rollout_ref.model.fsdp_layer_prefixes="['transformer_blocks.','token_refiner.refiner_blocks.']" \ + '+actor_rollout_ref.actor.fsdp_config.wrap_policy.transformer_layer_cls_to_wrap=[MiniMaxH3TransformerBlock,MiniMaxH3TokenRefinerBlock]' \ + actor_rollout_ref.actor.strategy=fsdp2 \ + actor_rollout_ref.actor.optim.lr=3e-4 \ + actor_rollout_ref.actor.optim.weight_decay=0.0001 \ + actor_rollout_ref.actor.ppo_mini_batch_size=16 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.fsdp_config.model_dtype=bfloat16 \ + actor_rollout_ref.actor.fsdp_config.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm_omni \ + actor_rollout_ref.rollout.rollout_attn_backend=FLASH_ATTN_3_HUB \ + actor_rollout_ref.rollout.tensor_model_parallel_size=$ROLLOUT_TP \ + +actor_rollout_ref.rollout.engine_kwargs.vllm_omni.text_encoder_tp_size=$TEXT_ENCODER_TP \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.rollout.seed=42 \ + actor_rollout_ref.rollout.agent.num_workers=$((NUM_GPUS / ROLLOUT_TP)) \ + actor_rollout_ref.rollout.agent.default_agent_loop=minimax_h3_diffusion_single_turn_agent \ + actor_rollout_ref.rollout.max_prompt_embed_length=1024 \ + actor_rollout_ref.rollout.load_format=safetensors \ + actor_rollout_ref.rollout.calculate_log_probs=True \ + actor_rollout_ref.rollout.pipeline.height=$HEIGHT \ + actor_rollout_ref.rollout.pipeline.width=$WIDTH \ + actor_rollout_ref.rollout.pipeline.aspect_ratio=${ASPECT_RATIO} \ + actor_rollout_ref.rollout.pipeline.num_frames=$NUM_FRAMES \ + actor_rollout_ref.rollout.pipeline.frame_rate=24 \ + actor_rollout_ref.rollout.pipeline.num_inference_steps=$INFER_STEPS \ + actor_rollout_ref.rollout.pipeline.true_cfg_scale=1.0 \ + actor_rollout_ref.rollout.pipeline.max_sequence_length=1024 \ + +actor_rollout_ref.rollout.pipeline.output_type=np \ + actor_rollout_ref.rollout.val_kwargs.pipeline.height=$VAL_HEIGHT \ + actor_rollout_ref.rollout.val_kwargs.pipeline.width=$VAL_WIDTH \ + actor_rollout_ref.rollout.val_kwargs.pipeline.num_frames=$NUM_FRAMES \ + actor_rollout_ref.rollout.val_kwargs.pipeline.frame_rate=24.0 \ + actor_rollout_ref.rollout.val_kwargs.pipeline.num_inference_steps=40 \ + actor_rollout_ref.rollout.val_kwargs.pipeline.true_cfg_scale=1.0 \ + +actor_rollout_ref.rollout.val_kwargs.pipeline.output_type=pt \ + actor_rollout_ref.rollout.algo.noise_level=0.8 \ + actor_rollout_ref.rollout.algo.sde_type=cps \ + actor_rollout_ref.rollout.algo.sde_window_range='[0,8]' \ + actor_rollout_ref.rollout.algo.sde_window_size=3 \ + actor_rollout_ref.rollout.algo.sde_contiguous=True \ + actor_rollout_ref.rollout.algo.sde_window_seed=42 \ + reward.num_workers=$REWARD_NUM_WORKERS \ + reward.reward_model.enable=False \ + reward.custom_reward_function.path=pkg://verl_omni.reward_loop.reward_manager.multi \ + reward.custom_reward_function.name=_multi_reward_placeholder \ + reward.reward_manager.name=MultiVisualRewardManager \ + reward.reward_manager.module.path=pkg://verl_omni.reward_loop.reward_manager \ + "+reward.reward_functions.clap.path=$repo_root/verl_omni/utils/reward_score/clap.py" \ + '+reward.reward_functions.clap.name=compute_score' \ + '+reward.reward_functions.clap.weight=1.0' \ + '+reward.reward_functions.clap.required=true' \ + "+reward.reward_functions.clap.device=$REWARD_DEVICE:0" \ + "+reward.reward_functions.clap.model_name_or_path=$CLAP_MODEL_PATH" \ + "+reward.reward_functions.imagebind.path=$repo_root/verl_omni/utils/reward_score/imagebind.py" \ + '+reward.reward_functions.imagebind.name=compute_score' \ + '+reward.reward_functions.imagebind.weight=1.0' \ + '+reward.reward_functions.imagebind.required=true' \ + "+reward.reward_functions.imagebind.device=$REWARD_DEVICE:1" \ + "+reward.reward_functions.imagebind.model_name_or_path=$IMAGEBIND_MODEL_PATH" \ + '+reward.reward_functions.imagebind.mode=audio_video' \ + reward.aggregation=weighted_sum \ + trainer.logger='["console","wandb"]' \ + trainer.project_name=flow_grpo \ + trainer.experiment_name=minimax_h3_t2va_lora_v1_gpu \ + trainer.default_local_dir=$checkpoint_dir \ + trainer.validation_data_dir=$output_dir/validation_data \ + trainer.rollout_data_dir=$output_dir/rollout_data \ + trainer.rollout_data_save_freq=10 \ + trainer.log_val_generations=8 \ + trainer.video_fps=24 \ + trainer.val_before_train=True \ + trainer.n_gpus_per_node=$NUM_GPUS \ + trainer.nnodes=1 \ + trainer.save_freq=10 \ + trainer.max_actor_ckpt_to_keep=1 \ + trainer.test_freq=10 \ + trainer.total_epochs=15 \ + trainer.total_training_steps=$TOTAL_TRAINING_STEPS \ + trainer.use_v1=true \ + trainer.v1.trainer_mode=sync "$@"