spec.yaml

$schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
name: translation_finetune
version: 0.0.72
type: command

is_deterministic: true

display_name: Translation Finetune
description: Component to finetune Hugging Face pretrained models for translation task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/translation_finetune) to learn more.

environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/87

code: ../../../src/finetune

distribution:
  type: pytorch

inputs:
  # Lora parameters
  # LoRA reduces the number of trainable parameters by learning pairs of rank-decompostion matrices while freezing the original weights. This vastly reduces the storage requirement for large language models adapted to specific tasks and enables efficient task-switching during deployment all without introducing inference latency. LoRA also outperforms several other adaptation methods including adapter, prefix-tuning, and fine-tuning. Currently, LoRA is supported for gpt2, bert, roberta, deberta, distilbert, t5, bart, mbart and camembert model families
  apply_lora:
    type: string
    enum:
    - "true"
    - "false"
    default: "false"
    optional: true
    description: If "true" enables lora.

  merge_lora_weights:
    type: string
    enum:
    - "true"
    - "false"
    default: "true"
    optional: true
    description: If "true", the lora weights are merged with the base Hugging Face model weights before saving.

  lora_alpha:
    type: integer
    default: 128
    optional: true
    description: alpha attention parameter for lora.

  lora_r:
    type: integer
    default: 8
    optional: true
    description: lora dimension

  lora_dropout:
    type: number
    default: 0.0
    optional: true
    description: lora dropout value

  # Training parameters
  num_train_epochs:
    type: integer
    min: 1
    default: 1
    optional: true
    description: Number of epochs to run for finetune.

  max_steps:
    type: integer
    default: -1
    optional: true
    description: If set to a positive number, the total number of training steps to perform. Overrides 'epochs'. In case of using a finite iterable dataset the training may stop before reaching the set number of steps when all data is exhausted.

  per_device_train_batch_size:
    type: integer
    min: 1
    default: 1
    optional: true
    description: Per gpu batch size used for training. The effective training batch size is _per_device_train_batch_size_ * _num_gpus_ * _num_nodes_

  per_device_eval_batch_size:
    type: integer
    min: 1
    default: 1
    optional: true
    description: Per gpu batch size used for validation. The default value is 1. The effective validation batch size is _per_device_eval_batch_size_ * _num_gpus_ * _num_nodes_.

  auto_find_batch_size:
    type: string
    enum:
    - "true"
    - "false"
    default: "false"
    optional: true
    description: If set to "true" and if the provided 'per_device_train_batch_size' goes into Out Of Memory (OOM) auto_find_batch_size will find the correct batch size by iteratively reducing batch size by a factor of 2 till the OOM is fixed

  optim:
    type: string
    default: adamw_hf
    optional: true
    enum:
    - adamw_hf
    - adamw_torch
      # - adamw_apex_fused
    - adafactor
    description: Optimizer to be used while training

  learning_rate:
    type: number
    default: 0.00002
    optional: true
    description: Start learning rate used for training.

  warmup_steps:
    type: integer
    default: 0
    optional: true
    description: Number of steps for the learning rate scheduler warmup phase.

  weight_decay:
    type: number
    default: 0.0
    optional: true
    description: Weight decay to apply (if not zero) to all layers except all bias and LayerNorm weights in AdamW optimizer

  adam_beta1:
    type: number
    default: 0.9
    optional: true
    description: beta1 hyperparameter for the AdamW optimizer

  adam_beta2:
    type: number
    default: 0.999
    optional: true
    description: beta2 hyperparameter for the AdamW optimizer

  adam_epsilon:
    type: number
    default: 1e-8
    optional: true
    description: epsilon hyperparameter for the AdamW optimizer

  gradient_accumulation_steps:
    type: integer
    default: 1
    optional: true
    description: Number of updates steps to accumulate the gradients for, before performing a backward/update pass

  eval_accumulation_steps:
    type: integer
    default: -1
    optional: true
    description: Number of predictions steps to accumulate before moving the tensors to the CPU, will be passed as None if set to -1

  lr_scheduler_type:
    type: string
    default: linear
    optional: true
    enum:
    - linear
    - cosine
    - cosine_with_restarts
    - polynomial
    - constant
    - constant_with_warmup
    description: learning rate scheduler to use.

  precision:
    type: string
    enum:
    - "32"
    - "16"
    default: "32"
    optional: true
    description: Apply mixed precision training. This can reduce memory footprint by performing operations in half-precision.

  seed:
    type: integer
    default: 42
    optional: true
    description: Random seed that will be set at the beginning of training

  enable_full_determinism:
    type: string
    enum:
    - "true"
    - "false"
    default: "false"
    optional: true
    description: Ensure reproducible behavior during distributed training. Check this link https://pytorch.org/docs/stable/notes/randomness.html for more details.

  dataloader_num_workers:
    type: integer
    default: 0
    optional: true
    description: Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process.

  ignore_mismatched_sizes:
    type: string
    enum:
    - "true"
    - "false"
    default: "true"
    optional: true
    description: Not setting this flag will raise an error if some of the weights from the checkpoint do not have the same size as the weights of the model.

  max_grad_norm:
    type: number
    default: 1.0
    optional: true
    description: Maximum gradient norm (for gradient clipping)

  evaluation_strategy:
    type: string
    default: epoch
    optional: true
    enum:
    - epoch
    - steps
    description: The evaluation strategy to adopt during training. If set to "steps", either the `evaluation_steps_interval` or `eval_steps` needs to be specified, which helps to determine the step at which the model evaluation needs to be computed else evaluation happens at end of each epoch.

  evaluation_steps_interval:
    type: number
    default: 0.0
    optional: true
    description: The evaluation steps in fraction of an epoch steps to adopt during training. Overwrites eval_steps if not 0.

  eval_steps:
    type: integer
    default: 500
    optional: true
    description: Number of update steps between two evals if evaluation_strategy='steps'

  logging_strategy:
    type: string
    default: steps
    optional: true
    enum:
    - epoch
    - steps
    description: The logging strategy to adopt during training. If set to "steps", the `logging_steps` will decide the frequency of logging else logging happens at the end of epoch.

  logging_steps:
    type: integer
    default: 10
    optional: true
    description: Number of update steps between two logs if logging_strategy='steps'

  metric_for_best_model:
    type: string
    default: loss
    optional: true
    enum:
    - loss
    - bleu
    description: metric to use to compare two different model checkpoints

  resume_from_checkpoint:
    type: string
    default: "false"
    optional: true
    enum:
    - "true"
    - "false"
    description: If set to "true", resumes the training from last saved checkpoint. Along with loading the saved weights, saved optimizer, scheduler and random states will be loaded if exist. The default value is "false"

  save_total_limit:
    type: integer
    default: -1
    optional: true
    description: If a positive value is passed, it will limit the total number of checkpoints saved. The value of -1 saves all the checkpoints, otherwise if the number of checkpoints exceed the _save_total_limit_, the older checkpoints gets deleted.

  # Early Stopping Parameters
  apply_early_stopping:
    type: string
    default: "false"
    optional: true
    enum:
    - "true"
    - "false"
    description: If set to "true", early stopping is enabled.

  early_stopping_patience:
    type: integer
    default: 1
    optional: true
    description: Stop training when the metric specified through _metric_for_best_model_ worsens for _early_stopping_patience_ evaluation calls.This value is only valid if _apply_early_stopping_ is set to true.

  early_stopping_threshold:
    type: number
    default: 0.0
    optional: true
    description: Denotes how much the specified metric must improve to satisfy early stopping conditions. This value is only valid if _apply_early_stopping_ is set to true.

  # Deepspeed Parameters
  # Deepspeed config is a JSON file that can be used to configure optimizer, scheduler, batch size and other training related parameters. A default deepspeed config is used when _apply_deepspeed_ is set to `true`. Alternatively, you can pass your custom deepspeed config. Please follow the [deepspeed docs](https://www.deepspeed.ai/docs/config-json/) to create the custom config.
  # Please note that to enable deepspeed, `apply_deepspeed` must be set to true, only passing the `deepspeed input` will not suffice
  apply_deepspeed:
    type: string
    enum:
    - "true"
    - "false"
    default: "false"
    optional: true
    description: If set to true, will enable deepspeed for training

  deepspeed:
    type: uri_file
    optional: true
    description: Deepspeed config to be used for finetuning
    mode: rw_mount

  deepspeed_stage:
    type: string
    optional: true
    default: "2"
    enum:
    - "2"
    - "3"
    description: This parameter configures which DEFAULT deepspeed config to be used - stage2 or stage3. The default choice is stage2. Note that, this parameter is ONLY applicable when user doesn't pass any config information via deepspeed port.

  # ORT Parameters
  # ONNX Runtime is a cross-platform machine-learning model accelerator, with a flexible interface to integrate hardware-specific libraries.
  apply_ort:
    type: string
    enum:
    - "true"
    - "false"
    default: "false"
    optional: true
    description: If set to true, will use the ONNXRunTime training

  # Data and Model inputs
  preprocess_output:
    type: uri_folder
    optional: false
    description: output folder of preprocess component containing encoded train, valid and test data. The tokenizer is also saved as part of preprocess output
    mode: rw_mount

  model_selector_output:
    type: uri_folder
    optional: false
    description: output folder of model import component containing model artifacts and a metadata file.
    mode: rw_mount

outputs:
  pytorch_model_folder:
    type: uri_folder
    description: output folder containing _best_ model as defined by _metric_for_best_model_. Along with the best model, output folder contains checkpoints saved after every evaluation which is defined by the _evaluation_strategy_. Each checkpoint contains the model weight(s), config, tokenizer, optimzer, scheduler and random number states.
    mode: rw_mount

command: >-
  python finetune.py $[[--apply_lora '${{inputs.apply_lora}}']] $[[--merge_lora_weights '${{inputs.merge_lora_weights}}']] $[[--lora_alpha '${{inputs.lora_alpha}}']] $[[--lora_r '${{inputs.lora_r}}']] $[[--lora_dropout '${{inputs.lora_dropout}}']] $[[--num_train_epochs '${{inputs.num_train_epochs}}']] $[[--max_steps '${{inputs.max_steps}}']] $[[--per_device_train_batch_size '${{inputs.per_device_train_batch_size}}']] $[[--per_device_eval_batch_size '${{inputs.per_device_eval_batch_size}}']] $[[--auto_find_batch_size '${{inputs.auto_find_batch_size}}']] $[[--optim '${{inputs.optim}}']] $[[--learning_rate '${{inputs.learning_rate}}']] $[[--warmup_steps '${{inputs.warmup_steps}}']] $[[--weight_decay '${{inputs.weight_decay}}']] $[[--adam_beta1 '${{inputs.adam_beta1}}']] $[[--adam_beta2 '${{inputs.adam_beta2}}']] $[[--adam_epsilon '${{inputs.adam_epsilon}}']] $[[--gradient_accumulation_steps '${{inputs.gradient_accumulation_steps}}']] $[[--eval_accumulation_steps '${{inputs.eval_accumulation_steps}}']] $[[--lr_scheduler_type '${{inputs.lr_scheduler_type}}']] $[[--precision '${{inputs.precision}}']] $[[--seed '${{inputs.seed}}']] $[[--enable_full_determinism '${{inputs.enable_full_determinism}}']] $[[--dataloader_num_workers '${{inputs.dataloader_num_workers}}']] $[[--ignore_mismatched_sizes '${{inputs.ignore_mismatched_sizes}}']] $[[--max_grad_norm '${{inputs.max_grad_norm}}']] $[[--evaluation_strategy '${{inputs.evaluation_strategy}}']] $[[--evaluation_steps_interval '${{inputs.evaluation_steps_interval}}']] $[[--eval_steps '${{inputs.eval_steps}}']] $[[--logging_strategy '${{inputs.logging_strategy}}']] $[[--logging_steps '${{inputs.logging_steps}}']] $[[--metric_for_best_model '${{inputs.metric_for_best_model}}']] $[[--resume_from_checkpoint '${{inputs.resume_from_checkpoint}}']] $[[--save_total_limit '${{inputs.save_total_limit}}']] $[[--apply_early_stopping '${{inputs.apply_early_stopping}}']] $[[--early_stopping_patience '${{inputs.early_stopping_patience}}']] $[[--early_stopping_threshold '${{inputs.early_stopping_threshold}}']] $[[--apply_ort '${{inputs.apply_ort}}']] $[[--apply_deepspeed '${{inputs.apply_deepspeed}}']] $[[--deepspeed '${{inputs.deepspeed}}']] $[[--deepspeed_stage '${{inputs.deepspeed_stage}}']] --model_selector_output '${{inputs.model_selector_output}}' --preprocess_output '${{inputs.preprocess_output}}' --pytorch_model_folder '${{outputs.pytorch_model_folder}}'