Add GR00T-H

Signed-off-by: Nigel Nelson <nigeln@nvidia.com>

Author: NigelNelson

README.md

@@ -1,18 +1,30 @@
 <div align="center">
 
-  <img src="media/header_compress.png" width="800" alt="NVIDIA Isaac GR00T N1.6 Header">
+  <img src="media/gr00t-h-header.png" width="800" alt="NVIDIA GR00T-H Header">
 
   <!-- --- -->
 
   <p style="font-size: 1.2em;">
-    <a href="https://developer.nvidia.com/isaac/gr00t"><strong>Website</strong></a> | 
-    <a href="https://huggingface.co/nvidia/GR00T-N1.6-3B"><strong>Model</strong></a> |
-    <a href="https://huggingface.co/datasets/nvidia/PhysicalAI-Robotics-GR00T-X-Embodiment-Sim"><strong>Dataset</strong></a> |
-    <a href="https://arxiv.org/abs/2503.14734"><strong>Paper</strong></a> |
-    <a href="https://research.nvidia.com/labs/gear/gr00t-n1_6/"><strong>Research Blog</strong></a>
+    <a href="https://huggingface.co/nvidia/GR00T-H"><strong>Model</strong></a> |
+    <a href="https://huggingface.co/datasets/nvidia/PhysicalAI-Robotics-Open-H-Embodiment"><strong>Open-H Dataset</strong></a> |
   </p>
 </div>
 
+> GR00T-H is a variant of GR00T N1.6 post-trained on the [Open-H dataset](open_h/README.md) for healthcare robotics autonomy. This repository is intended for developers working with the Open-H dataset or building on GR00T for healthcare applications. For general robotics use cases, the upstream [Isaac-GR00T](https://github.com/NVIDIA/Isaac-GR00T) project is a better starting point.
+
+## GR00T-H
+
+The primary differences from upstream Isaac-GR00T live in [`open_h/`](open_h/README.md):
+- Per-embodiment modality configs converting 16 healthcare robot datasets to a common action representation
+- Multi-embodiment training config and dataset preparation tooling
+- Extensions to the data pipeline (clutch-aware filtering, motion scaling, step filtering)
+
+*The rest of this README documents the base GR00T N1.6 model, which GR00T-H builds on.*
+
+---
+
+---
+
 ## NVIDIA Isaac GR00T
 
 <div align="center">

gr00t/configs/base_config.py

@@ -48,6 +48,11 @@ def load(self, path: Path):
     def load_dict(self, data: dict):
         if "model" in data:
             self.model = self.model.__class__(**data["model"])
+            # YAML loads sequences as lists, but some model fields expect tuples.
+            for attr in ("image_crop_size", "image_target_size"):
+                val = getattr(self.model, attr, None)
+                if isinstance(val, list):
+                    setattr(self.model, attr, tuple(val))
         if "data" in data:
             self.data = DataConfig(**data["data"])
             # Ensure nested datasets are converted to dataclass instances

gr00t/configs/data/data_config.py

@@ -31,6 +31,11 @@ class SingleDatasetConfig:
     # If not provided, falls back to dataset_paths for evaluation
     val_dataset_path: Optional[str] = None
 
+    # Optional split-based episode filtering (from meta/info.json splits)
+    # If include_splits is set, only those splits are used. Then exclude_splits is applied.
+    exclude_splits: List[str] | None = None
+    include_splits: List[str] | None = None
+
 
 @dataclass
 class DataConfig:

gr00t/configs/finetune_config.py

@@ -31,6 +31,18 @@ class FinetuneConfig:
     If None, use the pre-registered modality config in `gr00t/configs/data/embodiment_configs.py`. 
     """
 
+    include_splits: list[str] | None = None
+    """
+    Optional allowlist of dataset splits (from meta/info.json) to include.
+    If provided, only these splits are used for training and stats.
+    """
+
+    exclude_splits: list[str] | None = None
+    """
+    Optional denylist of dataset splits (from meta/info.json) to exclude.
+    Applied after include_splits (if set). Useful for skipping fail episodes.
+    """
+
     # --- Model Tuning Flags ---
     tune_llm: bool = False
     """If True, fine-tune the language model (LLM) backbone during training."""
@@ -49,6 +61,12 @@ class FinetuneConfig:
     Dropout probability applied to state inputs for regularization during training.
     """
 
+    state_dropout_prob_per_embodiment: dict[str, float] | None = None
+    """
+    Per-embodiment state dropout overrides. Keys are embodiment tag strings,
+    values are dropout probabilities in [0.0, 1.0].
+    """
+
     # --- Data Augmentation ---
     random_rotation_angle: int | None = None
     """Maximum rotation angle (in degrees) for random rotation augmentation of input images."""
@@ -85,6 +103,23 @@ class FinetuneConfig:
     If None, no extra augmentations are applied.
     """
 
+    image_size: tuple[int, int] | None = None
+    """
+    Intermediate padded size as (height, width) for resize with padding.
+    Images are resized (preserving aspect ratio) and padded to this size.
+    Should be >= image_crop_size to allow for cropping augmentation.
+    Example: (540, 720) to pad all images to 540×720 before cropping.
+    If None, uses shortest_image_edge with aspect-preserving crops (default albumentations).
+    """
+
+    image_crop_size: tuple[int, int] | None = None
+    """
+    Final output size as (height, width) after cropping. Only used when image_size is set.
+    This is the resolution the model actually sees.
+    Example: (480, 640) means final images are 480×640.
+    If None, defaults to image_size (no cropping, padded size is final size).
+    """
+
     # --- Training Configuration ---
     global_batch_size: int = 64
     """Total effective batch size across all GPUs and accumulation steps."""
@@ -134,3 +169,24 @@ class FinetuneConfig:
 
     num_shards_per_epoch: int = int(1e5)
     """Number of shards to use for the dataset. reduce this number if vram is limited."""
+
+    # --- Statistics Calculation Flags ---
+    calculate_norm_stats: bool = False
+    """
+    If True, only calculate normalization statistics and exit without training.
+    Uses skip_video=True for fast iteration. Statistics will be saved to
+    the norm_stats_output_path or the dataset's meta directory.
+    """
+
+    norm_stats_output_path: str | None = None
+    """
+    Path to save calculated normalization statistics. If None, saves to
+    the dataset's meta/percentile_stats.json file.
+    """
+
+    stats_num_workers: int | None = None
+    """
+    Number of parallel workers for statistics calculation. If None, uses CPU count.
+    Set to 1 to disable parallelism. Only used when calculate_norm_stats=True.
+    """
+

gr00t/configs/model/gr00t_n1d6.py

@@ -98,6 +98,7 @@ class Gr00tN1d6Config(PretrainedConfig):
 
     # State Augmentation parameters
     state_dropout_prob: float = 0.0  # State dropout probability
+    state_dropout_prob_per_embodiment: dict[str, float] | None = None  # Per-embodiment overrides
     state_additive_noise_scale: float = 0.0  # Scale for additive Gaussian noise on state features
 
     # Multi-embodiment parameters