Merge branch 'main' into 3d_ddpm

Author: ericspod

3d_classification/densenet_training_array.ipynb

@@ -200,7 +200,7 @@
    ],
    "source": [
     "if not os.path.isfile(images[0]):\n",
-    "    resource = \"https://drive.google.com/file/d/1f5odq9smadgeJmDeyEy_UOjEtE_pkKc0/view?usp=sharing\"\n",
+    "    resource = \"https://developer.download.nvidia.com/assets/Clara/monai/tutorials/IXI-T1.tar\"\n",
     "    md5 = \"34901a0593b41dd19c1a1f746eac2d58\"\n",
     "\n",
     "    dataset_dir = os.path.join(root_dir, \"ixi\")\n",

3d_registration/learn2reg_nlst_paired_lung_ct.ipynb

@@ -205,7 +205,7 @@
    "outputs": [],
    "source": [
     "if not os.path.isfile(images[0]):\n",
-    "    resource = \"https://drive.google.com/file/d/1f5odq9smadgeJmDeyEy_UOjEtE_pkKc0/view?usp=sharing\"\n",
+    "    resource = \"https://developer.download.nvidia.com/assets/Clara/monai/tutorials/IXI-T1.tar\"\n",
     "    md5 = \"34901a0593b41dd19c1a1f746eac2d58\"\n",
     "\n",
     "    dataset_dir = os.path.join(root_dir, \"ixi\")\n",

3d_regression/densenet_training_array.ipynb

@@ -45,7 +45,7 @@
     "\n",
     "https://www.synapse.org/#!Synapse:syn27046444/wiki/616992\n",
     "\n",
-    "The JSON file containing training and validation sets (internal split) needs to be downloaded from this [link](https://drive.google.com/file/d/1i-BXYe-wZ8R9Vp3GXoajGyqaJ65Jybg1/view?usp=sharing) and placed in the same folder as the dataset. As discussed in the following, this tutorial uses fold 1 for training a Swin UNETR model on the BraTS 21 challenge.\n",
+    "The JSON file containing training and validation sets (internal split) needs to be downloaded from this [link](https://developer.download.nvidia.com/assets/Clara/monai/tutorials/brats21_folds.json) and placed in the same folder as the dataset. As discussed in the following, this tutorial uses fold 1 for training a Swin UNETR model on the BraTS 21 challenge.\n",
     "\n",
     "### Tumor Characteristics\n",
     "\n",
@@ -114,7 +114,7 @@
     "  \"TrainingData/BraTS2021_01146/BraTS2021_01146_flair.nii.gz\"\n",
     "  \n",
     "\n",
-    "- Download the json file from this [link](https://drive.google.com/file/d/1i-BXYe-wZ8R9Vp3GXoajGyqaJ65Jybg1/view?usp=sharing) and placed in the same folder as the dataset.\n"
+    "- Download the json file from this [link](https://developer.download.nvidia.com/assets/Clara/monai/tutorials/brats21_folds.json) and placed in the same folder as the dataset.\n"
    ]
   },
   {

3d_segmentation/swin_unetr_brats21_segmentation_3d.ipynb

@@ -33,7 +33,7 @@
     "\n",
     "For this tutorial, the dataset needs to be downloaded from: https://www.synapse.org/#!Synapse:syn3193805/wiki/217752. More details are provided in the \"Download dataset\" section below.\n",
     "\n",
-    "In addition, the json file for data splits needs to be downloaded from this [link](https://drive.google.com/file/d/1qcGh41p-rI3H_sQ0JwOAhNiQSXriQqGi/view?usp=sharing). Once downloaded, place the json file in the same folder as the dataset. \n",
+    "In addition, the json file for data splits needs to be downloaded from this [link](https://developer.download.nvidia.com/assets/Clara/monai/tutorials/swin_unetr_btcv_dataset_0.json). Once downloaded, place the json file in the same folder as the dataset. \n",
     "\n",
     "For BTCV dataset, under Institutional Review Board (IRB) supervision, 50 abdomen CT scans of were randomly selected from a combination of an ongoing colorectal cancer chemotherapy trial, and a retrospective ventral hernia study. The 50 scans were captured during portal venous contrast phase with variable volume sizes (512 x 512 x 85 - 512 x 512 x 198) and field of views (approx. 280 x 280 x 280 mm3 - 500 x 500 x 650 mm3). The in-plane resolution varies from 0.54 x 0.54 mm2 to 0.98 x 0.98 mm2, while the slice thickness ranges from 2.5 mm to 5.0 mm. \n",
     "\n",
@@ -98,8 +98,6 @@
     "\n",
     "We use weights from self-supervised pre-training of Swin UNETR encoder (3D Swin Tranformer) on a cohort of 5050 CT scans from publicly available datasets. The encoder is pre-trained using reconstructin, rotation prediction and contrastive learning pre-text tasks as shown below. For more details, please refer to [1] (CVPR paper) and see this [repository](https://github.com/Project-MONAI/research-contributions/tree/main/SwinUNETR/Pretrain). \n",
     "\n",
-    "![image](https://lh3.googleusercontent.com/pw/AM-JKLVLgduGZ9naCSasWg09U665NBdd3UD4eLTy15wJiwbmKLS_p5WSZ2MBcRePEJO2tv9X3TkC52MsbnomuPy5JT3vSVeCji1MOEuAzcsxily88TdbHuAt6PzccefwKupbXyOCumK5hzz5Ul38kZnlEQ84=w397-h410-no?authuser=2)\n",
-    "\n",
     "Please download the pre-trained weights from this [link](https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/model_swinvit.pt) and place it in the root directory of this tutorial. \n",
     "\n",
     "If training from scratch is desired, please skip the step for initializing from pre-trained weights. "
@@ -321,7 +319,7 @@
     "\n",
     "3. Make a JSON file to define train/val split and other relevant parameters. Place the JSON file at `./data/dataset_0.json`.\n",
     "\n",
-    "    You can download an example of the JSON file [here](https://drive.google.com/file/d/1qcGh41p-rI3H_sQ0JwOAhNiQSXriQqGi/view?usp=sharing), or, equivalently, use the following `wget` command. If you would like to use this directly, please move it into the `./data` folder."
+    "    You can download an example of the JSON file [here](https://developer.download.nvidia.com/assets/Clara/monai/tutorials/swin_unetr_btcv_dataset_0.json), or, equivalently, use the following `wget` command. If you would like to use this directly, please move it into the `./data` folder."
    ]
   },
   {
@@ -331,7 +329,7 @@
    "outputs": [],
    "source": [
     "# uncomment this command to download the JSON file directly\n",
-    "# wget -O data/dataset_0.json 'https://drive.google.com/uc?export=download&id=1qcGh41p-rI3H_sQ0JwOAhNiQSXriQqGi'"
+    "# wget -O data/dataset_0.json 'https://developer.download.nvidia.com/assets/Clara/monai/tutorials/swin_unetr_btcv_dataset_0.json'"
    ]
   },
   {

3d_segmentation/swin_unetr_btcv_segmentation_3d.ipynb

@@ -19,7 +19,6 @@
     "\n",
     "\n",
     "This tutorial demonstrates how to construct a training workflow of UNETR [1] on multi-organ segmentation task using the BTCV challenge dataset.\n",
-    "![image](https://lh3.googleusercontent.com/pw/AM-JKLU2eTW17rYtCmiZP3WWC-U1HCPOHwLe6pxOfJXwv2W-00aHfsNy7jeGV1dwUq0PXFOtkqasQ2Vyhcu6xkKsPzy3wx7O6yGOTJ7ZzA01S6LSh8szbjNLfpbuGgMe6ClpiS61KGvqu71xXFnNcyvJNFjN=w1448-h496-no?authuser=0)\n",
     "\n",
     "And it contains the following features:\n",
     "1. Transforms for dictionary format data.\n",
@@ -51,7 +50,7 @@
     "3. [Efficient multi-atlas abdominal segmentation on clinically acquired CT with SIMPLE context learning (MIA)](https://www.sciencedirect.com/science/article/abs/pii/S1361841515000766?via%3Dihub)\n",
     "\n",
     "\n",
-    "![image](https://lh3.googleusercontent.com/pw/AM-JKLX0svvlMdcrchGAgiWWNkg40lgXYjSHsAAuRc5Frakmz2pWzSzf87JQCRgYpqFR0qAjJWPzMQLc_mmvzNjfF9QWl_1OHZ8j4c9qrbR6zQaDJWaCLArRFh0uPvk97qAa11HtYbD6HpJ-wwTCUsaPcYvM=w1724-h522-no?authuser=0)\n",
+    "![image](../figures/BTCV_organs.png)\n",
     "\n",
     "\n",
     "\n",
@@ -586,7 +585,7 @@
     "    hidden_size=768,\n",
     "    mlp_dim=3072,\n",
     "    num_heads=12,\n",
-    "    pos_embed=\"perceptron\",\n",
+    "    proj_type=\"perceptron\",\n",
     "    norm_name=\"instance\",\n",
     "    res_block=True,\n",
     "    dropout_rate=0.0,\n",

3d_segmentation/unetr_btcv_segmentation_3d.ipynb

@@ -423,7 +423,7 @@
     "            hidden_size=768,\n",
     "            mlp_dim=3072,\n",
     "            num_heads=12,\n",
-    "            pos_embed=\"perceptron\",\n",
+    "            proj_type=\"perceptron\",\n",
     "            norm_name=\"instance\",\n",
     "            res_block=True,\n",
     "            conv_block=True,\n",

3d_segmentation/unetr_btcv_segmentation_3d_lightning.ipynb

@@ -113,6 +113,8 @@ This folder provides a simple baseline method for training, validation, and infe
 This notebook demonstrates how to construct a training workflow of UNETR on multi-organ segmentation task using the BTCV challenge dataset.
 ##### [unetr_btcv_segmentation_3d_lightning](./3d_segmentation/unetr_btcv_segmentation_3d_lightning.ipynb)
 This tutorial demonstrates how MONAI can be used in conjunction with [PyTorch Lightning](https://www.pytorchlightning.ai/) framework to construct a training workflow of UNETR on multi-organ segmentation task using the BTCV challenge dataset.
+##### [vista3d](./3d_segmentation/vista3d)
+This tutorial showcases the process of fine-tuning VISTA3D on [MSD Spleen dataset](http://medicaldecathlon.com) using MONAI. For an in-depth exploration, please visit the [VISTA](https://github.com/Project-MONAI/VISTA) repository.
 
 #### <ins>**2D registration**</ins>
 ##### [registration using mednist](./2d_registration/registration_mednist.ipynb)
@@ -313,3 +315,24 @@ This tutorial shows the use cases of training and validating a 3D Latent Diffusi
 
 ##### [2D latent diffusion model](./generative/2d_ldm)
 This tutorial shows the use cases of training and validating a 2D Latent Diffusion Model.
+
+##### [Brats 3D latent diffusion model](./3d_ldm/README.md)
+Example shows the use cases of training and validating a 3D Latent Diffusion Model on Brats 2016&2017 data, expanding on the above notebook.
+
+##### [MAISI 3D latent diffusion model](./maisi/README.md)
+Example shows the use cases of training and validating Nvidia MAISI (Medical AI for Synthetic Imaging) model, a 3D Latent Diffusion Model that can generate large CT images with paired segmentation masks, variable volume size and voxel size, as well as controllable organ/tumor size.
+
+##### [SPADE in VAE-GAN for Semantic Image Synthesis on 2D BraTS Data](./spade_gen)
+Example shows the use cases of applying SPADE, a VAE-GAN-based neural network for semantic image synthesis, to a subset of BraTS that was registered to MNI space and resampled to 2mm isotropic space, with segmentations obtained using Geodesic Information Flows (GIF).
+
+##### [Applying Latent Diffusion Models to 2D BraTS Data for Semantic Image Synthesis](./spade_ldm)
+Example shows the use cases of applying SPADE normalization to a latent diffusion model, following the methodology by Wang et al., for semantic image synthesis on a subset of BraTS registered to MNI space and resampled to 2mm isotropic space, with segmentations obtained using Geodesic Information Flows (GIF).
+
+##### [Diffusion Models for Implicit Image Segmentation Ensembles](./image_to_image_translation)
+Example shows the use cases of how to use MONAI for 2D segmentation of images using DDPMs. The same structure can also be used for conditional image generation, or image-to-image translation.
+
+##### [Evaluate Realism and Diversity of the generated images](./realism_diversity_metrics)
+Example shows the use cases of using MONAI to evaluate the performance of a generative model by computing metrics such as Frechet Inception Distance (FID) and Maximum Mean Discrepancy (MMD) for assessing realism, as well as MS-SSIM and SSIM for evaluating image diversity.
+
+#### [VISTA2D](./vista_2d)
+This tutorial demonstrates how to train a cell segmentation model using the [MONAI](https://monai.io/) framework and the [Segment Anything Model (SAM)](https://github.com/facebookresearch/segment-anything) on the [Cellpose dataset](https://www.cellpose.org/).

README.md

@@ -54,7 +54,7 @@
 parser = argparse.ArgumentParser(description="Active Learning Setting")
 
 # Directory & Json & Seed
-parser.add_argument("--base_dir", default="/home/vishwesh/experiments/al_sanity_test_apr27_2023", type=str)
+parser.add_argument("--base_dir", default="./experiments/al_sanity_test_apr27_2023", type=str)
 parser.add_argument("--data_root", default="/scratch_2/data_2021/68111", type=str)
 parser.add_argument("--json_path", default="/scratch_2/data_2021/68111/dataset_val_test_0_debug.json", type=str)
 parser.add_argument("--seed", default=102, type=int)
@@ -155,7 +155,7 @@ def main():
     # Model Definition
     device = torch.device("cuda:0")
     network = UNet(
-        dimensions=3,
+        spatial_dims=3,
         in_channels=1,
         out_channels=3,
         channels=(16, 32, 64, 128, 256),
@@ -187,7 +187,7 @@ def main():
                 b_max=1.0,
                 clip=True,
             ),
-            CropForegroundd(keys=["image", "label"], source_key="image"),
+            CropForegroundd(keys=["image", "label"], source_key="image", allow_smaller=True),
             SpatialPadd(keys=["image", "label"], spatial_size=(96, 96, 96)),
             RandCropByPosNegLabeld(
                 keys=["image", "label"],
@@ -225,7 +225,7 @@ def main():
                 b_max=1.0,
                 clip=True,
             ),
-            CropForegroundd(keys=["image", "label"], source_key="image"),
+            CropForegroundd(keys=["image", "label"], source_key="image", allow_smaller=True),
             EnsureTyped(keys=["image", "label"]),
         ]
     )
@@ -240,7 +240,7 @@ def main():
                 mode=("bilinear"),
             ),
             ScaleIntensityRanged(keys="image", a_min=-21, a_max=189, b_min=0.0, b_max=1.0, clip=True),
-            CropForegroundd(keys=("image"), source_key="image"),
+            CropForegroundd(keys=("image"), source_key="image", allow_smaller=True),
             EnsureTyped(keys=["image"]),
         ]
     )
@@ -315,7 +315,7 @@ def main():
         unl_loader = DataLoader(unl_ds, batch_size=1)
 
         # Calculation of Epochs based on steps
-        max_epochs = np.int(args.steps / (np.ceil(len(train_d) / args.batch_size)))
+        max_epochs = int(args.steps / (np.ceil(len(train_d) / args.batch_size)))
         print("Epochs Estimated are {} for Active Iter {} with {} Vols".format(max_epochs, active_iter, len(train_d)))
 
         # Model Training begins for one active iteration
@@ -393,7 +393,7 @@ def main():
         prev_best_ckpt = os.path.join(active_model_dir, "model.pt")
 
         device = torch.device("cuda:0")
-        ckpt = torch.load(prev_best_ckpt)
+        ckpt = torch.load(prev_best_ckpt, weights_only=True)
         network.load_state_dict(ckpt)
         network.to(device=device)
 
@@ -487,16 +487,16 @@ def main():
 
                     variance_dims = np.shape(variance)
                     score_list.append(np.nanmean(variance))
-                    name_list.append(unl_data["image_meta_dict"]["filename_or_obj"][0])
+                    name_list.append(unl_data["image"].meta["filename_or_obj"][0])
                     print(
                         "Variance for image: {} is: {}".format(
-                            unl_data["image_meta_dict"]["filename_or_obj"][0], np.nanmean(variance)
+                            unl_data["image"].meta["filename_or_obj"][0], np.nanmean(variance)
                         )
                     )
 
                     # Plot with matplotlib and save all slices
                     plt.figure(1)
-                    plt.imshow(np.squeeze(variance[:, :, np.int(variance_dims[2] / 2)]))
+                    plt.imshow(np.squeeze(variance[:, :, int(variance_dims[2] / 2)]))
                     plt.colorbar()
                     plt.title("Dropout Uncertainty")
                     fig_path = os.path.join(fig_base_dir, "active_{}_file_{}.png".format(active_iter, counter))