Skip to content

Commit 719d050

Browse files
Can-ZhaoCan-Zhao
committed
train for rflow tested
Signed-off-by: Can-Zhao <[email protected]>
1 parent 8680ae8 commit 719d050

File tree

2 files changed

+101
-94
lines changed

2 files changed

+101
-94
lines changed

‎generation/maisi/maisi_train_controlnet_tutorial.ipynb‎

Lines changed: 46 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -67,11 +67,11 @@
6767
"name": "stdout",
6868
"output_type": "stream",
6969
"text": [
70-
"MONAI version: 1.4.1rc1+31.g90de55b1\n",
70+
"MONAI version: 1.4.1rc1+32.g34f37973\n",
7171
"Numpy version: 1.26.4\n",
7272
"Pytorch version: 2.5.0+cu124\n",
7373
"MONAI flags: HAS_EXT = False, USE_COMPILED = False, USE_META_DICT = False\n",
74-
"MONAI rev id: 90de55b1340ffc8eccf960d111229535bcb2f81d\n",
74+
"MONAI rev id: 34f379735c5e18e7f809453eb1b3606c225c788b\n",
7575
"MONAI __file__: /localhome/<username>/.local/lib/python3.10/site-packages/monai/__init__.py\n",
7676
"\n",
7777
"Optional dependencies:\n",
@@ -141,12 +141,12 @@
141141
"name": "stderr",
142142
"output_type": "stream",
143143
"text": [
144-
"[2025-03-12 22:27:22.838][ INFO](notebook) - MAISI version is maisi3d-ddpm, whether to use body_region is True\n"
144+
"[2025-03-14 16:29:13.938][ INFO](notebook) - MAISI version is maisi3d-rflow, whether to use body_region is False\n"
145145
]
146146
}
147147
],
148148
"source": [
149-
"maisi_version = \"maisi3d-ddpm\"\n",
149+
"maisi_version = \"maisi3d-rflow\"\n",
150150
"if maisi_version == \"maisi3d-ddpm\":\n",
151151
" model_def_path = \"./configs/config_maisi3d-ddpm.json\"\n",
152152
"elif maisi_version == \"maisi3d-rflow\":\n",
@@ -248,9 +248,10 @@
248248
"name": "stderr",
249249
"output_type": "stream",
250250
"text": [
251-
"[2025-03-12 22:27:25.046][ INFO](notebook) - Generated simulated images.\n",
252-
"[2025-03-12 22:27:25.047][ INFO](notebook) - img_emb shape: (64, 64, 32, 4)\n",
253-
"[2025-03-12 22:27:25.048][ INFO](notebook) - label shape: (256, 256, 128)\n"
251+
"[2025-03-14 16:29:13.952][ INFO](notebook) - Save data list json file to ./temp_work_dir_controlnet_train_demo/sim_datalist.json\n",
252+
"[2025-03-14 16:29:16.033][ INFO](notebook) - Generated simulated images.\n",
253+
"[2025-03-14 16:29:16.034][ INFO](notebook) - img_emb shape: (64, 64, 32, 4)\n",
254+
"[2025-03-14 16:29:16.035][ INFO](notebook) - label shape: (256, 256, 128)\n"
254255
]
255256
}
256257
],
@@ -266,6 +267,7 @@
266267
"datalist_file = os.path.join(work_dir, \"sim_datalist.json\")\n",
267268
"with open(datalist_file, \"w\") as f:\n",
268269
" json.dump(sim_datalist, f, indent=4)\n",
270+
"logger.info(f\"Save data list json file to {datalist_file}\")\n",
269271
"\n",
270272
"for d in sim_datalist[\"training\"]:\n",
271273
" # The image embedding is downsampled twice by Autoencoder.\n",
@@ -322,8 +324,8 @@
322324
"name": "stderr",
323325
"output_type": "stream",
324326
"text": [
325-
"[2025-03-12 22:27:25.062][ INFO](notebook) - files and folders under work_dir: ['config_maisi.json', 'models', 'config_maisi_controlnet_train.json', 'outputs', 'sim_dataroot', 'environment_maisi_controlnet_train.json', 'sim_datalist.json'].\n",
326-
"[2025-03-12 22:27:25.063][ INFO](notebook) - number of GPUs: 1.\n"
327+
"[2025-03-14 16:29:16.049][ INFO](notebook) - files and folders under work_dir: ['config_maisi.json', 'models', 'config_maisi_controlnet_train.json', 'outputs', 'sim_dataroot', '.ipynb_checkpoints', 'environment_maisi_controlnet_train.json', 'sim_datalist.json'].\n",
328+
"[2025-03-14 16:29:16.050][ INFO](notebook) - number of GPUs: 1.\n"
327329
]
328330
}
329331
],
@@ -468,29 +470,30 @@
468470
"name": "stderr",
469471
"output_type": "stream",
470472
"text": [
471-
"[2025-03-12 22:27:25.074][ INFO](notebook) - Training the model...\n"
473+
"[2025-03-14 16:29:16.061][ INFO](notebook) - Training the model...\n"
472474
]
473475
},
474476
{
475477
"name": "stdout",
476478
"output_type": "stream",
477479
"text": [
478-
"[2025-03-12 22:27:33.707][ INFO](maisi.controlnet.training) - Number of GPUs: 8\n",
479-
"[2025-03-12 22:27:33.708][ INFO](maisi.controlnet.training) - World_size: 1\n",
480-
"[2025-03-12 22:27:35.410][ INFO](maisi.controlnet.training) - trained diffusion model is not loaded.\n",
481-
"[2025-03-12 22:27:35.410][ INFO](maisi.controlnet.training) - set scale_factor -> 1.0.\n",
482-
"2025-03-12 22:27:35,902 - INFO - 'dst' model updated: 158 of 206 variables.\n",
483-
"[2025-03-12 22:27:35.907][ INFO](maisi.controlnet.training) - train controlnet model from scratch.\n",
484-
"[2025-03-12 22:27:35.930][ INFO](maisi.controlnet.training) - total number of training steps: 4.0.\n",
485-
"[2025-03-12 22:27:38.006][ INFO](maisi.controlnet.training) -\n",
486-
"[Epoch 1/2] [Batch 1/2] [LR: 0.00000563] [loss: 0.7976] ETA: 0:00:02.073507\n",
487-
"[2025-03-12 22:27:38.147][ INFO](maisi.controlnet.training) -\n",
488-
"[Epoch 1/2] [Batch 2/2] [LR: 0.00000250] [loss: 0.7985] ETA: 0:00:00\n",
489-
"[2025-03-12 22:27:38.683][ INFO](maisi.controlnet.training) - best loss -> 0.7980280518531799.\n",
490-
"[2025-03-12 22:27:39.955][ INFO](maisi.controlnet.training) -\n",
491-
"[Epoch 2/2] [Batch 1/2] [LR: 0.00000063] [loss: 0.7992] ETA: 0:00:01.807460\n",
492-
"[2025-03-12 22:27:40.086][ INFO](maisi.controlnet.training) -\n",
493-
"[Epoch 2/2] [Batch 2/2] [LR: 0.00000000] [loss: 0.7980] ETA: 0:00:00\n",
480+
"[2025-03-14 16:29:23.336][ INFO](maisi.controlnet.training) - Number of GPUs: 8\n",
481+
"[2025-03-14 16:29:23.336][ INFO](maisi.controlnet.training) - World_size: 1\n",
482+
"[2025-03-14 16:29:24.771][ INFO](maisi.controlnet.training) - trained diffusion model is not loaded.\n",
483+
"[2025-03-14 16:29:24.771][ INFO](maisi.controlnet.training) - set scale_factor -> 1.0.\n",
484+
"2025-03-14 16:29:25,271 - INFO - 'dst' model updated: 180 of 231 variables.\n",
485+
"[2025-03-14 16:29:25.277][ INFO](maisi.controlnet.training) - train controlnet model from scratch.\n",
486+
"[2025-03-14 16:29:25.300][ INFO](maisi.controlnet.training) - total number of training steps: 4.0.\n",
487+
"[2025-03-14 16:29:26.826][ INFO](maisi.controlnet.training) -\n",
488+
"[Epoch 1/2] [Batch 1/2] [LR: 0.00000563] [loss: 0.8278] ETA: 0:00:01.523338\n",
489+
"[2025-03-14 16:29:26.974][ INFO](maisi.controlnet.training) -\n",
490+
"[Epoch 1/2] [Batch 2/2] [LR: 0.00000250] [loss: 0.8289] ETA: 0:00:00\n",
491+
"[2025-03-14 16:29:27.585][ INFO](maisi.controlnet.training) - best loss -> 0.8283329606056213.\n",
492+
"[2025-03-14 16:29:28.909][ INFO](maisi.controlnet.training) -\n",
493+
"[Epoch 2/2] [Batch 1/2] [LR: 0.00000063] [loss: 0.8288] ETA: 0:00:01.934548\n",
494+
"[2025-03-14 16:29:29.052][ INFO](maisi.controlnet.training) -\n",
495+
"[Epoch 2/2] [Batch 2/2] [LR: 0.00000000] [loss: 0.8277] ETA: 0:00:00\n",
496+
"[2025-03-14 16:29:29.716][ INFO](maisi.controlnet.training) - best loss -> 0.8282470703125.\n",
494497
"\n"
495498
]
496499
}
@@ -533,32 +536,32 @@
533536
"name": "stderr",
534537
"output_type": "stream",
535538
"text": [
536-
"[2025-03-12 22:27:42.632][ INFO](notebook) - Inference...\n"
539+
"[2025-03-14 16:29:32.229][ INFO](notebook) - Inference...\n"
537540
]
538541
},
539542
{
540543
"name": "stdout",
541544
"output_type": "stream",
542545
"text": [
543-
"[2025-03-12 22:27:53.399][ INFO](maisi.controlnet.infer) - Number of GPUs: 8\n",
544-
"[2025-03-12 22:27:53.400][ INFO](maisi.controlnet.infer) - World_size: 1\n",
545-
"[2025-03-12 22:27:54.101][ INFO](maisi.controlnet.infer) - trained autoencoder model is not loaded.\n",
546-
"[2025-03-12 22:27:55.286][ INFO](maisi.controlnet.infer) - trained diffusion model is not loaded.\n",
547-
"[2025-03-12 22:27:55.286][ INFO](maisi.controlnet.infer) - set scale_factor -> 1.0.\n",
548-
"2025-03-12 22:27:55,756 - INFO - 'dst' model updated: 158 of 206 variables.\n",
549-
"[2025-03-12 22:27:55.761][ INFO](maisi.controlnet.infer) - trained controlnet is not loaded.\n",
550-
"[2025-03-12 22:27:56.340][ INFO](root) - `controllable_anatomy_size` is not provided.\n",
551-
"[2025-03-12 22:27:56.344][ INFO](root) - ---- Start generating latent features... ----\n",
552-
"[2025-03-12 22:27:58.065][ INFO](root) - ---- Latent features generation time: 1.7215001583099365 seconds ----\n",
553-
"[2025-03-12 22:27:58.066][ INFO](root) - ---- Start decoding latent features into images... ----\n",
554-
"[2025-03-12 22:27:58.838][ INFO](root) - ---- Image decoding time: 0.7712326049804688 seconds ----\n",
555-
"2025-03-12 22:27:59,142 INFO image_writer.py:197 - writing: temp_work_dir_controlnet_train_demo/outputs/sample_20250312_222759_124463_image.nii.gz\n",
556-
"2025-03-12 22:27:59,487 INFO image_writer.py:197 - writing: temp_work_dir_controlnet_train_demo/outputs/sample_20250312_222759_124463_label.nii.gz\n",
546+
"[2025-03-14 16:29:39.519][ INFO](maisi.controlnet.infer) - Number of GPUs: 8\n",
547+
"[2025-03-14 16:29:39.519][ INFO](maisi.controlnet.infer) - World_size: 1\n",
548+
"[2025-03-14 16:29:39.990][ INFO](maisi.controlnet.infer) - trained autoencoder model is not loaded.\n",
549+
"[2025-03-14 16:29:41.213][ INFO](maisi.controlnet.infer) - trained diffusion model is not loaded.\n",
550+
"[2025-03-14 16:29:41.213][ INFO](maisi.controlnet.infer) - set scale_factor -> 1.0.\n",
551+
"2025-03-14 16:29:41,716 - INFO - 'dst' model updated: 180 of 231 variables.\n",
552+
"[2025-03-14 16:29:41.721][ INFO](maisi.controlnet.infer) - trained controlnet is not loaded.\n",
553+
"[2025-03-14 16:29:42.102][ INFO](root) - `controllable_anatomy_size` is not provided.\n",
554+
"[2025-03-14 16:29:42.104][ INFO](root) - ---- Start generating latent features... ----\n",
555+
"[2025-03-14 16:29:42.670][ INFO](root) - ---- DM/ControlNet Latent features generation time: 0.565190315246582 seconds ----\n",
556+
"[2025-03-14 16:29:42.672][ INFO](root) - ---- Start decoding latent features into images... ----\n",
557+
"[2025-03-14 16:29:43.314][ INFO](root) - ---- Image VAE decoding time: 0.6416211128234863 seconds ----\n",
558+
"2025-03-14 16:29:43,602 INFO image_writer.py:197 - writing: temp_work_dir_controlnet_train_demo/outputs/sample_20250314_162943_586788_image.nii.gz\n",
559+
"2025-03-14 16:29:43,940 INFO image_writer.py:197 - writing: temp_work_dir_controlnet_train_demo/outputs/sample_20250314_162943_586788_label.nii.gz\n",
557560
"\n",
558561
"\n",
559562
" 0%| | 0/1 [00:00<?, ?it/s]\n",
560-
"100%|██████████| 1/1 [00:01<00:00, 1.62s/it]\n",
561-
"100%|██████████| 1/1 [00:01<00:00, 1.62s/it]\n",
563+
"100%|██████████| 1/1 [00:00<00:00, 2.02it/s]\n",
564+
"100%|██████████| 1/1 [00:00<00:00, 2.02it/s]\n",
562565
"\n"
563566
]
564567
}

0 commit comments

Comments
 (0)