|
| 1 | +import os, subprocess, shutil |
| 2 | +import argparse |
| 3 | +from tqdm.auto import tqdm |
| 4 | +from monai.apps import download_url |
| 5 | +from pathlib import Path |
| 6 | +from huggingface_hub import snapshot_download |
| 7 | +from typing import List, Dict, Optional |
| 8 | + |
| 9 | +def fetch_to_hf_path_cmd( |
| 10 | + items: List[Dict[str, str]], |
| 11 | + root_dir: str = "./", # staging dir for CLI output |
| 12 | + revision: str = "main", |
| 13 | + overwrite: bool = False, |
| 14 | + token: Optional[str] = None, # or rely on env HUGGINGFACE_HUB_TOKEN |
| 15 | +) -> list[str]: |
| 16 | + """ |
| 17 | + items: list of {"repo_id": "...", "filename": "path/in/repo.ext", "path": "local/target.ext"} |
| 18 | + Returns list of saved local paths (in the same order as items). |
| 19 | + """ |
| 20 | + saved = [] |
| 21 | + root = Path(root_dir) |
| 22 | + root.mkdir(parents=True, exist_ok=True) |
| 23 | + |
| 24 | + # Env for subprocess; keep Rust fast-path off to avoid notebook progress quirks |
| 25 | + env = os.environ.copy() |
| 26 | + if token: |
| 27 | + env["HUGGINGFACE_HUB_TOKEN"] = token |
| 28 | + env.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "0") # safer in Jupyter |
| 29 | + env.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "0") # show CLI progress in terminal |
| 30 | + |
| 31 | + for it in items: |
| 32 | + repo_id = it["repo_id"] |
| 33 | + repo_file = it["filename"] |
| 34 | + dst = Path(it["path"]) |
| 35 | + dst.parent.mkdir(parents=True, exist_ok=True) |
| 36 | + |
| 37 | + if dst.exists() and not overwrite: |
| 38 | + saved.append(str(dst)) |
| 39 | + continue |
| 40 | + |
| 41 | + # Build command (no shell=True; no quoting issues) |
| 42 | + cmd = [ |
| 43 | + "huggingface-cli", "download", |
| 44 | + repo_id, |
| 45 | + "--include", repo_file, |
| 46 | + "--revision", revision, |
| 47 | + "--local-dir", str(root), |
| 48 | + ] |
| 49 | + # Run |
| 50 | + subprocess.run(cmd, check=True, env=env) |
| 51 | + |
| 52 | + # Source path where CLI placed the file |
| 53 | + src = root / repo_file |
| 54 | + if not src.exists(): |
| 55 | + raise FileNotFoundError( |
| 56 | + f"Expected downloaded file missing: {src}\n" |
| 57 | + f"Tip: authenticate (`huggingface-cli login` or pass token=...)," |
| 58 | + f" and avoid shared-IP 429s." |
| 59 | + ) |
| 60 | + |
| 61 | + # Move to desired target |
| 62 | + if dst.exists() and overwrite: |
| 63 | + dst.unlink() |
| 64 | + if src.resolve() != dst.resolve(): |
| 65 | + dst.parent.mkdir(parents=True, exist_ok=True) |
| 66 | + shutil.move(str(src), str(dst)) |
| 67 | + saved.append(str(dst)) |
| 68 | + |
| 69 | + return saved |
| 70 | + |
| 71 | + |
| 72 | + |
| 73 | +def download_model_data(generate_version,root_dir, model_only=False): |
| 74 | + # TODO: remove the `files` after the files are uploaded to the NGC |
| 75 | + if generate_version == "ddpm-ct" or generate_version == "rflow-ct": |
| 76 | + files = [ |
| 77 | + { |
| 78 | + "path": "models/autoencoder_v1.pt", |
| 79 | + "repo_id": "nvidia/NV-Generate-CT", |
| 80 | + "filename":"models/autoencoder_v1.pt", |
| 81 | + }, |
| 82 | + { |
| 83 | + "path": "models/mask_generation_autoencoder.pt", |
| 84 | + "repo_id": "nvidia/NV-Generate-CT", |
| 85 | + "filename": "models/mask_generation_autoencoder.pt", |
| 86 | + }, |
| 87 | + { |
| 88 | + "path": "models/mask_generation_diffusion_unet.pt", |
| 89 | + "repo_id": "nvidia/NV-Generate-CT", |
| 90 | + "filename": "models/mask_generation_diffusion_unet.pt", |
| 91 | + }] |
| 92 | + if not model_only: |
| 93 | + files += [ |
| 94 | + { |
| 95 | + "path": "datasets/all_anatomy_size_conditions.json", |
| 96 | + "repo_id": "nvidia/NV-Generate-CT", |
| 97 | + "filename": "datasets/all_anatomy_size_conditions.json", |
| 98 | + }, |
| 99 | + { |
| 100 | + "path": "datasets/all_masks_flexible_size_and_spacing_4000.zip", |
| 101 | + "repo_id": "nvidia/NV-Generate-CT", |
| 102 | + "filename": "datasets/all_masks_flexible_size_and_spacing_4000.zip", |
| 103 | + }, |
| 104 | + ] |
| 105 | + elif generate_version == "rflow-mr": |
| 106 | + files = [ |
| 107 | + { |
| 108 | + "path": "models/autoencoder_v2.pt", |
| 109 | + "repo_id": "nvidia/NV-Generate-MR", |
| 110 | + "filename": "models/autoencoder_v2.pt", |
| 111 | + }, |
| 112 | + { |
| 113 | + "path": "models/diff_unet_3d_rflow-mr.pt", |
| 114 | + "repo_id": "nvidia/NV-Generate-MR", |
| 115 | + "filename": "models/diff_unet_3d_rflow-mr.pt", |
| 116 | + } |
| 117 | + ] |
| 118 | + else: |
| 119 | + raise ValueError(f"generate_version has to be chosen from ['ddpm-ct', 'rflow-ct', 'rflow-mr'], yet got {generate_version}.") |
| 120 | + if generate_version == "ddpm-ct": |
| 121 | + files += [ |
| 122 | + { |
| 123 | + "path": "models/diff_unet_3d_ddpm-ct.pt", |
| 124 | + "repo_id": "nvidia/NV-Generate-CT", |
| 125 | + "filename": "models/diff_unet_3d_ddpm-ct.pt", |
| 126 | + }, |
| 127 | + { |
| 128 | + "path": "models/controlnet_3d_ddpm-ct.pt", |
| 129 | + "repo_id": "nvidia/NV-Generate-CT", |
| 130 | + "filename": "models/controlnet_3d_ddpm-ct.pt", |
| 131 | + }] |
| 132 | + if not model_only: |
| 133 | + files += [ |
| 134 | + { |
| 135 | + "path": "datasets/candidate_masks_flexible_size_and_spacing_3000.json", |
| 136 | + "repo_id": "nvidia/NV-Generate-CT", |
| 137 | + "filename": "datasets/candidate_masks_flexible_size_and_spacing_3000.json", |
| 138 | + }, |
| 139 | + ] |
| 140 | + elif generate_version == "rflow-ct": |
| 141 | + files += [ |
| 142 | + { |
| 143 | + "path": "models/diff_unet_3d_rflow-ct.pt", |
| 144 | + "repo_id": "nvidia/NV-Generate-CT", |
| 145 | + "filename": "models/diff_unet_3d_rflow-ct.pt", |
| 146 | + }, |
| 147 | + { |
| 148 | + "path": "models/controlnet_3d_rflow-ct.pt", |
| 149 | + "repo_id": "nvidia/NV-Generate-CT", |
| 150 | + "filename": "models/controlnet_3d_rflow-ct.pt", |
| 151 | + }] |
| 152 | + if not model_only: |
| 153 | + files += [ |
| 154 | + { |
| 155 | + "path": "datasets/candidate_masks_flexible_size_and_spacing_4000.json", |
| 156 | + "repo_id": "nvidia/NV-Generate-CT", |
| 157 | + "filename": "datasets/candidate_masks_flexible_size_and_spacing_4000.json", |
| 158 | + }, |
| 159 | + ] |
| 160 | + |
| 161 | + for file in files: |
| 162 | + file["path"] = file["path"] if "datasets/" not in file["path"] else os.path.join(root_dir, file["path"]) |
| 163 | + if "repo_id" in file.keys(): |
| 164 | + path = fetch_to_hf_path_cmd([file],root_dir=root_dir, revision="main") |
| 165 | + print("saved to:", path) |
| 166 | + else: |
| 167 | + download_url(url=file["url"], filepath=file["path"]) |
| 168 | + return |
| 169 | + |
| 170 | + |
| 171 | +if __name__ == "__main__": |
| 172 | + parser = argparse.ArgumentParser(description="Model downloading") |
| 173 | + parser.add_argument( |
| 174 | + "--version", |
| 175 | + type=str, |
| 176 | + default="rflow-ct", |
| 177 | + ) |
| 178 | + parser.add_argument( |
| 179 | + "--root_dir", |
| 180 | + type=str, |
| 181 | + default="./", |
| 182 | + ) |
| 183 | + parser.add_argument("--model_only", dest="model_only", action="store_true", help="Download model only, not any dataset") |
| 184 | + |
| 185 | + args = parser.parse_args() |
| 186 | + download_model_data(args.version, args.root_dir, args.model_only) |
0 commit comments