Skip to content

Commit

Permalink
disable PYTORCH_CUDA_ALLOC_CONF to see if that fixes cicd
Browse files Browse the repository at this point in the history
  • Loading branch information
winglian committed Jul 16, 2024
1 parent 45b247c commit 599fbe1
Showing 1 changed file with 5 additions and 5 deletions.
10 changes: 5 additions & 5 deletions src/axolotl/train.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,11 @@ def train(
*, cfg: DictDefault, cli_args: TrainerCliArgs, dataset_meta: TrainDatasetMeta
) -> Tuple[Union[PeftModel, PreTrainedModel], PreTrainedTokenizer]:
# enable expandable segments for cuda allocation to improve VRAM usage
torch_version = torch.__version__.split(".")
torch_major, torch_minor = int(torch_version[0]), int(torch_version[1])
if torch_major == 2 and torch_minor >= 2:
if os.getenv("PYTORCH_CUDA_ALLOC_CONF") is None:
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
# torch_version = torch.__version__.split(".")
# torch_major, torch_minor = int(torch_version[0]), int(torch_version[1])
# if torch_major == 2 and torch_minor >= 2:
# if os.getenv("PYTORCH_CUDA_ALLOC_CONF") is None:
# os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"

# load the tokenizer first
LOG.debug(
Expand Down

0 comments on commit 599fbe1

Please sign in to comment.