From ea654a46efc0838c4882805335aa9415f51d1eb3 Mon Sep 17 00:00:00 2001
From: James Thewlis <james@unitary.ai>
Date: Wed, 1 Jan 2025 19:22:22 +0000
Subject: [PATCH] Set TOKENIZERS_PARALLELISM to false in train.py

Avoids the warning about "The current process just got forked. Disabling parallelism to avoid deadlocks"
---
 train.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/train.py b/train.py
index 674f96a..6b842a7 100644
--- a/train.py
+++ b/train.py
@@ -1,9 +1,12 @@
+# flake8: noqa: E402
 import argparse
 import json
 import os
 
-import pytorch_lightning as pl
+# Disable tokenizer parallelism to avoid deadlocks due to forking
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
+import pytorch_lightning as pl
 import src.data_loaders as module_data
 import torch
 from pytorch_lightning.callbacks import ModelCheckpoint