Correctly set random seed for pytext training (#269)

Summary:
Pull Request resolved: #269

Random seed was configured in Trainer but unused; move this to Task (because model parameters are initialized in Task, before Trainer) and correctly set random / numpy / torch seeds.

Reviewed By: seayoung1112

Differential Revision: D13950646

fbshipit-source-id: f37427311466bb6ffcf00677f81bd63cc858f8c8

demo/configs/rnng.json

@@ -43,7 +43,6 @@
         },
         "trainer": {
           "real_trainer": {
-            "random_seed": 0,
             "epochs": 1,
             "early_stop_after": 0,
             "max_clip_norm": null,

pytext/task/disjoint_multitask.py

@@ -111,6 +111,7 @@ def from_config(cls, task_config, metadata=None, model_state=None):
             lr_scheduler=Scheduler(
                 optimizer, task_config.scheduler, metric_reporter.lower_is_better
             ),
+            random_seed=task_config.random_seed,
         )
 
     def __init__(self, target_task_name, exporters, **kwargs):

pytext/task/task.py

@@ -55,6 +55,7 @@ class Config(ConfigBase):
         optimizer: Optimizer.Config = Adam.Config()
         scheduler: Optional[Scheduler.Config] = Scheduler.Config()
         exporter: Optional[ModelExporter.Config] = None
+        random_seed: int = 0
 
     @classmethod
     def from_config(cls, task_config, metadata=None, model_state=None):
@@ -116,6 +117,7 @@ def from_config(cls, task_config, metadata=None, model_state=None):
                 optimizer, task_config.scheduler, metric_reporter.lower_is_better
             ),
             exporter=exporter,
+            random_seed=task_config.random_seed,
         )
 
     def __init__(
@@ -127,6 +129,7 @@ def __init__(
         optimizer: torch.optim.Optimizer,
         lr_scheduler: List[torch.optim.lr_scheduler._LRScheduler],
         exporter: Optional[ModelExporter],
+        random_seed: int,
     ) -> None:
         self.trainer: Trainer = trainer
         self.data_handler: DataHandler = data_handler
@@ -135,6 +138,7 @@ def __init__(
         self.optimizer: torch.optim.Optimizer = optimizer
         self.lr_scheduler: List[torch.optim.lr_scheduler._LRScheduler] = lr_scheduler
         self.exporter = exporter
+        self.random_seed = random_seed
 
     def train(self, train_config, rank=0, world_size=1):
         """
@@ -146,7 +150,9 @@ def train(self, train_config, rank=0, world_size=1):
             world_size (int): for distributed training only, total gpu to use, default
                 is 1
         """
-        return self.trainer.train(
+        # Check seed is set correctly.
+        assert torch.initial_seed() == self.random_seed
+        result = self.trainer.train(
             self.data_handler.get_train_iter(rank, world_size),
             self.data_handler.get_eval_iter(),
             self.model,
@@ -156,6 +162,9 @@ def train(self, train_config, rank=0, world_size=1):
             self.lr_scheduler,
             rank=rank,
         )
+        # Check seed is not tampered with by other code.
+        assert torch.initial_seed() == self.random_seed
+        return result
 
     def test(self, test_path):
         """

pytext/trainers/trainer.py

@@ -30,7 +30,6 @@ class Trainer(TrainerBase):
         2 Test trained model, compute and publish metrics against a blind test set.
 
     Attributes:
-        random_seed (int): Manual random seed
         epochs (int): Training epochs
         early_stop_after (int): Stop after how many epochs when the eval metric
             is not improving
@@ -40,8 +39,6 @@ class Trainer(TrainerBase):
     """
 
     class Config(ConfigBase):
-        # Manual random seed
-        random_seed: int = 0
         # Training epochs
         epochs: int = 10
         # Stop after how many epochs when the eval metric is not improving

pytext/utils/python_utils.py

@@ -1,6 +1,16 @@
 #!/usr/bin/env python3
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import random
+
+import numpy as np
+import torch
 
 
 def cls_vars(cls):
     return [v for n, v in vars(cls).items() if not n.startswith("_")]
+
+
+def set_random_seeds(seed):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)

pytext/workflow.py

@@ -10,6 +10,7 @@
 from pytext.metric_reporters.channel import Channel
 from pytext.task import Task, create_task, load, save
 from pytext.utils.dist_utils import dist_init
+from pytext.utils.python_utils import set_random_seeds
 
 from .utils import cuda_utils
 
@@ -90,6 +91,8 @@ def prepare_task(
 
     print("\nParameters: {}\n".format(config))
     _set_cuda(config.use_cuda_if_available, device_id, world_size)
+    set_random_seeds(config.task.random_seed)
+
     if config.load_snapshot_path and os.path.isfile(config.load_snapshot_path):
         task = load(config.load_snapshot_path)
     else: