deploy: 83ab799

_downloads/819e10305ddd6839cd7da05935b17060/mass-inference.txt

@@ -1,17 +1,17 @@
-2024-09-14 01:33:49 (INFO): Running in local mode without elastic launch (single gpu only)
-2024-09-14 01:33:49 (INFO): Setting env PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
-2024-09-14 01:33:49 (INFO): Project root: /home/runner/work/fairchem/fairchem/src/fairchem
-2024-09-14 01:33:51 (INFO): amp: false
+2024-09-14 01:39:01 (INFO): Running in local mode without elastic launch (single gpu only)
+2024-09-14 01:39:01 (INFO): Setting env PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+2024-09-14 01:39:01 (INFO): Project root: /home/runner/work/fairchem/fairchem/src/fairchem
+2024-09-14 01:39:03 (INFO): amp: false
 cmd:
-  checkpoint_dir: /home/runner/work/fairchem/fairchem/docs/core/checkpoints/2024-09-14-01-33-52
-  commit: 7d40b20
+  checkpoint_dir: /home/runner/work/fairchem/fairchem/docs/core/checkpoints/2024-09-14-01-38-08
+  commit: 83ab799
   identifier: ''
-  logs_dir: /home/runner/work/fairchem/fairchem/docs/core/logs/tensorboard/2024-09-14-01-33-52
+  logs_dir: /home/runner/work/fairchem/fairchem/docs/core/logs/tensorboard/2024-09-14-01-38-08
   print_every: 10
-  results_dir: /home/runner/work/fairchem/fairchem/docs/core/results/2024-09-14-01-33-52
+  results_dir: /home/runner/work/fairchem/fairchem/docs/core/results/2024-09-14-01-38-08
   seed: 0
-  timestamp_id: 2024-09-14-01-33-52
-  version: 0.1.dev1+g7d40b20
+  timestamp_id: 2024-09-14-01-38-08
+  version: 1.2.0
 dataset: {}
 evaluation_metrics:
   metrics:
@@ -113,24 +113,24 @@ test_dataset:
 trainer: ocp
 val_dataset: {}
 
-2024-09-14 01:33:51 (INFO): Loading model: gemnet_t
-2024-09-14 01:33:53 (INFO): Loaded GemNetT with 31671825 parameters.
-2024-09-14 01:33:53 (WARNING): log_summary for Tensorboard not supported
-2024-09-14 01:33:53 (WARNING): Could not find dataset metadata.npz files in '[PosixPath('data.db')]'
-2024-09-14 01:33:53 (WARNING): Disabled BalancedBatchSampler because num_replicas=1.
-2024-09-14 01:33:53 (WARNING): Failed to get data sizes, falling back to uniform partitioning. BalancedBatchSampler requires a dataset that has a metadata attributed with number of atoms.
-2024-09-14 01:33:53 (INFO): rank: 0: Sampler created...
-2024-09-14 01:33:53 (INFO): Created BalancedBatchSampler with sampler=<fairchem.core.common.data_parallel.StatefulDistributedSampler object at 0x7fc29edea6d0>, batch_size=16, drop_last=False
-2024-09-14 01:33:53 (INFO): Attemping to load user specified checkpoint at /tmp/fairchem_checkpoints/gndt_oc22_all_s2ef.pt
-2024-09-14 01:33:53 (INFO): Loading checkpoint from: /tmp/fairchem_checkpoints/gndt_oc22_all_s2ef.pt
-2024-09-14 01:33:53 (INFO): Overwriting scaling factors with those loaded from checkpoint. If you're generating predictions with a pretrained checkpoint, this is the correct behavior. To disable this, delete `scale_dict` from the checkpoint. 
-2024-09-14 01:33:53 (WARNING): Scale factor comment not found in model
-2024-09-14 01:33:53 (INFO): Predicting on test.
+2024-09-14 01:39:03 (INFO): Loading model: gemnet_t
+2024-09-14 01:39:04 (INFO): Loaded GemNetT with 31671825 parameters.
+2024-09-14 01:39:04 (WARNING): log_summary for Tensorboard not supported
+2024-09-14 01:39:04 (WARNING): Could not find dataset metadata.npz files in '[PosixPath('data.db')]'
+2024-09-14 01:39:04 (WARNING): Disabled BalancedBatchSampler because num_replicas=1.
+2024-09-14 01:39:04 (WARNING): Failed to get data sizes, falling back to uniform partitioning. BalancedBatchSampler requires a dataset that has a metadata attributed with number of atoms.
+2024-09-14 01:39:04 (INFO): rank: 0: Sampler created...
+2024-09-14 01:39:04 (INFO): Created BalancedBatchSampler with sampler=<fairchem.core.common.data_parallel.StatefulDistributedSampler object at 0x7f68c417c7d0>, batch_size=16, drop_last=False
+2024-09-14 01:39:04 (INFO): Attemping to load user specified checkpoint at /tmp/fairchem_checkpoints/gndt_oc22_all_s2ef.pt
+2024-09-14 01:39:04 (INFO): Loading checkpoint from: /tmp/fairchem_checkpoints/gndt_oc22_all_s2ef.pt
+2024-09-14 01:39:05 (INFO): Overwriting scaling factors with those loaded from checkpoint. If you're generating predictions with a pretrained checkpoint, this is the correct behavior. To disable this, delete `scale_dict` from the checkpoint. 
+2024-09-14 01:39:05 (WARNING): Scale factor comment not found in model
+2024-09-14 01:39:05 (INFO): Predicting on test.
 device 0:   0%|                                           | 0/3 [00:00<?, ?it/s]/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/torch_geometric/data/collate.py:145: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
   storage = elem.storage()._new_shared(numel)
 /opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/torch_geometric/data/collate.py:145: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
   storage = elem.storage()._new_shared(numel)
-device 0:  33%|███████████▋                       | 1/3 [00:03<00:07,  3.79s/it]device 0:  67%|███████████████████████▎           | 2/3 [00:06<00:03,  3.09s/it]device 0: 100%|███████████████████████████████████| 3/3 [00:08<00:00,  2.72s/it]device 0: 100%|███████████████████████████████████| 3/3 [00:08<00:00,  2.90s/it]
-2024-09-14 01:34:01 (INFO): Writing results to /home/runner/work/fairchem/fairchem/docs/core/results/2024-09-14-01-33-52/ocp_predictions.npz
-2024-09-14 01:34:01 (INFO): Total time taken: 8.841936349868774
-Elapsed time = 15.2 seconds
+device 0:  33%|███████████▋                       | 1/3 [00:03<00:06,  3.34s/it]device 0:  67%|███████████████████████▎           | 2/3 [00:06<00:02,  2.94s/it]device 0: 100%|███████████████████████████████████| 3/3 [00:06<00:00,  1.78s/it]device 0: 100%|███████████████████████████████████| 3/3 [00:06<00:00,  2.14s/it]
+2024-09-14 01:39:11 (INFO): Writing results to /home/runner/work/fairchem/fairchem/docs/core/results/2024-09-14-01-38-08/ocp_predictions.npz
+2024-09-14 01:39:11 (INFO): Total time taken: 6.570510149002075
+Elapsed time = 13.2 seconds

core/fine-tuning/fine-tuning-oxides.html

@@ -844,7 +844,7 @@ <h1>Fine tuning a model<a class="headerlink" href="#fine-tuning-a-model" title="
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Elapsed time 87.0 seconds.
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Elapsed time 85.9 seconds.
 </pre></div>
 </div>
 <img alt="../../_images/341531b0218eea408d04046d981d55fe218d43494f1ecf51297aeefb6668761e.png" src="../../_images/341531b0218eea408d04046d981d55fe218d43494f1ecf51297aeefb6668761e.png" />
@@ -923,7 +923,7 @@ <h1>Fine tuning the checkpoint<a class="headerlink" href="#fine-tuning-the-check
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2258/1448814737.py:12: DeprecationWarning: Please use atoms.calc = calc
+<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2247/1448814737.py:12: DeprecationWarning: Please use atoms.calc = calc
   atoms.set_calculator(calc)
 </pre></div>
 </div>
@@ -1213,7 +1213,7 @@ <h2>Running the training job<a class="headerlink" href="#running-the-training-jo
 <span class="expanded">Hide code cell output</span>
 </summary>
 <div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Elapsed time = 217.8 seconds
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Elapsed time = 217.6 seconds
 </pre></div>
 </div>
 </div>
@@ -1229,7 +1229,7 @@ <h2>Running the training job<a class="headerlink" href="#running-the-training-jo
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&#39;fine-tuning/checkpoints/2024-09-14-01-27-28-ft-oxides&#39;
+<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&#39;fine-tuning/checkpoints/2024-09-14-01-33-52-ft-oxides&#39;
 </pre></div>
 </div>
 </div>
@@ -1285,7 +1285,7 @@ <h2>Running the training job<a class="headerlink" href="#running-the-training-jo
 </div>
 </div>
 <div class="cell_output docutils container">
-<img alt="../../_images/c33aecb183b893e509cdf18f7ff69ce35b8d7d11a91aa965cfe2ef334e00aeb2.png" src="../../_images/c33aecb183b893e509cdf18f7ff69ce35b8d7d11a91aa965cfe2ef334e00aeb2.png" />
+<img alt="../../_images/3e086324df5f4806aeb9cc61c2e614e46cc98f8dc690125c4992aa6023d86829.png" src="../../_images/3e086324df5f4806aeb9cc61c2e614e46cc98f8dc690125c4992aa6023d86829.png" />
 </div>
 </div>
 <div class="cell docutils container">
@@ -1296,7 +1296,7 @@ <h2>Running the training job<a class="headerlink" href="#running-the-training-jo
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>New MAE = 0.180 eV/atom
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>New MAE = 0.142 eV/atom
 </pre></div>
 </div>
 </div>
@@ -1315,7 +1315,7 @@ <h2>Running the training job<a class="headerlink" href="#running-the-training-jo
 </div>
 </div>
 <div class="cell_output docutils container">
-<img alt="../../_images/21c255b2bb74dcf1f86d70bc6e00c96e8a44b9929a29d812d02499b7a68abff3.png" src="../../_images/21c255b2bb74dcf1f86d70bc6e00c96e8a44b9929a29d812d02499b7a68abff3.png" />
+<img alt="../../_images/ca91d66665fb300bf3857b66731f7234ed72a71232db0197e7667f146a14e11b.png" src="../../_images/ca91d66665fb300bf3857b66731f7234ed72a71232db0197e7667f146a14e11b.png" />
 </div>
 </div>
 <p>It is possible to continue refining the fit. The simple things to do are to use more epochs of training. Eventually the MAE will stabilize, and then it may be necessary to adjust other optimization parameters like the learning rate (usually you decrease it).</p>

core/gotchas.html

@@ -925,7 +925,7 @@ <h1>I get wildly different energies from the different models<a class="headerlin
 WARNING:root:No seed has been set in modelcheckpoint or OCPCalculator! Results may not be reproducible on re-run
 </pre></div>
 </div>
-<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2610/2356712572.py:11: DeprecationWarning: Please use atoms.calc = calc
+<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2601/2356712572.py:11: DeprecationWarning: Please use atoms.calc = calc
   slab.set_calculator(calc)
 </pre></div>
 </div>
@@ -959,7 +959,7 @@ <h1>I get wildly different energies from the different models<a class="headerlin
 <div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>WARNING:root:No seed has been set in modelcheckpoint or OCPCalculator! Results may not be reproducible on re-run
 </pre></div>
 </div>
-<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2610/2054440827.py:9: DeprecationWarning: Please use atoms.calc = calc
+<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2601/2054440827.py:9: DeprecationWarning: Please use atoms.calc = calc
   slab.set_calculator(calc)
 </pre></div>
 </div>
@@ -990,11 +990,11 @@ <h1>I get wildly different energies from the different models<a class="headerlin
 WARNING:root:No seed has been set in modelcheckpoint or OCPCalculator! Results may not be reproducible on re-run
 </pre></div>
 </div>
-<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2610/1817216860.py:7: DeprecationWarning: Please use atoms.calc = calc
+<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2601/1817216860.py:7: DeprecationWarning: Please use atoms.calc = calc
   slab.set_calculator(calc)
 </pre></div>
 </div>
-<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>1.662048101425171
+<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>1.6902730464935303
 </pre></div>
 </div>
 </div>
@@ -1453,7 +1453,7 @@ <h1>To tag or not?<a class="headerlink" href="#to-tag-or-not" title="Link to thi
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2610/3906293788.py:3: DeprecationWarning: Please use atoms.calc = calc
+<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2601/3906293788.py:3: DeprecationWarning: Please use atoms.calc = calc
   atoms.set_calculator(calc)
 </pre></div>
 </div>
@@ -1499,11 +1499,11 @@ <h1>To tag or not?<a class="headerlink" href="#to-tag-or-not" title="Link to thi
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2610/4094489779.py:3: DeprecationWarning: Please use atoms.calc = calc
+<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2601/4094489779.py:3: DeprecationWarning: Please use atoms.calc = calc
   atoms.set_calculator(calc)
 </pre></div>
 </div>
-<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>-0.42973652482032776
+<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>-0.4297371804714203
 </pre></div>
 </div>
 </div>
@@ -1552,21 +1552,21 @@ <h1>Stochastic simulation results<a class="headerlink" href="#stochastic-simulat
 WARNING:root:No seed has been set in modelcheckpoint or OCPCalculator! Results may not be reproducible on re-run
 </pre></div>
 </div>
-<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2610/3396863997.py:11: DeprecationWarning: Please use atoms.calc = calc
+<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2601/3396863997.py:11: DeprecationWarning: Please use atoms.calc = calc
   slab.set_calculator(calc)
 </pre></div>
 </div>
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>1.212760066986084 1.4313059825709947e-06
-1.2127587795257568
-1.2127587795257568
-1.2127609252929688
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>1.2127602577209473 1.897787927830925e-06
 1.212759256362915
-1.2127623558044434
+1.212756633758545
+1.212759017944336
 1.212761402130127
-1.2127573490142822
-1.2127611637115479
 1.2127604484558105
-1.2127602100372314
+1.2127623558044434
+1.2127621173858643
+1.2127585411071777
+1.2127597332000732
+1.2127630710601807
 </pre></div>
 </div>
 </div>
@@ -1606,11 +1606,11 @@ <h1>The forces don’t sum to zero<a class="headerlink" href="#the-forces-don-t-
 WARNING:root:No seed has been set in modelcheckpoint or OCPCalculator! Results may not be reproducible on re-run
 </pre></div>
 </div>
-<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2610/4037009387.py:11: DeprecationWarning: Please use atoms.calc = calc
+<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/tmp/ipykernel_2601/4037009387.py:11: DeprecationWarning: Please use atoms.calc = calc
   slab.set_calculator(calc)
 </pre></div>
 </div>
-<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>array([ 0.01599356,  0.00170369, -0.07197821], dtype=float32)
+<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>array([ 0.01599278,  0.00170265, -0.07197714], dtype=float32)
 </pre></div>
 </div>
 </div>
@@ -1623,7 +1623,7 @@ <h1>The forces don’t sum to zero<a class="headerlink" href="#the-forces-don-t-
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>array([-6.7404471e-08, -6.3620973e-08, -4.7683716e-07], dtype=float32)
+<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>array([1.0000076e-07, 7.3865522e-08, 0.0000000e+00], dtype=float32)
 </pre></div>
 </div>
 </div>