training code

Author: Gege Wen

.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+
+ECLIPSE/meta_data/*.npy
+*.pt
+logs/*

.ipynb_checkpoints/eval_sequential_prediction_dp-checkpoint.ipynb

@@ -0,0 +1,168 @@
+import math
+import torch
+from torch import Tensor
+from typing import List, Optional
+from torch.optim.optimizer import Optimizer
+
+
+def adam(params: List[Tensor],
+         grads: List[Tensor],
+         exp_avgs: List[Tensor],
+         exp_avg_sqs: List[Tensor],
+         max_exp_avg_sqs: List[Tensor],
+         state_steps: List[int],
+         *,
+         amsgrad: bool,
+         beta1: float,
+         beta2: float,
+         lr: float,
+         weight_decay: float,
+         eps: float):
+    r"""Functional API that performs Adam algorithm computation.
+    See :class:`~torch.optim.Adam` for details.
+    """
+
+    for i, param in enumerate(params):
+
+        grad = grads[i]
+        exp_avg = exp_avgs[i]
+        exp_avg_sq = exp_avg_sqs[i]
+        step = state_steps[i]
+
+        bias_correction1 = 1 - beta1 ** step
+        bias_correction2 = 1 - beta2 ** step
+
+        if weight_decay != 0:
+            grad = grad.add(param, alpha=weight_decay)
+
+        # Decay the first and second moment running average coefficient
+        exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)
+        exp_avg_sq.mul_(beta2).addcmul_(grad, grad.conj(), value=1 - beta2)
+        if amsgrad:
+            # Maintains the maximum of all 2nd moment running avg. till now
+            torch.maximum(max_exp_avg_sqs[i], exp_avg_sq, out=max_exp_avg_sqs[i])
+            # Use the max. for normalizing running avg. of gradient
+            denom = (max_exp_avg_sqs[i].sqrt() / math.sqrt(bias_correction2)).add_(eps)
+        else:
+            denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)
+
+        step_size = lr / bias_correction1
+
+        param.addcdiv_(exp_avg, denom, value=-step_size)
+
+
+class Adam(Optimizer):
+    r"""Implements Adam algorithm.
+    It has been proposed in `Adam: A Method for Stochastic Optimization`_.
+    The implementation of the L2 penalty follows changes proposed in
+    `Decoupled Weight Decay Regularization`_.
+    Args:
+        params (iterable): iterable of parameters to optimize or dicts defining
+            parameter groups
+        lr (float, optional): learning rate (default: 1e-3)
+        betas (Tuple[float, float], optional): coefficients used for computing
+            running averages of gradient and its square (default: (0.9, 0.999))
+        eps (float, optional): term added to the denominator to improve
+            numerical stability (default: 1e-8)
+        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
+        amsgrad (boolean, optional): whether to use the AMSGrad variant of this
+            algorithm from the paper `On the Convergence of Adam and Beyond`_
+            (default: False)
+    .. _Adam\: A Method for Stochastic Optimization:
+        https://arxiv.org/abs/1412.6980
+    .. _Decoupled Weight Decay Regularization:
+        https://arxiv.org/abs/1711.05101
+    .. _On the Convergence of Adam and Beyond:
+        https://openreview.net/forum?id=ryQu7f-RZ
+    """
+
+    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,
+                 weight_decay=0, amsgrad=False):
+        if not 0.0 <= lr:
+            raise ValueError("Invalid learning rate: {}".format(lr))
+        if not 0.0 <= eps:
+            raise ValueError("Invalid epsilon value: {}".format(eps))
+        if not 0.0 <= betas[0] < 1.0:
+            raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
+        if not 0.0 <= betas[1] < 1.0:
+            raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
+        if not 0.0 <= weight_decay:
+            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
+        defaults = dict(lr=lr, betas=betas, eps=eps,
+                        weight_decay=weight_decay, amsgrad=amsgrad)
+        super(Adam, self).__init__(params, defaults)
+
+    def __setstate__(self, state):
+        super(Adam, self).__setstate__(state)
+        for group in self.param_groups:
+            group.setdefault('amsgrad', False)
+
+    @torch.no_grad()
+    def step(self, closure=None):
+        """Performs a single optimization step.
+        Args:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            with torch.enable_grad():
+                loss = closure()
+
+        for group in self.param_groups:
+            params_with_grad = []
+            grads = []
+            exp_avgs = []
+            exp_avg_sqs = []
+            max_exp_avg_sqs = []
+            state_steps = []
+            beta1, beta2 = group['betas']
+
+            for p in group['params']:
+                if p.grad is not None:
+                    params_with_grad.append(p)
+                    if p.grad.is_sparse:
+                        raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')
+                    grads.append(p.grad)
+
+                    state = self.state[p]
+                    # Lazy state initialization
+                    if len(state) == 0:
+                        state['step'] = 0
+                        # Exponential moving average of gradient values
+                        state['exp_avg'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+                        # Exponential moving average of squared gradient values
+                        state['exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+                        if group['amsgrad']:
+                            # Maintains max of all exp. moving avg. of sq. grad. values
+                            state['max_exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+
+                    exp_avgs.append(state['exp_avg'])
+                    exp_avg_sqs.append(state['exp_avg_sq'])
+
+                    if group['amsgrad']:
+                        max_exp_avg_sqs.append(state['max_exp_avg_sq'])
+
+                    # update the steps for each param group update
+                    state['step'] += 1
+                    # record the step after step update
+                    state_steps.append(state['step'])
+
+            adam(params_with_grad,
+                 grads,
+                 exp_avgs,
+                 exp_avg_sqs,
+                 max_exp_avg_sqs,
+                 state_steps,
+                 amsgrad=group['amsgrad'],
+                 beta1=beta1,
+                 beta2=beta2,
+                 lr=group['lr'],
+                 weight_decay=group['weight_decay'],
+                 eps=group['eps'])
+        return loss
+    
+    
+    
+    
+    

.ipynb_checkpoints/eval_sequential_prediction_sg-checkpoint.ipynb

@@ -0,0 +1,47 @@
+from torch.utils.data import Dataset
+import random 
+import os
+import torch
+
+def GLOBAL_to_LGR_path(global_lists, key, names, var):
+    lgr_list = []
+    for path in global_lists:
+        case = path.split('/')[-1]
+        slope = case[:7]
+        idx = case.split('_')[2]
+        for nwell in range(1,5):
+            if var == 'dP':
+                string = f'{slope}_{idx}_{key}_WELL{nwell}_DP.pt'
+                if string in names:
+                    home_path = f'/dP_{key}/'
+                    lgr_list.append(home_path + string)
+            elif var == 'SG':
+                string = f'{slope}_{idx}_{key}_WELL{nwell}_SG.pt'
+                if string in names:
+                    home_path = f'/SG_{key}/'
+                    lgr_list.append(home_path + string)
+                    
+    return lgr_list
+
+class CustomDataset(Dataset):
+    def __init__(self, root_path, names):
+        self.names = names
+        self.root_path = root_path
+        
+    def __len__(self):
+        return len(self.names)
+
+    def __getitem__(self, idx):
+        path = self.names[idx]
+        data = torch.load(self.root_path+path)
+        
+        name = path.split('/')[-1]
+        slope, idx, well = name[:7], name.split('_')[2], name.split('_')[-2]
+        
+        x = data['input'].permute(0,4,1,2,3,5)[0,...]
+        y = data['output'].permute(0,4,1,2,3,5)[0,...,:1]
+    
+        D = {'x': x, 
+             'y': y,
+             'path': [slope, idx, well]}
+        return D

Adam.py

@@ -0,0 +1,156 @@
+import torch
+import numpy as np
+import torch.nn as nn
+import torch.nn.functional as F
+
+import operator
+from functools import reduce
+from functools import partial
+
+from timeit import default_timer
+
+torch.manual_seed(0)
+np.random.seed(0)
+
+class SpectralConv4d(nn.Module):
+    def __init__(self, in_channels, out_channels, modes1, modes2, modes3, modes4):
+        super(SpectralConv4d, self).__init__()
+
+        """
+        4D Fourier layer. It does FFT, linear transform, and Inverse FFT.    
+        """
+
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.modes1 = modes1 #Number of Fourier modes to multiply, at most floor(N/2) + 1
+        self.modes2 = modes2
+        self.modes3 = modes3
+        self.modes4 = modes4
+        
+        self.scale = (1 / (in_channels * out_channels))
+        self.weights1 = nn.Parameter(self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, self.modes3, self.modes4, dtype=torch.cfloat))
+        self.weights2 = nn.Parameter(self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, self.modes3, self.modes4, dtype=torch.cfloat))
+        self.weights3 = nn.Parameter(self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, self.modes3, self.modes4, dtype=torch.cfloat))
+        self.weights4 = nn.Parameter(self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, self.modes3, self.modes4, dtype=torch.cfloat))
+        self.weights5 = nn.Parameter(self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, self.modes3, self.modes4, dtype=torch.cfloat))
+        self.weights6 = nn.Parameter(self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, self.modes3, self.modes4, dtype=torch.cfloat))
+        self.weights7 = nn.Parameter(self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, self.modes3, self.modes4, dtype=torch.cfloat))
+        self.weights8 = nn.Parameter(self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, self.modes3, self.modes4, dtype=torch.cfloat))
+
+    # Complex multiplication
+    def compl_mul4d(self, input, weights):
+        # (batch, in_channel, x,y,t ), (in_channel, out_channel, x,y,t) -> (batch, out_channel, x,y,t)
+        return torch.einsum("bixyzt,ioxyzt->boxyzt", input, weights)
+
+    def forward(self, x):
+        batchsize = x.shape[0]
+        #Compute Fourier coeffcients up to factor of e^(- something constant)
+        x_ft = torch.fft.rfftn(x, dim=[-4,-3,-2,-1])
+
+        # Multiply relevant Fourier modes
+        out_ft = torch.zeros(batchsize, self.out_channels, x.size(-4), x.size(-3), x.size(-2), x.size(-1)//2 + 1, dtype=torch.cfloat, device=x.device)
+        
+        out_ft[:, :, :self.modes1, :self.modes2, :self.modes3, :self.modes4] = self.compl_mul4d(x_ft[:, :, :self.modes1, :self.modes2, :self.modes3, :self.modes4], self.weights1)
+        out_ft[:, :, -self.modes1:, :self.modes2, :self.modes3, :self.modes4] = self.compl_mul4d(x_ft[:, :, -self.modes1:, :self.modes2, :self.modes3, :self.modes4], self.weights2)
+        out_ft[:, :, :self.modes1, -self.modes2:, :self.modes3, :self.modes4] = self.compl_mul4d(x_ft[:, :, :self.modes1, -self.modes2:, :self.modes3, :self.modes4], self.weights3)
+        out_ft[:, :, :self.modes1, :self.modes2, -self.modes3:, :self.modes4] = self.compl_mul4d(x_ft[:, :, :self.modes1, :self.modes2, -self.modes3:, :self.modes4], self.weights4)
+        out_ft[:, :, -self.modes1:, -self.modes2:, :self.modes3, :self.modes4] = self.compl_mul4d(x_ft[:, :, -self.modes1:, -self.modes2:, :self.modes3, :self.modes4], self.weights5)
+        out_ft[:, :, -self.modes1:, :self.modes2, -self.modes3:, :self.modes4] = self.compl_mul4d(x_ft[:, :, -self.modes1:, :self.modes2, -self.modes3:, :self.modes4], self.weights6)
+        out_ft[:, :, :self.modes1, -self.modes2:, -self.modes3:, :self.modes4] = self.compl_mul4d(x_ft[:, :, :self.modes1, -self.modes2:, -self.modes3:, :self.modes4], self.weights7)
+        out_ft[:, :, -self.modes1:, -self.modes2:, -self.modes3:, :self.modes4] = self.compl_mul4d(x_ft[:, :, -self.modes1:, -self.modes2:, -self.modes3:, :self.modes4], self.weights8)        
+
+        #Return to physical space
+        x = torch.fft.irfftn(out_ft, s=(x.size(-4), x.size(-3), x.size(-2), x.size(-1)))
+        return x
+
+class Block4d(nn.Module):
+    def __init__(self, width, width2, modes1, modes2, modes3, modes4, out_dim):
+        super(Block4d, self).__init__()
+        self.modes1 = modes1
+        self.modes2 = modes2
+        self.modes3 = modes3
+        self.modes4 = modes4
+        
+        self.width = width
+        self.width2 = width2
+        self.out_dim = out_dim
+        self.padding = 8
+        
+        # channel
+        self.conv0 = SpectralConv4d(self.width, self.width, self.modes1, self.modes2, self.modes3, self.modes4)
+        self.conv1 = SpectralConv4d(self.width, self.width, self.modes1, self.modes2, self.modes3, self.modes4)
+        self.conv2 = SpectralConv4d(self.width, self.width, self.modes1, self.modes2, self.modes3, self.modes4)
+        self.conv3 = SpectralConv4d(self.width, self.width, self.modes1, self.modes2, self.modes3, self.modes4)
+        self.w0 = nn.Conv1d(self.width, self.width, 1)
+        self.w1 = nn.Conv1d(self.width, self.width, 1)
+        self.w2 = nn.Conv1d(self.width, self.width, 1)
+        self.w3 = nn.Conv1d(self.width, self.width, 1)
+        self.fc1 = nn.Linear(self.width, self.width2)
+        self.fc2 = nn.Linear(self.width2, self.out_dim)
+        
+    def forward(self, x):
+        batchsize = x.shape[0]
+        size_x, size_y, size_z, size_t = x.shape[2], x.shape[3], x.shape[4], x.shape[5]
+#         print(size_x, size_y, size_z, size_t)
+        # channel
+#         print(x.shape)
+        x1 = self.conv0(x)
+#         print(x1.shape)
+        x2 = self.w0(x.view(batchsize, self.width, -1)).view(batchsize, self.width, size_x, size_y, size_z, size_t)
+        x = x1 + x2
+        x = F.gelu(x)
+        
+        x1 = self.conv1(x)
+        x2 = self.w1(x.view(batchsize, self.width, -1)).view(batchsize, self.width, size_x, size_y, size_z, size_t)
+        x = x1 + x2
+        x = F.gelu(x)
+
+        x1 = self.conv2(x)
+        x2 = self.w2(x.view(batchsize, self.width, -1)).view(batchsize, self.width, size_x, size_y, size_z, size_t)
+        x = x1 + x2
+        x = F.gelu(x)
+
+        x1 = self.conv3(x)
+        x2 = self.w3(x.view(batchsize, self.width, -1)).view(batchsize, self.width, size_x, size_y, size_z, size_t)
+        x = x1 + x2
+        
+        x = x[:, :, self.padding:-self.padding, self.padding*2:-self.padding*2, 
+              self.padding*2:-self.padding*2, self.padding:-self.padding]
+        
+        x = x.permute(0, 2, 3, 4, 5, 1)  # pad the domain if input is non-periodic
+        x1 = self.fc1(x)
+        x = F.gelu(x1)
+        x = self.fc2(x)
+        
+        return x
+    
+class FNO4d(nn.Module):
+    def __init__(self, modes1, modes2, modes3, modes4, width, in_dim):
+        super(FNO4d, self).__init__()
+
+        self.modes1 = modes1
+        self.modes2 = modes2
+        self.modes3 = modes3
+        self.modes4 = modes4
+        self.width = width
+        self.width2 = width*4
+        self.in_dim = in_dim
+        self.out_dim = 1
+        self.padding = 8  # pad the domain if input is non-periodic
+        
+        self.fc0 = nn.Linear(self.in_dim, self.width)
+        self.conv = Block4d(self.width, self.width2, 
+                               self.modes1, self.modes2, self.modes3, self.modes4, self.out_dim)
+
+    def forward(self, x, gradient=False):
+        x = self.fc0(x) 
+        x = x.permute(0, 5, 1, 2, 3, 4)
+        x = F.pad(x, [self.padding, self.padding, self.padding*2, self.padding*2, self.padding*2, 
+                      self.padding*2, self.padding, self.padding])  
+        
+        x = self.conv(x)
+    
+        return x
+    
+    
+