Merge pull request #21 from yasahi-hpc/optimization

Optimization
yasahi-hpc · Jun 20, 2023 · 7d833d5 · 7d833d5
2 parents e068203 + e072cd6
commit 7d833d5
Show file tree

Hide file tree

Showing 16 changed files with 190 additions and 61 deletions.
diff --git a/mini-apps/lbm2d-letkf/da_functors.hpp b/mini-apps/lbm2d-letkf/da_functors.hpp
@@ -30,14 +30,14 @@ struct nudging_functor {
 
   MDSPAN_FORCE_INLINE_FUNCTION
   void operator()(const int ix, const int iy) const {
-    auto rho_tmp = maybeLinearInterp2D(rho_, ix, iy);
-    auto u_tmp   = maybeLinearInterp2D(u_, ix, iy);
-    auto v_tmp   = maybeLinearInterp2D(v_, ix, iy);
+    value_type rho_tmp = maybeLinearInterp2D(rho_, ix, iy);
+    value_type u_tmp   = maybeLinearInterp2D(u_, ix, iy);
+    value_type v_tmp   = maybeLinearInterp2D(v_, ix, iy);
 
     for(int q=0; q<Q_; q++) {
-      const auto f_calc = f_(ix, iy, q);
-      const auto f_obs  = feq(rho_tmp, u_tmp, v_tmp, q); 
-      f_(ix, iy, q) = alpha_ * f_obs + (1-alpha_) * f_calc;
+      const value_type f_calc = f_(ix, iy, q);
+      const value_type f_obs  = feq(rho_tmp, u_tmp, v_tmp, q);
+      f_(ix, iy, q) = alpha_ * f_obs + (1.0 - alpha_) * f_calc;
     }
   }
 

diff --git a/mini-apps/lbm2d-letkf/stdpar/da_models.hpp b/mini-apps/lbm2d-letkf/stdpar/da_models.hpp
@@ -20,12 +20,13 @@ class DA_Model {
 
 public:
   DA_Model(Config& conf, IOConfig& io_conf) : conf_(conf), io_conf_(io_conf) {
-    base_dir_name_ = io_conf_.base_dir_ + "/" + io_conf_.in_case_name_;
+    base_dir_name_ = io_conf_.base_dir_ + "/" + io_conf_.in_case_name_ + "/observed/ens0000";
   }
 
   DA_Model(Config& conf, IOConfig& io_conf, MPIConfig& mpi_conf) : conf_(conf), io_conf_(io_conf) {
-    base_dir_name_ = io_conf_.base_dir_ + "/" + io_conf_.in_case_name_;
+    base_dir_name_ = io_conf_.base_dir_ + "/" + io_conf_.in_case_name_ + "/observed/ens0000";
   }
+
   virtual ~DA_Model(){}
   virtual void initialize()=0;
   virtual void apply(std::unique_ptr<DataVars>& data_vars, const int it, std::vector<Timer*>& timers)=0;
@@ -38,7 +39,8 @@ class DA_Model {
     std::string variables[3] = {"rho", "u", "v"};
     for(int it=0; it<nb_expected_files; it++) {
       for(int i=0; i<3; i++) {
-        auto file_name = base_dir_name_ + "/" + variables[i] + "obs_step" + Impl::zfill(it, 10) + ".dat";
+        auto step = it * conf_.settings_.io_interval_;
+        auto file_name = base_dir_name_ + "/" + variables[i] + "_obs_step" + Impl::zfill(step, 10) + ".dat";
         if(!Impl::isFileExists(file_name)) {
           std::runtime_error("Expected observation file does not exist." + file_name);
         }
@@ -47,17 +49,15 @@ class DA_Model {
   }
 
   void load(std::unique_ptr<DataVars>& data_vars, const int it) {
-    auto step = it / conf_.settings_.io_interval_;
-    from_file(data_vars->rho_obs(), step);
-    from_file(data_vars->u_obs(), step);
-    from_file(data_vars->v_obs(), step);
+    from_file(data_vars->rho_obs(), it);
+    from_file(data_vars->u_obs(), it);
+    from_file(data_vars->v_obs(), it);
   }
 
 private:
   template <class ViewType>
   void from_file(ViewType& value, const int step) {
-    auto file_name = base_dir_name_ + "/" + value.name() + "_step"
-                   + Impl::zfill(step, 10) + ".dat";
+    auto file_name = base_dir_name_ + "/" + value.name() + "_step" + Impl::zfill(step, 10) + ".dat";
     auto mdspan = value.mdspan();
     Impl::from_binary(file_name, mdspan);
   }

diff --git a/mini-apps/lbm2d-letkf/stdpar/lbm2d.hpp b/mini-apps/lbm2d-letkf/stdpar/lbm2d.hpp
@@ -163,10 +163,6 @@ class LBM2D : public Model {
   }
 
   void reset(std::unique_ptr<DataVars>& data_vars, const std::string mode) {
-    // Always reset counts
-    it_ = 0;
-    diag_it_ = 0;
-
     if(mode == "purturbulate") {
       purturbulate(data_vars);
     }
@@ -188,11 +184,9 @@ class LBM2D : public Model {
     auto& f  = data_vars->f();
     auto& fn = data_vars->fn();
     fn.swap(f);
-
-    it_++;
   }
 
-  void diag(std::unique_ptr<DataVars>& data_vars){
+  void diag(std::unique_ptr<DataVars>& data_vars, const int it){
     /* 
      * 0. Nature run or perturbed run (as reference)
      *    Save rho, u, v and vor into /nature (as is) and /observed (with noise)
@@ -201,24 +195,24 @@ class LBM2D : public Model {
      *    Save rho, u, v and vor into /calc (as is)
      *
      * */
-    if(it_ % conf_.settings_.io_interval_ != 0) return;
-    if(is_master_) inspect(data_vars);
+    if(it % conf_.settings_.io_interval_ != 0) return;
+    if(is_master_) inspect(data_vars, it);
 
     // Save values calculated by this ensemble member
     // Save simulation results without noises
     std::string sim_result_name = "calc";
     auto rho = data_vars->rho();
     auto u = data_vars->u();
     auto v = data_vars->v();
-    save_to_files(sim_result_name, rho, u, v, it_);
+    save_to_files(sim_result_name, rho, u, v, it);
 
     // Save noisy results
     if(is_reference_) {
       observe(data_vars); // adding noise to u_obs_, v_obs_, rho_obs_
       auto rho_obs = data_vars->rho_obs();
       auto u_obs = data_vars->u_obs();
       auto v_obs = data_vars->v_obs();
-      save_to_files("observed", rho_obs, u_obs, v_obs, it_);
+      save_to_files("observed", rho_obs, u_obs, v_obs, it);
     }
   }
 
@@ -333,7 +327,7 @@ class LBM2D : public Model {
   }
 
 private:
-  void inspect(std::unique_ptr<DataVars>& data_vars) {
+  void inspect(std::unique_ptr<DataVars>& data_vars, const int it) {
     auto [nx, ny] = conf_.settings_.n_;
     value_type dx    = static_cast<value_type>(conf_.settings_.dx_);
     value_type u_ref = static_cast<value_type>(conf_.phys_.u_ref_);
@@ -483,6 +477,7 @@ class LBM2D : public Model {
     auto vel2             = std::get<8>(moments) / (nx * ny);
 
     std::cout << std::scientific << std::setprecision(16) << std::flush;
+    std::cout << " it/nbiter: " << it << "/" << conf_.settings_.nbiter_ << std::endl;
     std::cout << " RMS, max speed: " << std::sqrt(vel2) << ", " << std::sqrt(maxvel2) << " [m/s]" << std::endl;
     //std::cout << " mean energy: " << energy << " [m2/s2]" << std::endl;
     //std::cout << " mean enstrophy: " << enstrophy << " [/s2]" << std::endl;
@@ -553,7 +548,7 @@ class LBM2D : public Model {
     value.updateSelf();
 
     std::string file_name = dir_name + "/" + value.name() + "_step"
-                          + Impl::zfill(it / conf_.settings_.io_interval_, 10) + ".dat";
+                          + Impl::zfill(it, 10) + ".dat";
     Impl::to_binary(file_name, value.mdspan());
   }
 };

diff --git a/mini-apps/lbm2d-letkf/stdpar/letkf.hpp b/mini-apps/lbm2d-letkf/stdpar/letkf.hpp
@@ -70,8 +70,7 @@ class LETKF : public DA_Model {
 
   void apply(std::unique_ptr<DataVars>& data_vars, const int it, std::vector<Timer*>& timers){
     if(it == 0) return;
-    auto step = it / conf_.settings_.io_interval_;
-    if(step % conf_.settings_.da_interval_ != 0) {
+    if(it % conf_.settings_.da_interval_ != 0) {
       std::cout << __PRETTY_FUNCTION__ << ": t=" << it << ": skip" << std::endl;
       return;
     };

diff --git a/mini-apps/lbm2d-letkf/stdpar/models.hpp b/mini-apps/lbm2d-letkf/stdpar/models.hpp
@@ -10,17 +10,15 @@ class Model {
 protected:
   Config conf_;
   IOConfig io_conf_;
-  int it_;
-  int diag_it_;
 
 public:
   Model()=delete;
-  Model(Config& conf, IOConfig& io_conf) : it_(0), diag_it_(0), conf_(conf), io_conf_(io_conf) {}
+  Model(Config& conf, IOConfig& io_conf) : conf_(conf), io_conf_(io_conf) {}
   virtual ~Model(){}
   virtual void initialize(std::unique_ptr<DataVars>& data_vars)=0;
   virtual void reset(std::unique_ptr<DataVars>& data_vars, const std::string mode)=0;
   virtual void solve(std::unique_ptr<DataVars>& data_vars)=0;
-  virtual void diag(std::unique_ptr<DataVars>& data_vars)=0;
+  virtual void diag(std::unique_ptr<DataVars>& data_vars, const int it)=0;
   virtual void finalize()=0;
 };
 

diff --git a/mini-apps/lbm2d-letkf/stdpar/nudging.hpp b/mini-apps/lbm2d-letkf/stdpar/nudging.hpp
@@ -16,8 +16,7 @@ class Nudging : public DA_Model {
 
   void apply(std::unique_ptr<DataVars>& data_vars, const int it, std::vector<Timer*>& timers){
     if(it == 0) return;
-    auto step = it / conf_.settings_.io_interval_;
-    if(step % conf_.settings_.da_interval_ != 0) {
+    if(it % conf_.settings_.da_interval_ != 0) {
       std::cout << __PRETTY_FUNCTION__ << ": t=" << it << ": skip" << std::endl;
       return;
     };

diff --git a/mini-apps/lbm2d-letkf/stdpar/solver.hpp b/mini-apps/lbm2d-letkf/stdpar/solver.hpp
@@ -75,7 +75,7 @@ class Solver {
       timers_[TimerEnum::DA]->end();
 
       timers_[TimerEnum::Diag]->begin();
-      model_->diag(data_vars_);
+      model_->diag(data_vars_, it);
       timers_[TimerEnum::Diag]->end();
 
       timers_[TimerEnum::LBMSolver]->begin();
@@ -154,6 +154,11 @@ class Solver {
       io_conf_.in_case_name_ = json_data["Settings"]["in_case_name"].get<std::string>();
     }
 
+    // da_interval should be divisible by io_interval.
+    if(conf_.settings_.da_interval_ % conf_.settings_.io_interval_ == 0) {
+      std::runtime_error("da_interval must be divisible by io_interval.");
+    }
+
     // Saving json file to output directory
     const std::string out_dir = io_conf_.base_dir_ + "/" + io_conf_.case_name_;
     Impl::mkdirs(out_dir, 0755);

diff --git a/wk/letkf.json b/wk/letkf.json
@@ -17,12 +17,12 @@
         "sim_type": "letkf",
         "case_name": "letkf",
         "in_case_name": "nature",
-        "nx": 256,
-        "ny": 256,
-        "spinup": 1000,
-        "nbiter": 1000,
-        "io_interval": 100,
-        "da_interval": 10,
+        "nx": 512,
+        "ny": 512,
+        "spinup": 20000,
+        "nbiter": 10000,
+        "io_interval": 20,
+        "da_interval": 20,
         "obs_interval": 1,
         "lyapnov": false,
         "les": true,

diff --git a/wk/letkf_256.json b/wk/letkf_256.json
@@ -0,0 +1,33 @@
+{
+    "Physics": {
+        "rho_ref": 1.0,
+        "u_ref": 1.0,
+        "nu": 1.0e-4,
+        "friction_rate": 5.0e-4,
+        "kf": 4.0,
+        "fkf": 5.6,
+        "dk": 10,
+        "sigma": 5,
+        "p_amp": 0.01,
+        "obs_error_rho": 0.01,
+        "obs_error_u": 0.1
+    },
+    "Settings": {
+        "base_dir": "/work/03/jh220030a/i18048/2023P3HPC/executor_testing/wk",
+        "sim_type": "letkf",
+        "case_name": "letkf256",
+        "in_case_name": "nature256",
+        "nx": 256,
+        "ny": 256,
+        "spinup": 10000,
+        "nbiter": 10000,
+        "io_interval": 20,
+        "da_interval": 20,
+        "obs_interval": 1,
+        "lyapnov": false,
+        "les": true,
+        "da_nud_rate": 0.1,
+        "beta": 1.0,
+        "rloc_len": 1
+    }
+}
diff --git a/wk/nature.json b/wk/nature.json
@@ -16,12 +16,12 @@
         "base_dir": "/work/03/jh220030a/i18048/2023P3HPC/executor_testing/wk",
         "sim_type": "nature",
         "case_name": "nature",
-        "nx": 256,
-        "ny": 256,
-        "spinup": 1000,
-        "nbiter": 1000,
-        "io_interval": 100,
-        "da_interval": 10,
+        "nx": 512,
+        "ny": 512,
+        "spinup": 10000,
+        "nbiter": 10000,
+        "io_interval": 20,
+        "da_interval": 20,
         "obs_interval": 1,
         "lyapnov": false,
         "les": true,

diff --git a/wk/nature_256.json b/wk/nature_256.json
@@ -0,0 +1,30 @@
+{
+    "Physics": {
+        "rho_ref": 1.0,
+        "u_ref": 1.0,
+        "nu": 1.0e-4,
+        "friction_rate": 5.0e-4,
+        "kf": 4.0,
+        "fkf": 5.6,
+        "dk": 10,
+        "sigma": 5,
+        "p_amp": 0.01,
+        "obs_error_rho": 0.01,
+        "obs_error_u": 0.1
+    },
+    "Settings": {
+        "base_dir": "/work/03/jh220030a/i18048/2023P3HPC/executor_testing/wk",
+        "sim_type": "nature",
+        "case_name": "nature256",
+        "nx": 256,
+        "ny": 256,
+        "spinup": 10000,
+        "nbiter": 10000,
+        "io_interval": 20,
+        "da_interval": 20,
+        "obs_interval": 1,
+        "lyapnov": false,
+        "les": true,
+        "da_nud_rate": 0.1
+    }
+}
diff --git a/wk/no_da_256.json b/wk/no_da_256.json
@@ -0,0 +1,31 @@
+{
+    "Physics": {
+        "rho_ref": 1.0,
+        "u_ref": 1.0,
+        "nu": 1.0e-4,
+        "friction_rate": 5.0e-4,
+        "kf": 4.0,
+        "fkf": 5.6,
+        "dk": 10,
+        "sigma": 5,
+        "p_amp": 0.01,
+        "obs_error_rho": 0.01,
+        "obs_error_u": 0.1
+    },
+    "Settings": {
+        "base_dir": "/work/03/jh220030a/i18048/2023P3HPC/executor_testing/wk",
+        "sim_type": "no_da",
+        "case_name": "no_da",
+        "in_case_name": "nature256",
+        "nx": 256,
+        "ny": 256,
+        "spinup": 10000,
+        "nbiter": 10000,
+        "io_interval": 20,
+        "da_interval": 20,
+        "obs_interval": 1,
+        "lyapnov": false,
+        "les": true,
+        "da_nud_rate": 0.1
+    }
+}
diff --git a/wk/nudging.json b/wk/nudging.json
@@ -17,12 +17,12 @@
         "sim_type": "nudging",
         "case_name": "nudging",
         "in_case_name": "nature",
-        "nx": 256,
-        "ny": 256,
-        "spinup": 1000,
-        "nbiter": 1000,
-        "io_interval": 100,
-        "da_interval": 10,
+        "nx": 512,
+        "ny": 512,
+        "spinup": 20000,
+        "nbiter": 10000,
+        "io_interval": 20,
+        "da_interval": 20,
         "obs_interval": 1,
         "lyapnov": false,
         "les": true,

diff --git a/wk/nudging_256.json b/wk/nudging_256.json
@@ -0,0 +1,31 @@
+{
+    "Physics": {
+        "rho_ref": 1.0,
+        "u_ref": 1.0,
+        "nu": 1.0e-4,
+        "friction_rate": 5.0e-4,
+        "kf": 4.0,
+        "fkf": 5.6,
+        "dk": 10,
+        "sigma": 5,
+        "p_amp": 0.01,
+        "obs_error_rho": 0.01,
+        "obs_error_u": 0.1
+    },
+    "Settings": {
+        "base_dir": "/work/03/jh220030a/i18048/2023P3HPC/executor_testing/wk",
+        "sim_type": "nudging",
+        "case_name": "nudging256",
+        "in_case_name": "nature256",
+        "nx": 256,
+        "ny": 256,
+        "spinup": 10000,
+        "nbiter": 10000,
+        "io_interval": 20,
+        "da_interval": 20,
+        "obs_interval": 1,
+        "lyapnov": false,
+        "les": true,
+        "da_nud_rate": 0.1
+    }
+}