Skip to content

Commit

Permalink
Merge pull request #206 from PDoakORNL/fix_no_mpi_build
Browse files Browse the repository at this point in the history 
fixing no mpi build
  • Loading branch information
@PDoakORNL
PDoakORNL authored Aug 3, 2020
2 parents f3ac453 + d2cd13c commit 6fcfcb4
Show file tree
Hide file tree
Showing 14 changed files with 203 additions and 228 deletions.
31 changes: 16 additions & 15 deletions applications/dca/main_dca.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,22 +73,23 @@ int main(int argc, char** argv) {
dca_data.initialize();

dca::DistType distribution = parameters.get_g4_distribution();
switch(distribution) {
case dca::DistType::MPI:
{
DCALoopDispatch<dca::DistType::MPI> dca_loop_dispatch;
dca_loop_dispatch(parameters, dca_data, concurrency);
switch (distribution) {
#ifdef DCA_HAVE_MPI
case dca::DistType::MPI: {
DCALoopDispatch<dca::DistType::MPI> dca_loop_dispatch;
dca_loop_dispatch(parameters, dca_data, concurrency);
} break;
#else
case dca::DistType::MPI: {
throw std::runtime_error(
"Input calls for function MPI distribution but DCA is not built with MPI.");
} break;
#endif
case dca::DistType::NONE: {
DCALoopDispatch<dca::DistType::NONE> dca_loop_dispatch;
dca_loop_dispatch(parameters, dca_data, concurrency);
} break;
}
break;
case dca::DistType::NONE:
{
DCALoopDispatch<dca::DistType::NONE> dca_loop_dispatch;
dca_loop_dispatch(parameters, dca_data, concurrency);
}
break;
}


}
catch (const std::exception& err) {
std::cout << "Unhandled exception in main function:\n\t" << err.what();
Expand Down
6 changes: 6 additions & 0 deletions include/dca/distribution/dist_types.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,20 @@
// See CITATION.md for citation guidelines, if DCA++ is used for scientific publications.
//
// Author: Peter Doak ([email protected])
// Giovanni Balduzzi ([email protected])
//
// This file provides distribution strategy tags

#ifndef DCA_DIST_TYPE_HPP
#define DCA_DIST_TYPE_HPP

#include <string>

namespace dca {
enum class DistType { NONE, MPI };

DistType stringToDistType(const std::string& name);
std::string toString(DistType type);
} // namespace dca

#endif // DCA_DIST_TYPE_HPP
8 changes: 7 additions & 1 deletion include/dca/function/function.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,14 @@
#include "dca/distribution/dist_types.hpp"
#include "dca/function/scalar_cast.hpp"
#include "dca/function/set_to_zero.hpp"
#include "dca/util/ignore.hpp"
#include "dca/util/pack_operations.hpp"
#include "dca/util/type_utils.hpp"

#include "dca/parallel/util/get_workload.hpp"
#ifdef DCA_HAVE_MPI
#include "mpi.h"
#endif

namespace dca {
namespace func {
Expand Down Expand Up @@ -302,12 +305,15 @@ function<scalartype, domain>::function(const std::string& name, DistType dist)
size_sbdm(dmn.get_leaf_domain_sizes()),
step_sbdm(dmn.get_leaf_domain_steps()),
fnc_values(nullptr) {
if (name.substr(0, 2) == "G4" && dist == DistType::MPI) {
dca::util::ignoreUnused(dist);
#ifdef DCA_HAVE_MPI
if (dist == DistType::MPI) {
int my_rank, mpi_size;
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
nb_elements_ = dca::parallel::util::getWorkload(dmn.get_size(), mpi_size, my_rank);
}
#endif // DCA_HAVE_MPI
fnc_values = new scalartype[nb_elements_];
for (int linind = 0; linind < nb_elements_; ++linind)
setToZero(fnc_values[linind]);
Expand Down
4 changes: 0 additions & 4 deletions include/dca/parallel/no_concurrency/serial_gather.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,8 @@

#include <vector>

#include <mpi.h>

#include "dca/function/domains.hpp"
#include "dca/function/function.hpp"
#include "dca/parallel/mpi_concurrency/mpi_gang.hpp"
#include "dca/parallel/mpi_concurrency/mpi_type_map.hpp"
#include "dca/util/integer_division.hpp"

namespace dca {
Expand Down
7 changes: 4 additions & 3 deletions include/dca/phys/dca_step/cluster_solver/ctaux/ctaux_accumulator.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,17 +43,18 @@
#ifdef DCA_HAVE_CUDA
#include "dca/phys/dca_step/cluster_solver/shared_tools/accumulation/sp/sp_accumulator_gpu.hpp"
#include "dca/phys/dca_step/cluster_solver/shared_tools/accumulation/tp/tp_accumulator_gpu.hpp"
#endif // DCA_HAVE_CUDA
#ifdef DCA_HAVE_MPI
#include "dca/phys/dca_step/cluster_solver/shared_tools/accumulation/tp/tp_accumulator_mpi_gpu.hpp"
#endif // DCA_HAVE_MPI
#endif // DCA_HAVE_CUDA

namespace dca {
namespace phys {
namespace solver {
namespace ctaux {
// dca::phys::solver::ctaux::

template <dca::linalg::DeviceType device_t, class Parameters, class Data, DistType DIST, typename Real = double>
template <dca::linalg::DeviceType device_t, class Parameters, class Data, DistType DIST = dca::DistType::NONE, typename Real = double>
class CtauxAccumulator : public MC_accumulator_data {
public:
using this_type = CtauxAccumulator<device_t, Parameters, Data, DIST, Real>;
Expand Down Expand Up @@ -162,7 +163,7 @@ class CtauxAccumulator : public MC_accumulator_data {
}

static std::size_t staticDeviceFingerprint() {
return accumulator::TpAccumulator<Parameters, device_t>::staticDeviceFingerprint();
return accumulator::TpAccumulator<Parameters, device_t, DIST>::staticDeviceFingerprint();
}

private:
Expand Down
88 changes: 46 additions & 42 deletions include/dca/phys/dca_step/cluster_solver/shared_tools/accumulation/tp/tp_accumulator.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ namespace accumulator {
template <class Parameters, linalg::DeviceType device = linalg::CPU, DistType DT = DistType::NONE>
class TpAccumulator;

template <class Parameters>
class TpAccumulator<Parameters, linalg::CPU, dca::DistType::NONE> {
template <class Parameters, DistType DT>
class TpAccumulator<Parameters, linalg::CPU, DT> {
public:
using Real = typename Parameters::TP_measurement_scalar_type;

Expand All @@ -63,7 +63,6 @@ class TpAccumulator<Parameters, linalg::CPU, dca::DistType::NONE> {
using WTpExtPosDmn = func::dmn_0<domains::vertex_frequency_domain<domains::EXTENDED_POSITIVE>>;
using WExchangeDmn = func::dmn_0<domains::FrequencyExchangeDomain>;

using this_type = TpAccumulator<Parameters>;
using Data = DcaData<Parameters>;
using TpGreensFunction = typename Data::TpGreensFunction;

Expand Down Expand Up @@ -109,7 +108,7 @@ class TpAccumulator<Parameters, linalg::CPU, dca::DistType::NONE> {
const auto& get_sign_times_G4() const;

// Sums the accumulated Green's function to the accumulated Green's function of other_acc.
void sumTo(this_type& other_acc);
void sumTo(TpAccumulator& other_acc);

void synchronizeCopy() {}

Expand All @@ -134,7 +133,7 @@ class TpAccumulator<Parameters, linalg::CPU, dca::DistType::NONE> {

void getGMultiband(int s, int k1, int k2, int w1, int w2, Matrix& G, Complex beta = 0) const;

Complex getGSingleband(int s, int k1, int k2, int w1, int w2) const;
auto getGSingleband(int s, int k1, int k2, int w1, int w2) -> Complex const;

template <class Configuration, typename RealIn>
float computeM(const std::array<linalg::Matrix<RealIn, linalg::CPU>, 2>& M_pair,
Expand Down Expand Up @@ -181,8 +180,8 @@ class TpAccumulator<Parameters, linalg::CPU, dca::DistType::NONE> {
Matrix G0_M_, G_a_, G_b_;
};

template <class Parameters>
TpAccumulator<Parameters, linalg::CPU>::TpAccumulator(
template <class Parameters, DistType DT>
TpAccumulator<Parameters, linalg::CPU, DT>::TpAccumulator(
const func::function<std::complex<double>, func::dmn_variadic<NuDmn, NuDmn, KDmn, WDmn>>& G0,
const Parameters& pars, const int thread_id)
: G0_ptr_(&G0),
Expand All @@ -195,6 +194,11 @@ TpAccumulator<Parameters, linalg::CPU>::TpAccumulator(
G0_M_(n_bands_),
G_a_(n_bands_),
G_b_(n_bands_) {
if constexpr (DT == DistType::MPI) {
std::cerr << "The MPI distribution of G4 on the CPU is not supported. Reverting to no "
"distribution.\n";
}

if (WDmn::dmn_size() < WTpExtDmn::dmn_size())
throw(std::logic_error("The number of single particle frequencies is too small."));
initializeG0();
Expand All @@ -207,16 +211,16 @@ TpAccumulator<Parameters, linalg::CPU>::TpAccumulator(
}
}

template <class Parameters>
void TpAccumulator<Parameters, linalg::CPU>::resetAccumulation(unsigned int /*dca_loop*/) {
template <class Parameters, DistType DT>
void TpAccumulator<Parameters, linalg::CPU, DT>::resetAccumulation(unsigned int /*dca_loop*/) {
for (auto& G4_channel : G4_)
G4_channel = 0.;

initializeG0();
}

template <class Parameters>
void TpAccumulator<Parameters, linalg::CPU>::initializeG0() {
template <class Parameters, DistType DT>
void TpAccumulator<Parameters, linalg::CPU, DT>::initializeG0() {
const int sp_index_offset = (WDmn::dmn_size() - WTpExtDmn::dmn_size()) / 2;

for (int w = 0; w < WTpExtDmn::dmn_size(); ++w) {
Expand All @@ -229,9 +233,9 @@ void TpAccumulator<Parameters, linalg::CPU>::initializeG0() {
}
}

template <class Parameters>
template <class Parameters, DistType DT>
template <class Configuration, typename RealIn>
double TpAccumulator<Parameters, linalg::CPU>::accumulate(
double TpAccumulator<Parameters, linalg::CPU, DT>::accumulate(
const std::array<linalg::Matrix<RealIn, linalg::CPU>, 2>& M_pair,
const std::array<Configuration, 2>& configs, const int sign) {
Profiler profiler("accumulate", "tp-accumulation", __LINE__, thread_id_);
Expand All @@ -249,9 +253,9 @@ double TpAccumulator<Parameters, linalg::CPU>::accumulate(
return gflops;
}

template <class Parameters>
template <class Parameters, DistType DT>
template <class Configuration, typename RealIn>
float TpAccumulator<Parameters, linalg::CPU>::computeM(
float TpAccumulator<Parameters, linalg::CPU, DT>::computeM(
const std::array<linalg::Matrix<RealIn, linalg::CPU>, 2>& M_pair,
const std::array<Configuration, 2>& configs) {
float flops = 0.;
Expand All @@ -272,8 +276,8 @@ float TpAccumulator<Parameters, linalg::CPU>::computeM(
return flops;
}

template <class Parameters>
double TpAccumulator<Parameters, linalg::CPU>::computeG() {
template <class Parameters, DistType DT>
double TpAccumulator<Parameters, linalg::CPU, DT>::computeG() {
Profiler prf("ComputeG", "tp-accumulation", __LINE__, thread_id_);
for (int w2 = 0; w2 < WTpExtDmn::dmn_size(); ++w2)
for (int w1 = 0; w1 < WTpExtPosDmn::dmn_size(); ++w1)
Expand All @@ -293,10 +297,10 @@ double TpAccumulator<Parameters, linalg::CPU>::computeG() {
return 1e-9 * flops;
}

template <class Parameters>
void TpAccumulator<Parameters, linalg::CPU>::computeGSingleband(const int s, const int k1,
const int k2, const int w1,
const int w2) {
template <class Parameters, DistType DT>
void TpAccumulator<Parameters, linalg::CPU, DT>::computeGSingleband(const int s, const int k1,
const int k2, const int w1,
const int w2) {
assert(w1 < WTpExtPosDmn::dmn_size());
assert(w2 < WTpExtDmn::dmn_size());

Expand All @@ -310,10 +314,10 @@ void TpAccumulator<Parameters, linalg::CPU>::computeGSingleband(const int s, con
G_(0, 0, s, k1, k2, w1, w2) = -G0_w1 * M_val * G0_w2;
}

template <class Parameters>
void TpAccumulator<Parameters, linalg::CPU>::computeGMultiband(const int s, const int k1,
const int k2, const int w1,
const int w2) {
template <class Parameters, DistType DT>
void TpAccumulator<Parameters, linalg::CPU, DT>::computeGMultiband(const int s, const int k1,
const int k2, const int w1,
const int w2) {
assert(w1 < WTpExtPosDmn::dmn_size());
assert(w2 < WTpExtDmn::dmn_size());

Expand All @@ -334,10 +338,10 @@ void TpAccumulator<Parameters, linalg::CPU>::computeGMultiband(const int s, cons
}
}

template <class Parameters>
std::complex<typename TpAccumulator<Parameters, linalg::CPU>::Real> TpAccumulator<
Parameters, linalg::CPU>::getGSingleband(const int s, const int k1, const int k2, const int w1,
const int w2) const {
template <class Parameters, DistType DT>
auto TpAccumulator<Parameters, linalg::CPU, DT>::getGSingleband(const int s, const int k1,
const int k2, const int w1,
const int w2) -> Complex const {
const int w2_ext = w2 + extension_index_offset_;
const int w1_ext = w1 + extension_index_offset_;
auto minus_w1 = [=](const int w) { return n_pos_frqs_ - 1 - w; };
Expand All @@ -354,9 +358,9 @@ std::complex<typename TpAccumulator<Parameters, linalg::CPU>::Real> TpAccumulato
return std::conj(G_(0, 0, s, minus_k(k1), minus_k(k2), minus_w1(w1_ext), minus_w2(w2_ext)));
}

template <class Parameters>
void TpAccumulator<Parameters, linalg::CPU>::getGMultiband(int s, int k1, int k2, int w1, int w2,
Matrix& G, const Complex beta) const {
template <class Parameters, DistType DT>
void TpAccumulator<Parameters, linalg::CPU, DT>::getGMultiband(int s, int k1, int k2, int w1, int w2,
Matrix& G, const Complex beta) const {
const int w2_ext = w2 + extension_index_offset_;
const int w1_ext = w1 + extension_index_offset_;
auto minus_w1 = [=](const int w) { return n_pos_frqs_ - 1 - w; };
Expand All @@ -383,8 +387,8 @@ void TpAccumulator<Parameters, linalg::CPU>::getGMultiband(int s, int k1, int k2
}
}

template <class Parameters>
double TpAccumulator<Parameters, linalg::CPU>::updateG4(const int channel_id) {
template <class Parameters, DistType DT>
double TpAccumulator<Parameters, linalg::CPU, DT>::updateG4(const int channel_id) {
// G4 is stored with the following band convention:
// b1 ------------------------ b3
// | |
Expand Down Expand Up @@ -575,8 +579,8 @@ double TpAccumulator<Parameters, linalg::CPU>::updateG4(const int channel_id) {
return 1e-9 * flops;
}

template <class Parameters>
void TpAccumulator<Parameters, linalg::CPU>::updateG4Atomic(
template <class Parameters, DistType DT>
void TpAccumulator<Parameters, linalg::CPU, DT>::updateG4Atomic(
Complex* G4_ptr, const int s_a, const int k1_a, const int k2_a, const int w1_a, const int w2_a,
const int s_b, const int k1_b, const int k2_b, const int w1_b, const int w2_b, const Real alpha,
const bool cross_legs) {
Expand Down Expand Up @@ -612,8 +616,8 @@ void TpAccumulator<Parameters, linalg::CPU>::updateG4Atomic(
}
}

template <class Parameters>
void TpAccumulator<Parameters, linalg::CPU>::updateG4SpinDifference(
template <class Parameters, DistType DT>
void TpAccumulator<Parameters, linalg::CPU, DT>::updateG4SpinDifference(
Complex* G4_ptr, const int sign, const int k1_a, const int k2_a, const int w1_a, const int w2_a,
const int k1_b, const int k2_b, const int w1_b, const int w2_b, const Real alpha,
const bool cross_legs) {
Expand Down Expand Up @@ -655,16 +659,16 @@ void TpAccumulator<Parameters, linalg::CPU>::updateG4SpinDifference(
}
}

template <class Parameters>
const auto& TpAccumulator<Parameters, linalg::CPU>::get_sign_times_G4() const {
template <class Parameters, DistType DT>
const auto& TpAccumulator<Parameters, linalg::CPU, DT>::get_sign_times_G4() const {
if (G4_.empty())
throw std::logic_error("There is no G4 stored in this class.");

return G4_;
}

template <class Parameters>
void TpAccumulator<Parameters, linalg::CPU>::sumTo(this_type& other_one) {
template <class Parameters, DistType DT>
void TpAccumulator<Parameters, linalg::CPU, DT>::sumTo(TpAccumulator& other_one) {
if (other_one.G4_.size() != G4_.size())
throw std::logic_error("Objects accumulate different number of channels.");

Expand Down
2 changes: 0 additions & 2 deletions include/dca/phys/dca_step/cluster_solver/shared_tools/accumulation/tp/tp_accumulator_gpu.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,6 @@
#include "dca/phys/dca_step/cluster_solver/shared_tools/accumulation/tp/ndft/cached_ndft_gpu.hpp"
#include "dca/util/integer_division.hpp"

#include "dca/parallel/mpi_concurrency/mpi_type_map.hpp"

namespace dca {
namespace phys {
namespace solver {
Expand Down
Loading

0 comments on commit 6fcfcb4

Please sign in to comment.