Skip to content

Commit

Permalink
Use only transmissibilities when allow-distributed-wells=true
Browse files Browse the repository at this point in the history
  • Loading branch information
@lisajulia
lisajulia committed Jan 22, 2025
1 parent 6db542b commit 092367a
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 209 deletions.
54 changes: 20 additions & 34 deletions opm/grid/common/MetisPartition.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,14 +156,20 @@ metisSerialGraphPartitionGridOnRoot(const CpGrid& cpgrid,
#endif

std::shared_ptr<CombinedGridWellGraph> gridAndWells;
if( wells )
{
if (allowDistributedWells) {
gridAndWells.reset(new CombinedGridWellGraph(cpgrid,
wells,
possibleFutureConnections,
transmissibilities,
false,
edgeWeightsMethod));
nullptr, // if we allow distributed wells, we construct the CombinedGridWellGraph without the influence of the wells, just the transmissibilities
{},
transmissibilities,
false,
edgeWeightsMethod));
} else {
gridAndWells.reset(new CombinedGridWellGraph(cpgrid,
wells, // if we allow distributed wells, we construct the CombinedGridWellGraph with the wells, possible future connections and the transmissibilities
possibleFutureConnections,
transmissibilities,
false,
edgeWeightsMethod));
}

std::vector<int> partitionVector;
Expand Down Expand Up @@ -236,17 +242,8 @@ metisSerialGraphPartitionGridOnRoot(const CpGrid& cpgrid,

//////// Now, we define all variables that *do depend* on whether there are wells or not

if( wells )
{
for (int i = 0; i < n; i++) {
xadj[i+1] = xadj[i] + Dune::cpgrid::getNumberOfEdgesForSpecificCellForGridWithWells(*gridAndWells, lids[i]);
}
}
else
{
for (int i = 0; i < n; i++) {
xadj[i+1] = xadj[i] + Dune::cpgrid::getNumberOfEdgesForSpecificCell(cpgrid, lids[i]);
}
for (int i = 0; i < n; i++) {
xadj[i+1] = xadj[i] + Dune::cpgrid::getNumberOfEdgesForSpecificCell(*gridAndWells, lids[i]);
}

// The number of edges depends on whether there are wells or not, twoM = 2*m, where m = number of edges.
Expand All @@ -263,21 +260,10 @@ metisSerialGraphPartitionGridOnRoot(const CpGrid& cpgrid,
// The weights of the edges (if any) are stored in an additional array called adjwgt. This array contains 2m elements, and the weight of edge adjncy[j] is stored at location adjwgt[j]
idx_t* adjwgt = new idx_t[twoM];

if( wells )
int neighborCounter = 0;
for( int cell = 0; cell < n; cell++ )
{
int neighborCounter = 0;
for( int cell = 0; cell < n; cell++ )
{
fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounterForGridWithWells(*gridAndWells, lids[cell], gids, neighborCounter, adjncy, adjwgt);
}
}
else
{
int neighborCounter = 0;
for( int cell = 0; cell < n; cell++ )
{
fillNBORGIDForSpecificCellAndIncrementNeighborCounter(cpgrid, lids[cell], gids, neighborCounter, adjncy);
}
fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounter(*gridAndWells, lids[cell], gids, neighborCounter, adjncy, adjwgt);
}

// Decide which partition method to use, both methods create k partitions, where
Expand All @@ -296,7 +282,7 @@ metisSerialGraphPartitionGridOnRoot(const CpGrid& cpgrid,
adjncy,
nullptr, // vwgt
nullptr, // vsize,
wells ? adjwgt : nullptr,
adjwgt,
&nparts,
nullptr, // tpwgts,
&ubvec,
Expand All @@ -311,7 +297,7 @@ metisSerialGraphPartitionGridOnRoot(const CpGrid& cpgrid,
adjncy,
nullptr, // vwgt
nullptr, // vsize,
wells ? adjwgt : nullptr,
adjwgt,
&nparts,
nullptr, // tpwgts,
&ubvec,
Expand Down
150 changes: 17 additions & 133 deletions opm/grid/common/ZoltanGraphFunctions.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,42 +88,8 @@ void getNullNumEdgesList(void *cpGridPointer, int sizeGID, int sizeLID,
*err = ZOLTAN_OK;
}

int getNumberOfEdgesForSpecificCell(const Dune::CpGrid& grid, int localCellId) {
// For the graph there is an edge only if the face has two neighbors.
// Therefore we need to check each face
int edges = 0;
for ( int local_face = 0; local_face < grid.numCellFaces(localCellId); ++local_face )
{
const int face = grid.cellFace(localCellId, local_face);
if ( grid.faceCell(face, 0) != -1 && grid.faceCell(face, 1) != -1 )
{
++edges;
}
}
return edges;
}

void getCpGridNumEdgesList(void *cpGridPointer, int sizeGID, int sizeLID,
int numCells,
ZOLTAN_ID_PTR globalID, ZOLTAN_ID_PTR localID,
int *numEdges, int *err)
{
(void) globalID;
const Dune::CpGrid& grid = *static_cast<const Dune::CpGrid*>(cpGridPointer);
if ( sizeGID != 1 || sizeLID != 1 || numCells != grid.numCells() )
{
*err = ZOLTAN_FATAL;
return;
}
for( int i = 0; i < numCells; i++ )
{
numEdges[i] = getNumberOfEdgesForSpecificCell(grid, localID[i]);
}

*err = ZOLTAN_OK;
}

int getNumberOfEdgesForSpecificCellForGridWithWells(const CombinedGridWellGraph& graph, int localCellId) {
int getNumberOfEdgesForSpecificCell(const CombinedGridWellGraph& graph, int localCellId) {
const Dune::CpGrid& grid = graph.getGrid();
// Initial set of faces is the ones of the well completions
auto edges = graph.getWellsGraph()[localCellId];
Expand Down Expand Up @@ -162,7 +128,7 @@ void getCpGridWellsNumEdgesList(void *graphPointer, int sizeGID, int sizeLID,
}
for( int i = 0; i < numCells; i++ )
{
numEdges[i] = getNumberOfEdgesForSpecificCellForGridWithWells(graph, localID[i]);
numEdges[i] = getNumberOfEdgesForSpecificCell(graph, localID[i]);
}
*err = ZOLTAN_OK;
}
Expand All @@ -179,79 +145,15 @@ void getNullEdgeList(void *cpGridPointer, int sizeGID, int sizeLID,
*err = ZOLTAN_OK;
}

template<typename ID>
void fillNBORGIDForSpecificCellAndIncrementNeighborCounter(const Dune::CpGrid& grid, int localCellId, ID globalID, int& neighborCounter, ID& nborGID) {
for ( int local_face = 0 ; local_face < grid.numCellFaces(localCellId); ++local_face )
{
const int face = grid.cellFace(localCellId, local_face);
int otherCell = grid.faceCell(face, 0);
if ( otherCell == localCellId || otherCell == -1 )
{
otherCell = grid.faceCell(face, 1);
if ( otherCell == localCellId || otherCell == -1 )
continue;
}
nborGID[neighborCounter++] = globalID[otherCell];
}
}

void getCpGridEdgeList(void *cpGridPointer, int sizeGID, int sizeLID,
int numCells, ZOLTAN_ID_PTR globalID, ZOLTAN_ID_PTR localID,
int *numEdges,
ZOLTAN_ID_PTR nborGID, int *nborProc,
int wgtDim, float *ewgts, int *err)
{
(void) wgtDim; (void) globalID; (void) numEdges; (void) ewgts;
const Dune::CpGrid& grid = *static_cast<const Dune::CpGrid*>(cpGridPointer);
if ( sizeGID != 1 || sizeLID != 1 || numCells != grid.numCells() )
{
*err = ZOLTAN_FATAL;
return;
}
#ifndef NDEBUG
int oldNeighborCounter = 0;
#endif
int neighborCounter = 0;

for( int cell = 0; cell < numCells; cell++ )
{
fillNBORGIDForSpecificCellAndIncrementNeighborCounter(grid, localID[cell], globalID, neighborCounter, nborGID);
#ifndef NDEBUG
assert(numEdges[cell] == neighborCounter - oldNeighborCounter);
oldNeighborCounter = neighborCounter;
#endif
}

const int myrank = grid.comm().rank();

for ( int i = 0; i < neighborCounter; ++i )
{
nborProc[i] = myrank;
}
#if defined(DEBUG) && false // The index set will not be initialized here!
// The above relies heavily on the grid not being distributed already.
// Therefore we check here that all cells are owned by us.
GlobalLookupIndexSet<Dune::CpGrid::ParallelIndexSet>
globalIdxSet(grid.getCellIndexSet(),
grid.numCells());
for ( int cell = 0; cell < numCells; cell++ )
{
if ( globalIdxSet.pair(cell)->local().attribute() !=
Dune::CpGrid::ParallelIndexSet::LocalIndex::Attribute::owner )
{
*err = ZOLTAN_FATAL;
}
}
#endif
}
template<typename ID, typename weightType>
void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounterForGridWithWells(const CombinedGridWellGraph& graph, const int localCellId, ID globalID, int& neighborCounter, ID& nborGID, weightType *ewgts) {
void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounter(const CombinedGridWellGraph& graph, const int localCellId, ID globalID, int& neighborCounter, ID& nborGID, weightType *ewgts) {
const Dune::CpGrid& grid = graph.getGrid();
// First the strong edges of the well completions.
auto wellEdges = graph.getWellsGraph()[localCellId];
for( auto edge : wellEdges)
{
nborGID[neighborCounter] = edge;
std::cout << "Setting weightType = max" << std::endl;
ewgts[neighborCounter++] = std::numeric_limits<weightType>::max();
}

Expand Down Expand Up @@ -310,7 +212,7 @@ void getCpGridWellsEdgeList(void *graphPointer, int sizeGID, int sizeLID,

for( int cell = 0; cell < numCells; cell++ )
{
fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounterForGridWithWells(graph, localID[cell], globalID, neighborCounter, nborGID, ewgts);
fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounter(graph, localID[cell], globalID, neighborCounter, nborGID, ewgts);
#ifndef NDEBUG
assert(neighborCounter-oldNeighborCounter==numEdges[cell]);
oldNeighborCounter = neighborCounter;
Expand Down Expand Up @@ -353,6 +255,8 @@ CombinedGridWellGraph::CombinedGridWellGraph(const CpGrid& grid,
// wellsGraph not needed
return;
}
if (edgeWeightsMethod == logTransEdgeWgt)
findMaxMinTrans();
wellsGraph_.resize(grid.numCells());
const auto& cpgdim = grid.logicalCartesianSize();
// create compressed lookup from cartesian.
Expand All @@ -362,33 +266,17 @@ CombinedGridWellGraph::CombinedGridWellGraph(const CpGrid& grid,
{
cartesian_to_compressed[grid.globalCell()[i]] = i;
}
well_indices_.init(*wells, possibleFutureConnections, cpgdim, cartesian_to_compressed);
if (wells) {
// If we're not taking the wells into account here we don't init the connections
well_indices_.init(*wells, possibleFutureConnections, cpgdim, cartesian_to_compressed);
addCompletionSetToGraph();
} else {
// std::vector<OpmWellType> emptyWells = {};
// well_indices_.init({}, {}, cpgdim, cartesian_to_compressed);
}
std::vector<int>().swap(cartesian_to_compressed); // free memory.
addCompletionSetToGraph();

if (edgeWeightsMethod == logTransEdgeWgt)
findMaxMinTrans();
}

void setCpGridZoltanGraphFunctions(Zoltan_Struct *zz, const Dune::CpGrid& grid,
bool pretendNull)
{
Dune::CpGrid *gridPointer = const_cast<Dune::CpGrid*>(&grid);
if ( pretendNull )
{
Zoltan_Set_Num_Obj_Fn(zz, getNullNumCells, gridPointer);
Zoltan_Set_Obj_List_Fn(zz, getNullVertexList, gridPointer);
Zoltan_Set_Num_Edges_Multi_Fn(zz, getNullNumEdgesList, gridPointer);
Zoltan_Set_Edge_List_Multi_Fn(zz, getNullEdgeList, gridPointer);
}
else
{
Zoltan_Set_Num_Obj_Fn(zz, getCpGridNumCells, gridPointer);
Zoltan_Set_Obj_List_Fn(zz, getCpGridVertexList, gridPointer);
Zoltan_Set_Num_Edges_Multi_Fn(zz, getCpGridNumEdgesList, gridPointer);
Zoltan_Set_Edge_List_Multi_Fn(zz, getCpGridEdgeList, gridPointer);
}
}

void setCpGridZoltanGraphFunctions(Zoltan_Struct *zz,
const CombinedGridWellGraph& graph,
Expand All @@ -414,14 +302,10 @@ void setCpGridZoltanGraphFunctions(Zoltan_Struct *zz,
// Explicit template instantiation for METIS
#if HAVE_METIS
template
void fillNBORGIDForSpecificCellAndIncrementNeighborCounter(const Dune::CpGrid&, int, int*, int&, int*& nborGID);
template
void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounterForGridWithWells(const CombinedGridWellGraph&, const int, int*, int&, int*&, int*);
void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounter(const CombinedGridWellGraph&, const int, int*, int&, int*&, int*);

template
void fillNBORGIDForSpecificCellAndIncrementNeighborCounter(Dune::CpGrid const&, int, long*, int&, long*&);
template
void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounterForGridWithWells(Dune::cpgrid::CombinedGridWellGraph const&, int, long*, int&, long*&, long*);
void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounter(Dune::cpgrid::CombinedGridWellGraph const&, int, long*, int&, long*&, long*);

#endif

Expand Down
23 changes: 2 additions & 21 deletions opm/grid/common/ZoltanGraphFunctions.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,25 +58,8 @@ void getCpGridVertexList(void* cpGridPointer, int numGlobalIds,
ZOLTAN_ID_PTR lids, int wgtDim,
float *objWgts, int *err);

/// \brief Get the number of edges for one vertex of the graph of the grid.
int getNumberOfEdgesForSpecificCell(const Dune::CpGrid& grid, int localCellId);

/// \brief Get the number of edges of the graph of the grid.
void getCpGridNumEdgesList(void *cpGridPointer, int sizeGID, int sizeLID,
int numCells,
ZOLTAN_ID_PTR globalID, ZOLTAN_ID_PTR localID,
int *numEdges, int *err);

/// \brief Get the list of edges for one cell of a grid witout wells
template <typename ID>
void fillNBORGIDForSpecificCellAndIncrementNeighborCounter(const Dune::CpGrid& grid, int localCellId, ID globalID, int& neighborCounter, ID& nborGID);

/// \brief Get the list of edges of the graph of the grid.
void getCpGridEdgeList(void *cpGridPointer, int sizeGID, int sizeLID,
int numCells, ZOLTAN_ID_PTR globalID, ZOLTAN_ID_PTR localID,
int *num_edges,
ZOLTAN_ID_PTR nborGID, int *nborProc,
int wgt_dim, float *ewgts, int *err);

/// \brief Get a list of vertices with zero enties
void getNullVertexList(void* cpGridPointer, int numGlobalIds,
Expand Down Expand Up @@ -243,19 +226,17 @@ class CombinedGridWellGraph
};

/// \brief Get the number of edges of the graph of the grid and the wells for one cell
int getNumberOfEdgesForSpecificCellForGridWithWells(const CombinedGridWellGraph& graph, int localCellId);
int getNumberOfEdgesForSpecificCell(const CombinedGridWellGraph& graph, int localCellId);

/// \brief Get the list of edges and weights for one cell of a grid with wells
template<typename ID, typename weightType>
void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounterForGridWithWells(const CombinedGridWellGraph& graph, const int localCellId, ID globalID, int& neighborCounter, ID& nborGID, weightType *ewgts);
void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounter(const CombinedGridWellGraph& graph, const int localCellId, ID globalID, int& neighborCounter, ID& nborGID, weightType *ewgts);

#ifdef HAVE_ZOLTAN
/// \brief Sets up the call-back functions for ZOLTAN's graph partitioning.
/// \param zz The struct with the information for ZOLTAN.
/// \param grid The grid to partition.
/// \param pretendNull If true, we will pretend that the grid has zero cells.
void setCpGridZoltanGraphFunctions(Zoltan_Struct *zz, const Dune::CpGrid& grid,
bool pretendNull=false);

void setCpGridZoltanGraphFunctions(Zoltan_Struct *zz,
const CombinedGridWellGraph& graph,
Expand Down
Loading

0 comments on commit 092367a

Please sign in to comment.