diff --git a/opm/grid/common/MetisPartition.cpp b/opm/grid/common/MetisPartition.cpp index 612d69e49..0a1061f7d 100644 --- a/opm/grid/common/MetisPartition.cpp +++ b/opm/grid/common/MetisPartition.cpp @@ -156,14 +156,20 @@ metisSerialGraphPartitionGridOnRoot(const CpGrid& cpgrid, #endif std::shared_ptr gridAndWells; - if( wells ) - { + if (allowDistributedWells) { gridAndWells.reset(new CombinedGridWellGraph(cpgrid, - wells, - possibleFutureConnections, - transmissibilities, - false, - edgeWeightsMethod)); + nullptr, // if we allow distributed wells, we construct the CombinedGridWellGraph without the influence of the wells, just the transmissibilities + {}, + transmissibilities, + false, + edgeWeightsMethod)); + } else { + gridAndWells.reset(new CombinedGridWellGraph(cpgrid, + wells, // if we allow distributed wells, we construct the CombinedGridWellGraph with the wells, possible future connections and the transmissibilities + possibleFutureConnections, + transmissibilities, + false, + edgeWeightsMethod)); } std::vector partitionVector; @@ -236,17 +242,8 @@ metisSerialGraphPartitionGridOnRoot(const CpGrid& cpgrid, //////// Now, we define all variables that *do depend* on whether there are wells or not - if( wells ) - { - for (int i = 0; i < n; i++) { - xadj[i+1] = xadj[i] + Dune::cpgrid::getNumberOfEdgesForSpecificCellForGridWithWells(*gridAndWells, lids[i]); - } - } - else - { - for (int i = 0; i < n; i++) { - xadj[i+1] = xadj[i] + Dune::cpgrid::getNumberOfEdgesForSpecificCell(cpgrid, lids[i]); - } + for (int i = 0; i < n; i++) { + xadj[i+1] = xadj[i] + Dune::cpgrid::getNumberOfEdgesForSpecificCell(*gridAndWells, lids[i]); } // The number of edges depends on whether there are wells or not, twoM = 2*m, where m = number of edges. @@ -263,21 +260,10 @@ metisSerialGraphPartitionGridOnRoot(const CpGrid& cpgrid, // The weights of the edges (if any) are stored in an additional array called adjwgt. This array contains 2m elements, and the weight of edge adjncy[j] is stored at location adjwgt[j] idx_t* adjwgt = new idx_t[twoM]; - if( wells ) + int neighborCounter = 0; + for( int cell = 0; cell < n; cell++ ) { - int neighborCounter = 0; - for( int cell = 0; cell < n; cell++ ) - { - fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounterForGridWithWells(*gridAndWells, lids[cell], gids, neighborCounter, adjncy, adjwgt); - } - } - else - { - int neighborCounter = 0; - for( int cell = 0; cell < n; cell++ ) - { - fillNBORGIDForSpecificCellAndIncrementNeighborCounter(cpgrid, lids[cell], gids, neighborCounter, adjncy); - } + fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounter(*gridAndWells, lids[cell], gids, neighborCounter, adjncy, adjwgt); } // Decide which partition method to use, both methods create k partitions, where @@ -296,7 +282,7 @@ metisSerialGraphPartitionGridOnRoot(const CpGrid& cpgrid, adjncy, nullptr, // vwgt nullptr, // vsize, - wells ? adjwgt : nullptr, + adjwgt, &nparts, nullptr, // tpwgts, &ubvec, @@ -311,7 +297,7 @@ metisSerialGraphPartitionGridOnRoot(const CpGrid& cpgrid, adjncy, nullptr, // vwgt nullptr, // vsize, - wells ? adjwgt : nullptr, + adjwgt, &nparts, nullptr, // tpwgts, &ubvec, diff --git a/opm/grid/common/ZoltanGraphFunctions.cpp b/opm/grid/common/ZoltanGraphFunctions.cpp index fdcd5384a..c84b1a5b8 100644 --- a/opm/grid/common/ZoltanGraphFunctions.cpp +++ b/opm/grid/common/ZoltanGraphFunctions.cpp @@ -88,42 +88,8 @@ void getNullNumEdgesList(void *cpGridPointer, int sizeGID, int sizeLID, *err = ZOLTAN_OK; } -int getNumberOfEdgesForSpecificCell(const Dune::CpGrid& grid, int localCellId) { - // For the graph there is an edge only if the face has two neighbors. - // Therefore we need to check each face - int edges = 0; - for ( int local_face = 0; local_face < grid.numCellFaces(localCellId); ++local_face ) - { - const int face = grid.cellFace(localCellId, local_face); - if ( grid.faceCell(face, 0) != -1 && grid.faceCell(face, 1) != -1 ) - { - ++edges; - } - } - return edges; -} -void getCpGridNumEdgesList(void *cpGridPointer, int sizeGID, int sizeLID, - int numCells, - ZOLTAN_ID_PTR globalID, ZOLTAN_ID_PTR localID, - int *numEdges, int *err) -{ - (void) globalID; - const Dune::CpGrid& grid = *static_cast(cpGridPointer); - if ( sizeGID != 1 || sizeLID != 1 || numCells != grid.numCells() ) - { - *err = ZOLTAN_FATAL; - return; - } - for( int i = 0; i < numCells; i++ ) - { - numEdges[i] = getNumberOfEdgesForSpecificCell(grid, localID[i]); - } - - *err = ZOLTAN_OK; -} - -int getNumberOfEdgesForSpecificCellForGridWithWells(const CombinedGridWellGraph& graph, int localCellId) { +int getNumberOfEdgesForSpecificCell(const CombinedGridWellGraph& graph, int localCellId) { const Dune::CpGrid& grid = graph.getGrid(); // Initial set of faces is the ones of the well completions auto edges = graph.getWellsGraph()[localCellId]; @@ -162,7 +128,7 @@ void getCpGridWellsNumEdgesList(void *graphPointer, int sizeGID, int sizeLID, } for( int i = 0; i < numCells; i++ ) { - numEdges[i] = getNumberOfEdgesForSpecificCellForGridWithWells(graph, localID[i]); + numEdges[i] = getNumberOfEdgesForSpecificCell(graph, localID[i]); } *err = ZOLTAN_OK; } @@ -179,79 +145,15 @@ void getNullEdgeList(void *cpGridPointer, int sizeGID, int sizeLID, *err = ZOLTAN_OK; } -template -void fillNBORGIDForSpecificCellAndIncrementNeighborCounter(const Dune::CpGrid& grid, int localCellId, ID globalID, int& neighborCounter, ID& nborGID) { - for ( int local_face = 0 ; local_face < grid.numCellFaces(localCellId); ++local_face ) - { - const int face = grid.cellFace(localCellId, local_face); - int otherCell = grid.faceCell(face, 0); - if ( otherCell == localCellId || otherCell == -1 ) - { - otherCell = grid.faceCell(face, 1); - if ( otherCell == localCellId || otherCell == -1 ) - continue; - } - nborGID[neighborCounter++] = globalID[otherCell]; - } -} - -void getCpGridEdgeList(void *cpGridPointer, int sizeGID, int sizeLID, - int numCells, ZOLTAN_ID_PTR globalID, ZOLTAN_ID_PTR localID, - int *numEdges, - ZOLTAN_ID_PTR nborGID, int *nborProc, - int wgtDim, float *ewgts, int *err) -{ - (void) wgtDim; (void) globalID; (void) numEdges; (void) ewgts; - const Dune::CpGrid& grid = *static_cast(cpGridPointer); - if ( sizeGID != 1 || sizeLID != 1 || numCells != grid.numCells() ) - { - *err = ZOLTAN_FATAL; - return; - } -#ifndef NDEBUG - int oldNeighborCounter = 0; -#endif - int neighborCounter = 0; - - for( int cell = 0; cell < numCells; cell++ ) - { - fillNBORGIDForSpecificCellAndIncrementNeighborCounter(grid, localID[cell], globalID, neighborCounter, nborGID); -#ifndef NDEBUG - assert(numEdges[cell] == neighborCounter - oldNeighborCounter); - oldNeighborCounter = neighborCounter; -#endif - } - - const int myrank = grid.comm().rank(); - - for ( int i = 0; i < neighborCounter; ++i ) - { - nborProc[i] = myrank; - } -#if defined(DEBUG) && false // The index set will not be initialized here! - // The above relies heavily on the grid not being distributed already. - // Therefore we check here that all cells are owned by us. - GlobalLookupIndexSet - globalIdxSet(grid.getCellIndexSet(), - grid.numCells()); - for ( int cell = 0; cell < numCells; cell++ ) - { - if ( globalIdxSet.pair(cell)->local().attribute() != - Dune::CpGrid::ParallelIndexSet::LocalIndex::Attribute::owner ) - { - *err = ZOLTAN_FATAL; - } - } -#endif -} template -void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounterForGridWithWells(const CombinedGridWellGraph& graph, const int localCellId, ID globalID, int& neighborCounter, ID& nborGID, weightType *ewgts) { +void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounter(const CombinedGridWellGraph& graph, const int localCellId, ID globalID, int& neighborCounter, ID& nborGID, weightType *ewgts) { const Dune::CpGrid& grid = graph.getGrid(); // First the strong edges of the well completions. auto wellEdges = graph.getWellsGraph()[localCellId]; for( auto edge : wellEdges) { nborGID[neighborCounter] = edge; + std::cout << "Setting weightType = max" << std::endl; ewgts[neighborCounter++] = std::numeric_limits::max(); } @@ -310,7 +212,7 @@ void getCpGridWellsEdgeList(void *graphPointer, int sizeGID, int sizeLID, for( int cell = 0; cell < numCells; cell++ ) { - fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounterForGridWithWells(graph, localID[cell], globalID, neighborCounter, nborGID, ewgts); + fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounter(graph, localID[cell], globalID, neighborCounter, nborGID, ewgts); #ifndef NDEBUG assert(neighborCounter-oldNeighborCounter==numEdges[cell]); oldNeighborCounter = neighborCounter; @@ -353,6 +255,8 @@ CombinedGridWellGraph::CombinedGridWellGraph(const CpGrid& grid, // wellsGraph not needed return; } + if (edgeWeightsMethod == logTransEdgeWgt) + findMaxMinTrans(); wellsGraph_.resize(grid.numCells()); const auto& cpgdim = grid.logicalCartesianSize(); // create compressed lookup from cartesian. @@ -362,33 +266,17 @@ CombinedGridWellGraph::CombinedGridWellGraph(const CpGrid& grid, { cartesian_to_compressed[grid.globalCell()[i]] = i; } - well_indices_.init(*wells, possibleFutureConnections, cpgdim, cartesian_to_compressed); + if (wells) { + // If we're not taking the wells into account here we don't init the connections + well_indices_.init(*wells, possibleFutureConnections, cpgdim, cartesian_to_compressed); + addCompletionSetToGraph(); + } else { + // std::vector emptyWells = {}; + // well_indices_.init({}, {}, cpgdim, cartesian_to_compressed); + } std::vector().swap(cartesian_to_compressed); // free memory. - addCompletionSetToGraph(); - - if (edgeWeightsMethod == logTransEdgeWgt) - findMaxMinTrans(); } -void setCpGridZoltanGraphFunctions(Zoltan_Struct *zz, const Dune::CpGrid& grid, - bool pretendNull) -{ - Dune::CpGrid *gridPointer = const_cast(&grid); - if ( pretendNull ) - { - Zoltan_Set_Num_Obj_Fn(zz, getNullNumCells, gridPointer); - Zoltan_Set_Obj_List_Fn(zz, getNullVertexList, gridPointer); - Zoltan_Set_Num_Edges_Multi_Fn(zz, getNullNumEdgesList, gridPointer); - Zoltan_Set_Edge_List_Multi_Fn(zz, getNullEdgeList, gridPointer); - } - else - { - Zoltan_Set_Num_Obj_Fn(zz, getCpGridNumCells, gridPointer); - Zoltan_Set_Obj_List_Fn(zz, getCpGridVertexList, gridPointer); - Zoltan_Set_Num_Edges_Multi_Fn(zz, getCpGridNumEdgesList, gridPointer); - Zoltan_Set_Edge_List_Multi_Fn(zz, getCpGridEdgeList, gridPointer); - } -} void setCpGridZoltanGraphFunctions(Zoltan_Struct *zz, const CombinedGridWellGraph& graph, @@ -414,14 +302,10 @@ void setCpGridZoltanGraphFunctions(Zoltan_Struct *zz, // Explicit template instantiation for METIS #if HAVE_METIS template -void fillNBORGIDForSpecificCellAndIncrementNeighborCounter(const Dune::CpGrid&, int, int*, int&, int*& nborGID); -template -void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounterForGridWithWells(const CombinedGridWellGraph&, const int, int*, int&, int*&, int*); +void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounter(const CombinedGridWellGraph&, const int, int*, int&, int*&, int*); template -void fillNBORGIDForSpecificCellAndIncrementNeighborCounter(Dune::CpGrid const&, int, long*, int&, long*&); -template -void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounterForGridWithWells(Dune::cpgrid::CombinedGridWellGraph const&, int, long*, int&, long*&, long*); +void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounter(Dune::cpgrid::CombinedGridWellGraph const&, int, long*, int&, long*&, long*); #endif diff --git a/opm/grid/common/ZoltanGraphFunctions.hpp b/opm/grid/common/ZoltanGraphFunctions.hpp index e73c00400..92233f45c 100644 --- a/opm/grid/common/ZoltanGraphFunctions.hpp +++ b/opm/grid/common/ZoltanGraphFunctions.hpp @@ -58,25 +58,8 @@ void getCpGridVertexList(void* cpGridPointer, int numGlobalIds, ZOLTAN_ID_PTR lids, int wgtDim, float *objWgts, int *err); -/// \brief Get the number of edges for one vertex of the graph of the grid. -int getNumberOfEdgesForSpecificCell(const Dune::CpGrid& grid, int localCellId); /// \brief Get the number of edges of the graph of the grid. -void getCpGridNumEdgesList(void *cpGridPointer, int sizeGID, int sizeLID, - int numCells, - ZOLTAN_ID_PTR globalID, ZOLTAN_ID_PTR localID, - int *numEdges, int *err); - -/// \brief Get the list of edges for one cell of a grid witout wells -template -void fillNBORGIDForSpecificCellAndIncrementNeighborCounter(const Dune::CpGrid& grid, int localCellId, ID globalID, int& neighborCounter, ID& nborGID); - -/// \brief Get the list of edges of the graph of the grid. -void getCpGridEdgeList(void *cpGridPointer, int sizeGID, int sizeLID, - int numCells, ZOLTAN_ID_PTR globalID, ZOLTAN_ID_PTR localID, - int *num_edges, - ZOLTAN_ID_PTR nborGID, int *nborProc, - int wgt_dim, float *ewgts, int *err); /// \brief Get a list of vertices with zero enties void getNullVertexList(void* cpGridPointer, int numGlobalIds, @@ -243,19 +226,17 @@ class CombinedGridWellGraph }; /// \brief Get the number of edges of the graph of the grid and the wells for one cell -int getNumberOfEdgesForSpecificCellForGridWithWells(const CombinedGridWellGraph& graph, int localCellId); +int getNumberOfEdgesForSpecificCell(const CombinedGridWellGraph& graph, int localCellId); /// \brief Get the list of edges and weights for one cell of a grid with wells template -void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounterForGridWithWells(const CombinedGridWellGraph& graph, const int localCellId, ID globalID, int& neighborCounter, ID& nborGID, weightType *ewgts); +void fillNBORGIDAndWeightsForSpecificCellAndIncrementNeighborCounter(const CombinedGridWellGraph& graph, const int localCellId, ID globalID, int& neighborCounter, ID& nborGID, weightType *ewgts); #ifdef HAVE_ZOLTAN /// \brief Sets up the call-back functions for ZOLTAN's graph partitioning. /// \param zz The struct with the information for ZOLTAN. /// \param grid The grid to partition. /// \param pretendNull If true, we will pretend that the grid has zero cells. -void setCpGridZoltanGraphFunctions(Zoltan_Struct *zz, const Dune::CpGrid& grid, - bool pretendNull=false); void setCpGridZoltanGraphFunctions(Zoltan_Struct *zz, const CombinedGridWellGraph& graph, diff --git a/opm/grid/common/ZoltanPartition.cpp b/opm/grid/common/ZoltanPartition.cpp index f18e8432b..0aa91826e 100644 --- a/opm/grid/common/ZoltanPartition.cpp +++ b/opm/grid/common/ZoltanPartition.cpp @@ -106,10 +106,12 @@ makeImportAndExportLists(const Dune::CpGrid& cpgrid, { if (allowDistributedWells) { + std::cout << "We have wells, and allowDistributedWells is true!" << std::endl; wellsOnProc = perforatingWellIndicesOnProc(parts, *wells, possibleFutureConnections, cpgrid); } else { + std::cout << "We have wells, and allowDistributedWells is false!" << std::endl; auto gidGetter = [&cpgrid](int i) { return cpgrid.globalIdSet().id(Dune::createEntity<0>(cpgrid, i, true));}; wellsOnProc = postProcessPartitioningForWells(parts, @@ -153,6 +155,8 @@ makeImportAndExportLists(const Dune::CpGrid& cpgrid, #endif } + } else { + std::cout << "We have no wells!" << std::endl; } std::vector> parallel_wells; @@ -332,23 +336,26 @@ zoltanGraphPartitionGridOnRoot(const CpGrid& cpgrid, std::shared_ptr gridAndWells; - if( wells ) - { - Zoltan_Set_Param(zz,"EDGE_WEIGHT_DIM","1"); + Zoltan_Set_Param(zz,"EDGE_WEIGHT_DIM","1"); + if (allowDistributedWells) { gridAndWells.reset(new CombinedGridWellGraph(cpgrid, - wells, - possibleFutureConnections, - transmissibilities, - partitionIsEmpty, - edgeWeightsMethod)); - Dune::cpgrid::setCpGridZoltanGraphFunctions(zz, *gridAndWells, - partitionIsEmpty); - } - else - { - Dune::cpgrid::setCpGridZoltanGraphFunctions(zz, cpgrid, partitionIsEmpty); + nullptr, + {}, + transmissibilities, + partitionIsEmpty, + edgeWeightsMethod)); + Dune::cpgrid::setCpGridZoltanGraphFunctions(zz, *gridAndWells, partitionIsEmpty); + } else { + gridAndWells.reset(new CombinedGridWellGraph(cpgrid, + wells, + possibleFutureConnections, + transmissibilities, + partitionIsEmpty, + edgeWeightsMethod)); + Dune::cpgrid::setCpGridZoltanGraphFunctions(zz, *gridAndWells, partitionIsEmpty); } + rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */ &changes, /* 1 if partitioning was changed, 0 otherwise */ &numGidEntries, /* Number of integers used for a global ID */ @@ -414,7 +421,11 @@ class ZoltanSerialPartitioner , numParts(_numParts) , params(param) { - if (wells) { + if (allowDistributedWells) { + const bool partitionIsEmpty = cc.rank() != root; + gridAndWells.reset( + new CombinedGridWellGraph(cpgrid, nullptr, {}, transmissibilities, partitionIsEmpty, edgeWeightsMethod)); + } else { const bool partitionIsEmpty = cc.rank() != root; gridAndWells.reset( new CombinedGridWellGraph(cpgrid, wells, possibleFutureConnections, transmissibilities, partitionIsEmpty, edgeWeightsMethod)); @@ -519,12 +530,8 @@ class ZoltanSerialPartitioner // all others an empty partition before loadbalancing. bool partitionIsEmpty = cc.rank() != root; - if (wells) { - Zoltan_Set_Param(zz, "EDGE_WEIGHT_DIM", "1"); - Dune::cpgrid::setCpGridZoltanGraphFunctions(zz, *gridAndWells, partitionIsEmpty); - } else { - Dune::cpgrid::setCpGridZoltanGraphFunctions(zz, cpgrid, partitionIsEmpty); - } + Zoltan_Set_Param(zz, "EDGE_WEIGHT_DIM", "1"); + Dune::cpgrid::setCpGridZoltanGraphFunctions(zz, *gridAndWells, partitionIsEmpty); rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */ &changes, /* 1 if partitioning was changed, 0 otherwise */