dune-fem  2.8-git
mpimanager.hh
Go to the documentation of this file.
1 #ifndef DUNE_FEM_MPIMANAGER_HH
2 #define DUNE_FEM_MPIMANAGER_HH
3 
4 #include <memory>
5 
6 #include <dune/common/parallel/mpicommunication.hh>
7 #include <dune/common/parallel/mpihelper.hh>
8 
10 
11 #if HAVE_PETSC
13 #endif
14 
16 
17 namespace Dune
18 {
19 
20  namespace Fem
21  {
22 
23  struct MPIManager
24  {
25  typedef Dune::CollectiveCommunication< MPIHelper::MPICommunicator >
27  private:
28  static MPIManager &instance ()
29  {
31  }
32 
33  static bool mpiFinalized ()
34  {
35  bool finalized = false ;
36 #if HAVE_MPI
37  // check that MPI was not already finalized
38  {
39  int wasFinalized = -1;
40  MPI_Finalized( &wasFinalized );
41  finalized = bool( wasFinalized );
42  }
43 #endif // #if HAVE_MPI
44  return finalized ;
45  }
46 
47 #if HAVE_PETSC
48  struct PETSc
49  {
50  ~PETSc()
51  {
52  if( ! mpiFinalized() )
53  {
54  ::Dune::Petsc::finalize();
55  }
56  }
57 
58  static void initialize( const bool verbose, int &argc, char **&argv )
59  {
60  // needed for later calling Petsc::finalize to the right time
62  ::Dune::Petsc::initialize( verbose, argc, argv );
63  }
64  };
65 #endif // #if HAVE_PETSC
66 
67  public:
69  {
70 #if HAVE_MPI
71  // if MPI_Init was called here and finalize has not been
72  // called yet, then this is the place to call it
73  if( wasInitializedHere_ && !mpiFinalized() )
74  {
75  MPI_Finalize();
76  }
77 #endif
78  }
79 
80  static void initialize ( int &argc, char **&argv )
81  {
82  MPIHelper *&helper = instance().helper_;
83  std::unique_ptr< CollectiveCommunication > &comm = instance().comm_;
84 
85  // the following initialization overrides the MPI_Init in dune-common
86  // to avoid a call to MPI_Finalize before all singletons have been deleted
87 #if HAVE_MPI
88  int wasInitialized = -1;
89  MPI_Initialized( &wasInitialized );
90  if(!wasInitialized)
91  {
92 #ifndef USE_SMP_PARALLEL
93  // standard MPI_Init
94  // call normal MPI_Init here to prevent MPIHelper to interfering
95  // with MPI_Finalize one program exit which would cause failure
96  {
97  int is_initialized = MPI_Init(&argc, &argv);
98  if( is_initialized != MPI_SUCCESS )
99  DUNE_THROW(InvalidStateException,"MPI_Init failed!");
100  }
101 #else // threaded init
102  {
103  int provided;
104  // use MPI_Init_thread for hybrid parallel programs
105  int is_initialized = MPI_Init_thread(&argc, &argv, MPI_THREAD_FUNNELED, &provided );
106 
107  if( is_initialized != MPI_SUCCESS )
108  DUNE_THROW(InvalidStateException,"MPI_Init_thread failed!");
109 
110 #if not defined NDEBUG && defined DUNE_DEVEL_MODE
111  // for OpenMPI provided seems to be MPI_THREAD_SINGLE
112  // but the bybrid version still works. On BlueGene systems
113  // the MPI_THREAD_FUNNELED is really needed
114  if( provided != MPI_THREAD_FUNNELED )
115  {
116  if( provided == MPI_THREAD_SINGLE )
117  dwarn << "MPI thread support = single (instead of funneled)!" << std::endl;
118  else
119  dwarn << "WARNING: MPI thread support = " << provided << " != MPI_THREAD_FUNNELED " << MPI_THREAD_FUNNELED << std::endl;
120  }
121 #endif // end NDEBUG
122  }
123 #endif // end USE_SMP_PARALLEL
124  instance().wasInitializedHere_ = true;
125 
126  } // end if(!wasInitialized)
127 #endif // end HAVE_MPI
128 
129  // if already initialized, do nothing further
130  if( helper && comm )
131  return ;
132 
133  // this will just initialize the static variables inside MPIHelper but
134  // not call MPI_Init again
135  helper = &MPIHelper::instance( argc, argv );
136  comm.reset( new CollectiveCommunication( helper->getCommunicator() ) );
137 
138 #if HAVE_PETSC
139  // initialize PETSc if pressent
140  PETSc::initialize( rank() == 0, argc, argv );
141 #endif
142 
143  // initialize static variables of QuadratureStorageRegistry
145  }
146 
147  static const CollectiveCommunication &comm ()
148  {
149  const std::unique_ptr< CollectiveCommunication > &comm = instance().comm_;
150  if( !comm )
151  DUNE_THROW( InvalidStateException, "MPIManager has not been initialized." );
152  return *comm;
153  }
154 
155  static int rank ()
156  {
157  return comm().rank();
158  }
159 
160  static int size ()
161  {
162  return comm().size();
163  }
164 
165  private:
166  MPIHelper *helper_ = nullptr;
167  std::unique_ptr< CollectiveCommunication > comm_;
168  bool wasInitializedHere_ = false ;
169  };
170 
171  } // namespace Fem
172 
173 } // namespace Dune
174 
175 #endif // #ifndef DUNE_FEM_MPIMANAGER_HH
Definition: bindguard.hh:11
Definition: mpimanager.hh:24
static const CollectiveCommunication & comm()
Definition: mpimanager.hh:147
Dune::CollectiveCommunication< MPIHelper::MPICommunicator > CollectiveCommunication
Definition: mpimanager.hh:26
~MPIManager()
Definition: mpimanager.hh:68
static int size()
Definition: mpimanager.hh:160
static int rank()
Definition: mpimanager.hh:155
static void initialize(int &argc, char **&argv)
Definition: mpimanager.hh:80
static void initialize()
initialize static variables
Definition: registry.hh:62
static Object & instance(Args &&... args)
return singleton instance of given Object type.
Definition: singleton.hh:101