dune-fem  2.8-git
domainthreaditerator.hh
Go to the documentation of this file.
1 #ifndef DUNE_FEM_DOMAINTHREADITERATOR_HH
2 #define DUNE_FEM_DOMAINTHREADITERATOR_HH
3 
4 #include <vector>
5 
6 #include <dune/common/exceptions.hh>
7 
13 
14 #ifdef USE_SMP_PARALLEL
15 #if HAVE_DUNE_ALUGRID
16 #define USE_THREADPARTITIONER
18 #endif
19 #endif
20 
21 namespace Dune {
22 
23  namespace Fem {
24 
26  template <class GridPart>
28  {
29  public:
30  typedef GridPart GridPartType;
31  typedef typename GridPartType :: GridType GridType;
32  typedef typename GridPartType :: IndexSetType IndexSetType;
33 
35 #ifdef USE_THREADPARTITIONER
36  typedef FilteredGridPart< GridPartType, FilterType, false > FilteredGridPartType ;
37 
38  typedef typename FilteredGridPartType :: template Codim< 0 > :: IteratorType
40 #else
41  typedef typename GridPartType :: template Codim< 0 > :: IteratorType IteratorType ;
42 #endif
43 
44  typedef typename IteratorType :: Entity EntityType ;
45 
47 
48  static const PartitionIteratorType pitype = GridPartType :: indexSetPartitionType ;
49 
50 #ifdef USE_THREADPARTITIONER
51  typedef ThreadPartitioner< GridPartType > ThreadPartitionerType;
52  typedef typename ThreadPartitionerType :: Method PartitioningMethodType ;
53 #endif // #ifdef USE_SMP_PARALLEL
54 
55  protected:
59 
60 #ifdef USE_THREADPARTITIONER
61  int sequence_;
62  std::vector< FilteredGridPartType* > filteredGridParts_;
63 
64  typedef typename FilterType :: DomainArrayType ThreadArrayType;
65  ThreadArrayType threadNum_;
66 #endif
67  // ratio of computing time needed by the master thread compared to the other threads
68  double masterRatio_ ;
69 
70 #ifdef USE_THREADPARTITIONER
71  const PartitioningMethodType method_;
72 #endif // #ifdef USE_SMP_PARALLEL
73 
74  // if true, thread 0 does only communication and no computation
76  const bool verbose_ ;
77 
78  protected:
79 #ifdef USE_THREADPARTITIONER
80  static PartitioningMethodType getMethod ( const ParameterReader &parameter )
81  {
82  // default is recursive
83  const std::string methodNames[] = { "recursive", "kway", "sfc" };
84  return (PartitioningMethodType ) parameter.getEnum("fem.threads.partitioningmethod", methodNames, 0 );
85  }
86 #endif // #ifdef USE_SMP_PARALLEL
87 
88  public:
90  explicit DomainDecomposedIterator( const GridPartType& gridPart, const ParameterReader &parameter = Parameter::container() )
91  : gridPart_( gridPart ),
92  dofManager_( DofManagerType :: instance( gridPart_.grid() ) ),
93  indexSet_( gridPart_.indexSet() )
94 #ifdef USE_THREADPARTITIONER
95  , sequence_( -1 )
96  , filteredGridParts_( Fem :: ThreadManager :: maxThreads() )
97 #endif
98  , masterRatio_( 1.0 )
99 #ifdef USE_THREADPARTITIONER
100  , method_( getMethod( parameter ) )
101 #endif // #ifdef USE_SMP_PARALLEL
102  , communicationThread_( parameter.getValue<bool>("fem.threads.communicationthread", false)
103  && Fem :: ThreadManager :: maxThreads() > 1 ) // only possible if maxThreads > 1
104  , verbose_( Parameter::verbose() &&
105  parameter.getValue<bool>("fem.threads.verbose", false ) )
106  {
107 #ifdef USE_THREADPARTITIONER
108  for(int thread=0; thread < Fem :: ThreadManager :: maxThreads(); ++thread )
109  {
110  // thread is the thread number of this filter
111  filteredGridParts_[ thread ]
112  = new FilteredGridPartType( const_cast<GridPartType &> (gridPart_),
113  FilterType( gridPart_, threadNum_, thread ) );
114  }
115 
116  threadNum_.setMemoryFactor( 1.1 );
117 #endif
118  update();
119  }
120 
121 #ifdef USE_THREADPARTITIONER
124  {
125  for(int thread=0; thread < Fem :: ThreadManager :: maxThreads(); ++thread )
126  {
127  // i is the thread number of this filter
128  delete filteredGridParts_[ thread ] ;
129  filteredGridParts_[ thread ] = 0 ;
130  }
131  }
132 
134  const FilterType& filter( const int thread ) const
135  {
136  return filteredGridParts_[ thread ]->filter();
137  }
138 #endif // USE_SMP_PARALLEL
139 
141  void update()
142  {
143 #ifdef USE_THREADPARTITIONER
144  const int sequence = gridPart_.sequence() ;
145  // if grid got updated also update iterators
146  if( sequence_ != sequence )
147  {
149  {
150  std::cerr << "Don't call ThreadIterator::update in a parallel environment!" << std::endl;
151  assert( false );
152  abort();
153  }
154 
155  const int commThread = communicationThread_ ? 1 : 0;
156  // get number of partitions possible
157  const size_t partitions = ThreadManager :: maxThreads() - commThread ;
158 
159  // create partitioner
160  ThreadPartitionerType db( gridPart_, partitions, masterRatio_ );
161  // do partitioning
162  db.serialPartition( method_ );
163 
164  // get end iterator
165  typedef typename GridPartType :: template Codim< 0 > :: IteratorType GPIteratorType;
166  const GPIteratorType endit = gridPart_.template end< 0 >();
167 
168  // get size for index set
169  const size_t size = indexSet_.size( 0 );
170 
171  // resize threads storage
172  threadNum_.resize( size );
173  // set all values to default value
174  for(size_t i = 0; i<size; ++i) threadNum_[ i ] = -1;
175 
176  {
177  // just for diagnostics
178  std::vector< int > counter( partitions+commThread , 0 );
179 
180  int numInteriorElems = 0;
181  for(GPIteratorType it = gridPart_.template begin< 0 >();
182  it != endit; ++it, ++numInteriorElems )
183  {
184  const EntityType& entity = * it;
185  const int rank = db.getRank( entity ) + commThread ;
186  assert( rank >= 0 );
187  //std::cout << "Got rank = " << rank << "\n";
188  threadNum_[ indexSet_.index( entity ) ] = rank ;
189  ++counter[ rank ];
190  }
191 
192  // update sequence number
193  sequence_ = sequence;
194 
195  if( verbose_ )
196  {
197  std::cout << "DomainDecomposedIterator: sequence = " << sequence_ << " size = " << numInteriorElems << std::endl;
198  const size_t counterSize = counter.size();
199  for(size_t i = 0; i<counterSize; ++i )
200  std::cout << "DomainDecomposedIterator: T[" << i << "] = " << counter[ i ] << std::endl;
201  }
202 
203 #ifndef NDEBUG
204  // check that all interior elements have got a valid thread number
205  for(GPIteratorType it = gridPart_.template begin< 0 >(); it != endit; ++it )
206  {
207  assert( threadNum_[ indexSet_.index( *it ) ] >= 0 );
208  }
209 #endif
210  //for(size_t i = 0; i<size; ++i )
211  //{
212  // //std::cout << threadNum_[ i ] << std::endl;
213  //}
214  }
215  }
216 #endif
217  }
218 
221  {
222 #ifdef USE_THREADPARTITIONER
223  const int thread = ThreadManager :: thread() ;
224  if( communicationThread_ && thread == 0 )
225  return filteredGridParts_[ thread ]->template end< 0 > ();
226  else
227  return filteredGridParts_[ thread ]->template begin< 0 > ();
228 #else
229  return gridPart_.template begin< 0 > ();
230 #endif
231  }
232 
235  {
236 #ifdef USE_THREADPARTITIONER
237  return filteredGridParts_[ ThreadManager :: thread() ]->template end< 0 > ();
238 #else
239  return gridPart_.template end< 0 > ();
240 #endif
241  }
242 
244  int index(const EntityType& entity ) const
245  {
246  return indexSet_.index( entity );
247  }
248 
250  int thread(const EntityType& entity ) const
251  {
252 #ifdef USE_THREADPARTITIONER
253  assert( (int)threadNum_.size() > (int) indexSet_.index( entity ) );
254  // NOTE: this number can also be negative for ghost elements or elements
255  // that do not belong to the set covered by the space iterators
256  return threadNum_[ indexSet_.index( entity ) ];
257 #else
258  return 0;
259 #endif
260  }
261 
262  void setMasterRatio( const double ratio )
263  {
264  masterRatio_ = 0.5 * (ratio + masterRatio_);
265  }
266  };
267 
268 
270  template <class GridPart>
272  : public ThreadIteratorStorageBase< DomainDecomposedIterator< GridPart > >
273  {
275  public:
276  DomainDecomposedIteratorStorage( const GridPart& gridPart )
277  : BaseType( gridPart )
278  {}
279  };
280 
281  } // end namespace Fem
282 
283 } // end namespace Dune
284 
285 #endif // #ifndef DUNE_FEM_DG_DOMAINTHREADITERATOR_HH
Definition: bindguard.hh:11
Definition: domainfilter.hh:55
DomainArrayImp DomainArrayType
type of array
Definition: domainfilter.hh:61
A FilteredGridPart allows to extract a set of entities from a grid satisfying a given constrainted de...
Definition: filteredgridpart.hh:228
int getEnum(const std::string &key, const std::string(&values)[n]) const
Definition: reader.hh:225
Container for User Specified Parameters.
Definition: io/parameter.hh:191
static ParameterContainer & container()
Definition: io/parameter.hh:193
Thread iterators using domain decomposition.
Definition: domainthreaditerator.hh:28
void setMasterRatio(const double ratio)
Definition: domainthreaditerator.hh:262
GridPartType ::template Codim< 0 >::IteratorType IteratorType
Definition: domainthreaditerator.hh:41
const bool communicationThread_
Definition: domainthreaditerator.hh:75
static const PartitionIteratorType pitype
Definition: domainthreaditerator.hh:48
IteratorType ::Entity EntityType
Definition: domainthreaditerator.hh:44
GridPart GridPartType
Definition: domainthreaditerator.hh:30
int index(const EntityType &entity) const
return thread number this entity belongs to
Definition: domainthreaditerator.hh:244
const IndexSetType & indexSet_
Definition: domainthreaditerator.hh:58
const bool verbose_
Definition: domainthreaditerator.hh:76
IteratorType begin() const
return begin iterator for current thread
Definition: domainthreaditerator.hh:220
IteratorType end() const
return end iterator for current thread
Definition: domainthreaditerator.hh:234
DomainFilter< GridPartType > FilterType
Definition: domainthreaditerator.hh:34
const DofManagerType & dofManager_
Definition: domainthreaditerator.hh:57
DofManager< GridType > DofManagerType
Definition: domainthreaditerator.hh:46
GridPartType ::IndexSetType IndexSetType
Definition: domainthreaditerator.hh:32
int thread(const EntityType &entity) const
return thread number this entity belongs to
Definition: domainthreaditerator.hh:250
const GridPartType & gridPart_
Definition: domainthreaditerator.hh:56
DomainDecomposedIterator(const GridPartType &gridPart, const ParameterReader &parameter=Parameter::container())
contructor creating thread iterators
Definition: domainthreaditerator.hh:90
GridPartType ::GridType GridType
Definition: domainthreaditerator.hh:31
void update()
update internal list of iterators
Definition: domainthreaditerator.hh:141
double masterRatio_
Definition: domainthreaditerator.hh:68
Storage of thread iterators using domain decomposition.
Definition: domainthreaditerator.hh:273
DomainDecomposedIteratorStorage(const GridPart &gridPart)
Definition: domainthreaditerator.hh:276
Storage of thread iterators using domain decomposition.
Definition: threaditeratorstorage.hh:23
Definition: threadmanager.hh:45
static int maxThreads()
return maximal number of threads possbile in the current run
Definition: threadmanager.hh:59
static bool singleThreadMode()
returns true if program is operating on one thread currently
Definition: threadmanager.hh:74
static int thread()
return thread number
Definition: threadmanager.hh:65
Definition: dofmanager.hh:762