Sierra Toolkit  Version of the Day
UnitTestSimple.cpp
1 /*------------------------------------------------------------------------*/
2 /* Copyright 2010, 2011 Sandia Corporation. */
3 /* Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive */
4 /* license for use of this work by or on behalf of the U.S. Government. */
5 /* Export of this program may require a license from the */
6 /* United States Government. */
7 /*------------------------------------------------------------------------*/
8 
9 
10 #include <stk_util/parallel/Parallel.hpp>
11 #include <stk_util/parallel/ParallelReduce.hpp>
12 #include <stk_util/unit_test_support/stk_utest_macros.hpp>
13 
14 #include <stk_mesh/base/Types.hpp>
15 #include <stk_mesh/base/MetaData.hpp>
16 #include <stk_mesh/base/BulkData.hpp>
17 #include <stk_mesh/base/FieldData.hpp>
18 
19 #include <stk_mesh/fem/CoordinateSystems.hpp>
20 #include <stk_mesh/fem/FEMMetaData.hpp>
21 #include <stk_mesh/fem/FEMHelpers.hpp>
22 
23 // relevant headers
24 #include <stk_rebalance/Rebalance.hpp>
25 #include <stk_rebalance/Partition.hpp>
26 #include <stk_rebalance_utils/RebalanceUtils.hpp>
27 // end relevant headers
28 
29 static const size_t NODE_RANK = stk_classic::mesh::fem::FEMMetaData::NODE_RANK;
30 
31 typedef stk_classic::mesh::Field<double> ScalarField ;
32 typedef stk_classic::mesh::Field<double, stk_classic::mesh::Cartesian> VectorField ;
33 
34 static const int spatial_dimension = 2;
35 
36 enum { nx = 2, ny = 2 };
37 
38 class MockPartition : public stk_classic::rebalance::Partition
39 {
40  public:
41 
42  enum BALANCE_TEST_STEP
43  { FIRST,
44  SECOND,
45  THIRD };
46 
47  MockPartition( stk_classic::mesh::fem::FEMMetaData & fmd, stk_classic::mesh::BulkData & bd ) :
48  stk_classic::rebalance::Partition(bd.parallel()),
49  m_fem_meta(fmd),
50  m_bulk_data(bd),
51  m_step(FIRST)
52  { }
53 
54  ~MockPartition() { }
55 
56  void set_balance_step(BALANCE_TEST_STEP step)
57  { m_step = step; }
58 
59  void set_mesh_info ( const std::vector<stk_classic::mesh::Entity *> &mesh_entities,
60  const VectorField * nodal_coord_ref,
61  const ScalarField * elem_weight_ref)
62  { total_number_entities_ = mesh_entities.size(); }
63 
64  unsigned num_elems() const
65  { return total_number_entities_; }
66 
67  void determine_new_partition(bool &RebalancingNeeded)
68  { RebalancingNeeded = (m_bulk_data.parallel_size() > 1); }
69 
70  int get_new_partition(std::vector<stk_classic::mesh::EntityProc> &new_partition);
71 
72  bool partition_dependents_needed() const
73  { return false; /* I handle both element and dependent node partitioning */ }
74 
75  private:
76 
77  unsigned total_number_entities_;
78  stk_classic::mesh::fem::FEMMetaData & m_fem_meta;
79  stk_classic::mesh::BulkData & m_bulk_data;
80  BALANCE_TEST_STEP m_step;
81 };
82 
83 int
84 MockPartition::get_new_partition(std::vector<stk_classic::mesh::EntityProc> &new_partition)
85 {
86  const unsigned p_size = m_bulk_data.parallel_size();
87  const unsigned p_rank = m_bulk_data.parallel_rank();
88  const stk_classic::mesh::EntityRank element_rank = m_fem_meta.element_rank();
89 
90  new_partition.clear();
91 
92  if ( 1 < p_size ) {
93 
94  if( FIRST == m_step )
95  {
96  if ( p_rank == 0 ) {
97  if ( p_size == 3 ) {
98  const unsigned nnx = nx + 1 ;
99  const unsigned nny = ny + 1 ;
100  for ( unsigned iy = nny / 2 ; iy < nny ; ++iy ) {
101  for ( unsigned ix = 0 ; ix < nnx ; ++ix ) {
102  stk_classic::mesh::EntityId id = 1 + ix + iy * nnx ;
103  unsigned proc = ix < nx/2 ? 1 : 2;
104  stk_classic::mesh::EntityProc tmp( m_bulk_data.get_entity( NODE_RANK , id ) , proc );
105  new_partition.push_back( tmp );
106  }
107  }
108  for ( unsigned iy = ny / 2 ; iy < ny ; ++iy ) {
109  for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
110  stk_classic::mesh::EntityId id = 1 + ix + iy * nx ;
111  unsigned proc = ix < nx/2 ? 1 : 2;
112  stk_classic::mesh::EntityProc tmp( m_bulk_data.get_entity( element_rank , id ) , proc );
113  new_partition.push_back( tmp );
114  }
115  }
116  }
117  else
118  {
119  const unsigned nnx = nx + 1 ;
120  const unsigned nny = ny + 1 ;
121  for ( unsigned iy = nny / 2 ; iy < nny ; ++iy ) {
122  for ( unsigned ix = 0 ; ix < nnx ; ++ix ) {
123  stk_classic::mesh::EntityId id = 1 + ix + iy * nnx ;
124  stk_classic::mesh::EntityProc tmp( m_bulk_data.get_entity( NODE_RANK , id ) , 1 );
125  new_partition.push_back( tmp );
126  }
127  }
128  for ( unsigned iy = ny / 2 ; iy < ny ; ++iy ) {
129  for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
130  stk_classic::mesh::EntityId id = 1 + ix + iy * nx ;
131  stk_classic::mesh::EntityProc tmp( m_bulk_data.get_entity( element_rank , id ) , 1 );
132  new_partition.push_back( tmp );
133  }
134  }
135  }
136  }
137  }
138 
139  else if( SECOND == m_step )
140  {
141  if ( p_rank == 0 ) {
142  const unsigned nnx = nx + 1 ;
143  const unsigned nny = ny + 1 ;
144  for ( unsigned iy = 0 ; iy < nny / 2 ; ++iy ) {
145  for ( unsigned ix = 0 ; ix < nnx ; ++ix ) {
146  stk_classic::mesh::EntityId id = 1 + ix + iy * nnx ;
147  stk_classic::mesh::EntityProc tmp( m_bulk_data.get_entity( NODE_RANK , id ) , 1 );
148  new_partition.push_back( tmp );
149  }
150  }
151  for ( unsigned iy = 0 ; iy < ny / 2 ; ++iy ) {
152  for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
153  stk_classic::mesh::EntityId id = 1 + ix + iy * nx ;
154  stk_classic::mesh::EntityProc tmp( m_bulk_data.get_entity( element_rank , id ) , 1 );
155  new_partition.push_back( tmp );
156  }
157  }
158  }
159  }
160 
161  else if( THIRD == m_step )
162  {
163  if ( p_size == 3 ) {
164  new_partition.clear();
165 
166  if ( p_rank == 2 ) {
167  const unsigned nnx = nx + 1 ;
168  const unsigned nny = ny + 1 ;
169  for ( unsigned iy = nny / 2 ; iy < nny ; ++iy ) {
170  for ( unsigned ix = nx / 2 ; ix < nnx ; ++ix ) {
171  stk_classic::mesh::EntityId id = 1 + ix + iy * nnx ;
172  unsigned proc = 1;
173  stk_classic::mesh::EntityProc tmp( m_bulk_data.get_entity( NODE_RANK , id ) , proc );
174  new_partition.push_back( tmp );
175  }
176  }
177  for ( unsigned iy = ny / 2 ; iy < ny ; ++iy ) {
178  for ( unsigned ix = nx / 2 ; ix < nx ; ++ix ) {
179  stk_classic::mesh::EntityId id = 1 + ix + iy * nx ;
180  unsigned proc = 1;
181  stk_classic::mesh::EntityProc tmp( m_bulk_data.get_entity( element_rank , id ) , proc );
182  new_partition.push_back( tmp );
183  }
184  }
185  }
186  }
187  }
188  }
189 
190  return 0;
191 }
192 
193 
194 STKUNIT_UNIT_TEST(UnitTestRebalanceSimple, testUnit)
195 {
196 #ifdef STK_HAS_MPI
197  stk_classic::ParallelMachine comm(MPI_COMM_WORLD);
198 #else
199  stk_classic::ParallelMachine comm(0);
200 #endif
201 
202  unsigned spatial_dimension = 2;
203  stk_classic::mesh::fem::FEMMetaData fem_meta;
204  fem_meta.FEM_initialize(spatial_dimension, stk_classic::mesh::fem::entity_rank_names(spatial_dimension) );
205  stk_classic::mesh::MetaData & meta_data = stk_classic::mesh::fem::FEMMetaData::get_meta_data(fem_meta);
206  stk_classic::mesh::BulkData bulk_data( meta_data , comm , 100 );
207  const stk_classic::mesh::EntityRank element_rank = fem_meta.element_rank();
208  stk_classic::mesh::fem::CellTopology quad_top(shards::getCellTopologyData<shards::Quadrilateral<4> >());
209  stk_classic::mesh::Part & quad_part( fem_meta.declare_part("quad", quad_top ) );
210  VectorField & coord_field( fem_meta.declare_field< VectorField >( "coordinates" ) );
211  ScalarField & weight_field( fem_meta.declare_field< ScalarField >( "element_weights" ) );
212 
213  stk_classic::mesh::put_field( coord_field , NODE_RANK , fem_meta.universal_part() );
214  stk_classic::mesh::put_field(weight_field , element_rank , fem_meta.universal_part() );
215 
216  fem_meta.commit();
217 
218  const unsigned p_size = bulk_data.parallel_size();
219  const unsigned p_rank = bulk_data.parallel_rank();
220 
221  // create initial mesh
222  bulk_data.modification_begin();
223 
224  if ( p_rank == 0 ) {
225  const unsigned nnx = nx + 1 ;
226  for ( unsigned iy = 0 ; iy < ny ; ++iy ) {
227  for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
228  stk_classic::mesh::EntityId elem = 1 + ix + iy * nx ;
229  stk_classic::mesh::EntityId nodes[4] ;
230  nodes[0] = 1 + ix + iy * nnx ;
231  nodes[1] = 2 + ix + iy * nnx ;
232  nodes[2] = 2 + ix + ( iy + 1 ) * nnx ;
233  nodes[3] = 1 + ix + ( iy + 1 ) * nnx ;
234 
235  stk_classic::mesh::fem::declare_element( bulk_data , quad_part , elem , nodes );
236  }
237  }
238  // end create initial mesh
239 
240  // assign element weights
241  for ( unsigned iy = 0 ; iy < ny ; ++iy ) {
242  for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
243  stk_classic::mesh::EntityId elem = 1 + ix + iy * nx ;
244  stk_classic::mesh::Entity * e = bulk_data.get_entity( element_rank, elem );
245  double * const e_weight = stk_classic::mesh::field_data( weight_field , *e );
246  *e_weight = 1.0;
247  }
248  }
249  // end assign element weights
250  }
251 
252  // Only P0 has any nodes or elements
253  if ( p_rank == 0 ) {
254  STKUNIT_ASSERT( ! bulk_data.buckets( NODE_RANK ).empty() );
255  STKUNIT_ASSERT( ! bulk_data.buckets( element_rank ).empty() );
256  }
257  else {
258  STKUNIT_ASSERT( bulk_data.buckets( NODE_RANK ).empty() );
259  STKUNIT_ASSERT( bulk_data.buckets( element_rank ).empty() );
260  }
261 
262  bulk_data.modification_end();
263 
264  // Create our Partition and Selector objects
265  MockPartition partition(fem_meta, bulk_data);
266  stk_classic::mesh::Selector selector(fem_meta.universal_part());
267 
268  partition.set_balance_step(MockPartition::FIRST);
269  // Exercise the threshhold calculation by using imblance_threshhold > 1.0
270  bool do_rebal = 1.5 < stk_classic::rebalance::check_balance(bulk_data, &weight_field, element_rank);
271  if( do_rebal )
272  {
273  // Pick a few values as negative to exercise a check in rebalance::rebalance(...)
274  // which converts negative weights to 1.0
275  if ( p_rank == 0 )
276  {
277  for ( unsigned iy = 0 ; iy < ny ; ++iy )
278  {
279  stk_classic::mesh::EntityId elem = 1 + iy * nx ;
280  stk_classic::mesh::Entity * e = bulk_data.get_entity( element_rank, elem );
281  double * const e_weight = stk_classic::mesh::field_data( weight_field , *e );
282  *e_weight = -2.0;
283  }
284  }
285  // Do the actual rebalance
286  stk_classic::rebalance::rebalance(bulk_data, selector, NULL, &weight_field, partition);
287  }
288 
289  partition.set_balance_step(MockPartition::SECOND);
290  stk_classic::rebalance::rebalance(bulk_data, selector, NULL, &weight_field, partition);
291 
292  partition.set_balance_step(MockPartition::THIRD);
293  stk_classic::rebalance::rebalance(bulk_data, selector, NULL, &weight_field, partition);
294 
295  if ( 1 < p_size ) {
296  // Only P1 has any nodes or elements
297  if ( p_rank == 1 ) {
298  STKUNIT_ASSERT( ! bulk_data.buckets( NODE_RANK ).empty() );
299  STKUNIT_ASSERT( ! bulk_data.buckets( element_rank ).empty() );
300  }
301  else {
302  STKUNIT_ASSERT( bulk_data.buckets( NODE_RANK ).empty() );
303  STKUNIT_ASSERT( bulk_data.buckets( element_rank ).empty() );
304  }
305  }
306 }
307 
stk_classic::mesh::put_field
field_type & put_field(field_type &field, EntityRank entity_rank, const Part &part, const void *init_value=NULL)
Declare a field to exist for a given entity type and Part.
stk_classic::rebalance::Partition::parallel
ParallelMachine parallel() const
Return the parallel communicator for this partition entity.
Definition: Partition.hpp:111
stk_classic::mesh::field_data
FieldTraits< field_type >::data_type * field_data(const field_type &f, const Bucket::iterator i)
Pointer to the field data array.
Definition: FieldData.hpp:116
stk_classic::rebalance::Partition::determine_new_partition
virtual void determine_new_partition(bool &RebalancingNeeded)=0
determine New Partition.
stk_classic::mesh::Field< double >
stk_classic::mesh::Part
An application-defined subset of a problem domain.
Definition: Part.hpp:49
stk_classic::mesh::fem::declare_element
Entity & declare_element(BulkData &mesh, Part &part, const EntityId elem_id, const EntityId node_id[])
Declare an element member of a Part with a CellTopology and nodes conformal to that topology.
Definition: FEMHelpers.cpp:72
stk_classic::rebalance::rebalance
bool rebalance(mesh::BulkData &bulk_data, const mesh::Selector &selector, const VectorField *coord_ref, const ScalarField *elem_weight_ref, Partition &partition, const stk_classic::mesh::EntityRank rank=stk_classic::mesh::InvalidEntityRank)
Rebalance with a Partition object.
Definition: Rebalance.cpp:164
stk_classic::rebalance::Partition::num_elems
virtual unsigned num_elems() const =0
Return the total number of mesh entities in all lists.
stk_classic::ParallelMachine
MPI_Comm ParallelMachine
Definition: Parallel.hpp:32
stk_classic::rebalance::Partition::partition_dependents_needed
virtual bool partition_dependents_needed() const =0
Query whether element dependents need to be rebalanced outside this Partition.
stk_classic::rebalance::Partition::Partition
Partition(stk_classic::ParallelMachine comm)
Constructors.
Definition: Partition.cpp:20
stk_classic::rebalance::Partition
Initialized with a list of mesh entities unique to each processor.
Definition: Partition.hpp:81
stk_classic::mesh::fem::FEMMetaData::get_meta_data
static MetaData & get_meta_data(FEMMetaData &fem_meta)
Getter for MetaData off of a FEMMetaData object.
Definition: FEMMetaData.hpp:113
stk_classic::mesh::fem::FEMMetaData::declare_part
Part & declare_part(const std::string &name, fem::CellTopology cell_topology)
Declare a part with a given cell topology.
Definition: FEMMetaData.hpp:240
stk_classic::mesh::fem::FEMMetaData::commit
void commit()
Commit the part and field declarations so that the meta data manager can be used to create mesh bulk ...
Definition: FEMMetaData.hpp:466
stk_classic
Sierra Toolkit.
Definition: AlgorithmRunner.cpp:16
stk_classic::mesh::MetaData
The manager of an integrated collection of parts and fields.
Definition: MetaData.hpp:56
stk_classic::mesh::EntityProc
std::pair< Entity *, unsigned > EntityProc
Pairing of an entity with a processor rank.
Definition: Types.hpp:111
stk_classic::mesh::Selector
This is a class for selecting buckets based on a set of meshparts and set logic.
Definition: Selector.hpp:112
Rebalance.hpp
Static functions for dynamic load balancing.
stk_classic::mesh::fem::FEMMetaData::declare_field
field_type & declare_field(const std::string &name, unsigned number_of_states=1)
Declare a field of the given field_type, test name, and number of states.
Definition: FEMMetaData.hpp:427
Partition.hpp
For partitioning of mesh entities over a processing grid.
stk_classic::mesh::fem::FEMMetaData::FEM_initialize
void FEM_initialize(size_t spatial_dimension, const std::vector< std::string > &in_entity_rank_names=std::vector< std::string >())
Initialize the spatial dimension and an optional list of entity rank names associated with each rank.
Definition: FEMMetaData.cpp:61
stk_classic::rebalance::Partition::get_new_partition
virtual int get_new_partition(stk_classic::mesh::EntityProcVec &new_partition)=0
Perform communication to create new partition.
stk_classic::rebalance::Partition::set_mesh_info
virtual void set_mesh_info(const std::vector< mesh::Entity * > &mesh_entities, const VectorField *nodal_coord_ref, const ScalarField *elem_weight_ref=NULL)=0
Define mesh entities to balance.
stk_classic::mesh::fem::FEMMetaData::element_rank
EntityRank element_rank() const
Returns the element rank which is always equal to spatial dimension.
Definition: FEMMetaData.hpp:160
stk_classic::mesh::fem::FEMMetaData
FEMMetaData is a class that implements a Finite Element Method skin on top of the Sierra Tool Kit Met...
Definition: FEMMetaData.hpp:54
stk_classic::mesh::Entity
A fundamental unit within the discretization of a problem domain, including but not limited to nodes,...
Definition: Entity.hpp:120
stk_classic::mesh::fem::FEMMetaData::universal_part
Part & universal_part() const
Universal subset for the problem domain. All other parts are a subset of the universal part.
Definition: FEMMetaData.hpp:272
stk_classic::mesh::BulkData
Manager for an integrated collection of entities, entity relations, and buckets of field data.
Definition: BulkData.hpp:49