Sierra Toolkit  Version of the Day
UnitTestZoltanSimple.cpp
1 /*------------------------------------------------------------------------*/
2 /* Copyright 2010, 2011 Sandia Corporation. */
3 /* Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive */
4 /* license for use of this work by or on behalf of the U.S. Government. */
5 /* Export of this program may require a license from the */
6 /* United States Government. */
7 /*------------------------------------------------------------------------*/
8 
9 #include <stk_util/parallel/Parallel.hpp>
10 #include <stk_util/parallel/ParallelReduce.hpp>
11 #include <stk_util/unit_test_support/stk_utest_macros.hpp>
12 
13 #include <stk_mesh/base/Types.hpp>
14 #include <stk_mesh/base/MetaData.hpp>
15 #include <stk_mesh/base/BulkData.hpp>
16 #include <stk_mesh/base/FieldData.hpp>
17 #include <stk_mesh/base/GetEntities.hpp>
18 
19 #include <stk_mesh/fem/CreateAdjacentEntities.hpp>
20 #include <stk_mesh/fem/TopologyDimensions.hpp>
21 #include <stk_mesh/fem/FEMMetaData.hpp>
22 #include <stk_mesh/fem/FEMHelpers.hpp>
23 
24 #include <stk_rebalance/Rebalance.hpp>
25 #include <stk_rebalance/Partition.hpp>
26 #include <stk_rebalance/ZoltanPartition.hpp>
27 
28 #include <stk_rebalance_utils/RebalanceUtils.hpp>
29 
30 static const size_t NODE_RANK = stk_classic::mesh::fem::FEMMetaData::NODE_RANK;
31 
32 typedef stk_classic::mesh::Field<double> ScalarField ;
33 typedef stk_classic::mesh::Field<double, stk_classic::mesh::Cartesian> VectorField ;
34 
35 enum { nx = 2, ny = 2 };
36 
37 STKUNIT_UNIT_TEST(UnitTestZoltanSimple, testUnit)
38 {
39 #ifdef STK_HAS_MPI
40  stk_classic::ParallelMachine comm(MPI_COMM_WORLD);
41 #else
42  stk_classic::ParallelMachine comm(0);
43 #endif
44 
45  unsigned spatial_dimension = 2;
46  std::vector<std::string> rank_names = stk_classic::mesh::fem::entity_rank_names(spatial_dimension);
47  const stk_classic::mesh::EntityRank constraint_rank = rank_names.size();
48  rank_names.push_back("Constraint");
49 
50  stk_classic::mesh::fem::FEMMetaData fem_meta;
51  fem_meta.FEM_initialize( spatial_dimension, rank_names );
52  stk_classic::mesh::MetaData & meta_data = stk_classic::mesh::fem::FEMMetaData::get_meta_data(fem_meta);
53  stk_classic::mesh::BulkData bulk_data( meta_data , comm , 100 );
54  const stk_classic::mesh::EntityRank element_rank = fem_meta.element_rank();
55 
56  stk_classic::mesh::fem::CellTopology quad_top(shards::getCellTopologyData<shards::Quadrilateral<4> >());
57  stk_classic::mesh::Part & quad_part( fem_meta.declare_part("quad", quad_top ) );
58  VectorField & coord_field( fem_meta.declare_field< VectorField >( "coordinates" ) );
59  ScalarField & weight_field( fem_meta.declare_field< ScalarField >( "element_weights" ) );
60 
61  stk_classic::mesh::put_field( coord_field , NODE_RANK , fem_meta.universal_part() );
62  stk_classic::mesh::put_field(weight_field , element_rank , fem_meta.universal_part() );
63 
64  fem_meta.commit();
65 
66  const unsigned p_size = bulk_data.parallel_size();
67  const unsigned p_rank = bulk_data.parallel_rank();
68 
69  bulk_data.modification_begin();
70 
71  if ( p_rank == 0 ) {
72 
73  std::vector<std::vector<stk_classic::mesh::Entity*> > quads(nx);
74  for ( unsigned ix = 0 ; ix < nx ; ++ix ) quads[ix].resize(ny);
75 
76  const unsigned nnx = nx + 1 ;
77  const unsigned nny = ny + 1 ;
78  for ( unsigned iy = 0 ; iy < ny ; ++iy ) {
79  for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
80  stk_classic::mesh::EntityId elem = 1 + ix + iy * nx ;
81  stk_classic::mesh::EntityId nodes[4] ;
82  nodes[0] = 1 + ix + iy * nnx ;
83  nodes[1] = 2 + ix + iy * nnx ;
84  nodes[2] = 2 + ix + ( iy + 1 ) * nnx ;
85  nodes[3] = 1 + ix + ( iy + 1 ) * nnx ;
86 
87  stk_classic::mesh::Entity &q = stk_classic::mesh::fem::declare_element( bulk_data , quad_part , elem , nodes );
88  quads[ix][iy] = &q;
89  }
90  }
91 
92  for ( unsigned iy = 0 ; iy < ny ; ++iy ) {
93  for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
94  stk_classic::mesh::EntityId elem = 1 + ix + iy * nx ;
95  stk_classic::mesh::Entity * e = bulk_data.get_entity( element_rank, elem );
96  double * const e_weight = stk_classic::mesh::field_data( weight_field , *e );
97  *e_weight = 1.0;
98  }
99  }
100 
101  for ( unsigned iy = 0 ; iy <= ny ; ++iy ) {
102  for ( unsigned ix = 0 ; ix <= nx ; ++ix ) {
103  stk_classic::mesh::EntityId nid = 1 + ix + iy * nnx ;
104  stk_classic::mesh::Entity * n = bulk_data.get_entity( NODE_RANK, nid );
105  double * const coord = stk_classic::mesh::field_data( coord_field , *n );
106  coord[0] = .1*ix;
107  coord[1] = .1*iy;
108  coord[2] = 0;
109  }
110  }
111 
112  {
113  const unsigned iy_left = 0;
114  const unsigned iy_right = ny;
115  stk_classic::mesh::PartVector add(1, &fem_meta.locally_owned_part());
116  for ( unsigned ix = 0 ; ix <= nx ; ++ix ) {
117  stk_classic::mesh::EntityId nid_left = 1 + ix + iy_left * nnx ;
118  stk_classic::mesh::EntityId nid_right = 1 + ix + iy_right * nnx ;
119  stk_classic::mesh::Entity * n_left = bulk_data.get_entity( NODE_RANK, nid_left );
120  stk_classic::mesh::Entity * n_right = bulk_data.get_entity( NODE_RANK, nid_right );
121  const stk_classic::mesh::EntityId constraint_entity_id = 1 + ix + nny * nnx;
122  stk_classic::mesh::Entity & c = bulk_data.declare_entity( constraint_rank, constraint_entity_id, add );
123  bulk_data.declare_relation( c , *n_left , 0 );
124  bulk_data.declare_relation( c , *n_right , 1 );
125  }
126  }
127 
128  }
129 
130  // Only P0 has any nodes or elements
131  if ( p_rank == 0 ) {
132  STKUNIT_ASSERT( ! bulk_data.buckets( NODE_RANK ).empty() );
133  STKUNIT_ASSERT( ! bulk_data.buckets( element_rank ).empty() );
134  }
135  else {
136  STKUNIT_ASSERT( bulk_data.buckets( NODE_RANK ).empty() );
137  STKUNIT_ASSERT( bulk_data.buckets( element_rank ).empty() );
138  }
139 
140 
141  bulk_data.modification_end();
142 
143  // create some sides and faces to rebalance.
144  stk_classic::mesh::PartVector add_parts;
145  stk_classic::mesh::create_adjacent_entities(bulk_data, add_parts);
146 
147  // Zoltan partition is specialized fomm a virtual base class, stk_classic::rebalance::Partition.
148  // Other specializations are possible.
149  Teuchos::ParameterList emptyList;
150  stk_classic::rebalance::Zoltan zoltan_partition(comm, spatial_dimension, emptyList);
151 
152  {
153  stk_classic::mesh::Selector selector(fem_meta.universal_part());
154 
155  stk_classic::rebalance::rebalance(bulk_data, selector, &coord_field, &weight_field, zoltan_partition);
156  }
157 
158  const double imbalance_threshold = stk_classic::rebalance::check_balance(bulk_data, &weight_field, element_rank);
159  const bool do_rebal = 1.5 < imbalance_threshold;
160 
161  // Check that we satisfy our threshhold
162  STKUNIT_ASSERT( !do_rebal );
163  if( (2 == p_size) || (4 == p_size) )
164  {
165  STKUNIT_ASSERT_NEAR(imbalance_threshold, 1.0, 1.e-8);
166  }
167  else
168  {
169  STKUNIT_ASSERT_LE(imbalance_threshold, 1.5);
170  }
171 
172  // And verify that all dependent entities are on the same proc as their parent element
173  {
174  stk_classic::mesh::EntityVector entities;
175  stk_classic::mesh::Selector selector = fem_meta.locally_owned_part();
176 
177  get_selected_entities(selector, bulk_data.buckets(NODE_RANK), entities);
178  bool result = stk_classic::rebalance::verify_dependent_ownership(element_rank, entities);
179  //get_selected_entities(selector, bulk_data.buckets(constraint_rank), entities);
180  //result &= stk_classic::rebalance::verify_dependent_ownership(element_rank, entities);
181  STKUNIT_ASSERT( result );
182  }
183 }
184 
stk_classic::mesh::put_field
field_type & put_field(field_type &field, EntityRank entity_rank, const Part &part, const void *init_value=NULL)
Declare a field to exist for a given entity type and Part.
stk_classic::mesh::field_data
FieldTraits< field_type >::data_type * field_data(const field_type &f, const Bucket::iterator i)
Pointer to the field data array.
Definition: FieldData.hpp:116
stk_classic::mesh::Field< double >
stk_classic::mesh::Part
An application-defined subset of a problem domain.
Definition: Part.hpp:49
stk_classic::mesh::fem::declare_element
Entity & declare_element(BulkData &mesh, Part &part, const EntityId elem_id, const EntityId node_id[])
Declare an element member of a Part with a CellTopology and nodes conformal to that topology.
Definition: FEMHelpers.cpp:72
stk_classic::rebalance::rebalance
bool rebalance(mesh::BulkData &bulk_data, const mesh::Selector &selector, const VectorField *coord_ref, const ScalarField *elem_weight_ref, Partition &partition, const stk_classic::mesh::EntityRank rank=stk_classic::mesh::InvalidEntityRank)
Rebalance with a Partition object.
Definition: Rebalance.cpp:164
stk_classic::mesh::fem::FEMMetaData::locally_owned_part
Part & locally_owned_part() const
Subset for the problem domain that is owned by the local process. Ghost entities are not members of t...
Definition: FEMMetaData.hpp:277
stk_classic::ParallelMachine
MPI_Comm ParallelMachine
Definition: Parallel.hpp:32
stk_classic::mesh::get_selected_entities
void get_selected_entities(const Selector &selector, const std::vector< Bucket * > &input_buckets, std::vector< Entity * > &entities)
Get entities in selected buckets (selected by the given selector instance), and sorted by ID.
Definition: GetEntities.cpp:77
stk_classic::mesh::fem::FEMMetaData::get_meta_data
static MetaData & get_meta_data(FEMMetaData &fem_meta)
Getter for MetaData off of a FEMMetaData object.
Definition: FEMMetaData.hpp:113
stk_classic::mesh::fem::FEMMetaData::declare_part
Part & declare_part(const std::string &name, fem::CellTopology cell_topology)
Declare a part with a given cell topology.
Definition: FEMMetaData.hpp:240
stk_classic::mesh::fem::FEMMetaData::commit
void commit()
Commit the part and field declarations so that the meta data manager can be used to create mesh bulk ...
Definition: FEMMetaData.hpp:466
stk_classic::mesh::MetaData
The manager of an integrated collection of parts and fields.
Definition: MetaData.hpp:56
stk_classic::mesh::PartVector
std::vector< Part * > PartVector
Collections of parts are frequently maintained as a vector of Part pointers.
Definition: Types.hpp:31
stk_classic::mesh::Selector
This is a class for selecting buckets based on a set of meshparts and set logic.
Definition: Selector.hpp:112
Rebalance.hpp
Static functions for dynamic load balancing.
stk_classic::mesh::fem::FEMMetaData::declare_field
field_type & declare_field(const std::string &name, unsigned number_of_states=1)
Declare a field of the given field_type, test name, and number of states.
Definition: FEMMetaData.hpp:427
Partition.hpp
For partitioning of mesh entities over a processing grid.
stk_classic::mesh::fem::FEMMetaData::FEM_initialize
void FEM_initialize(size_t spatial_dimension, const std::vector< std::string > &in_entity_rank_names=std::vector< std::string >())
Initialize the spatial dimension and an optional list of entity rank names associated with each rank.
Definition: FEMMetaData.cpp:61
ZoltanPartition.hpp
stk_classic::mesh::fem::FEMMetaData::element_rank
EntityRank element_rank() const
Returns the element rank which is always equal to spatial dimension.
Definition: FEMMetaData.hpp:160
stk_classic::mesh::fem::FEMMetaData
FEMMetaData is a class that implements a Finite Element Method skin on top of the Sierra Tool Kit Met...
Definition: FEMMetaData.hpp:54
stk_classic::mesh::Entity
A fundamental unit within the discretization of a problem domain, including but not limited to nodes,...
Definition: Entity.hpp:120
stk_classic::mesh::fem::FEMMetaData::universal_part
Part & universal_part() const
Universal subset for the problem domain. All other parts are a subset of the universal part.
Definition: FEMMetaData.hpp:272
stk_classic::mesh::BulkData
Manager for an integrated collection of entities, entity relations, and buckets of field data.
Definition: BulkData.hpp:49