Sierra Toolkit  Version of the Day
UnitTestZoltanGraph.cpp
1 /*------------------------------------------------------------------------*/
2 /* Copyright 2010, 2011 Sandia Corporation. */
3 /* Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive */
4 /* license for use of this work by or on behalf of the U.S. Government. */
5 /* Export of this program may require a license from the */
6 /* United States Government. */
7 /*------------------------------------------------------------------------*/
8 
9 #include <stk_util/parallel/Parallel.hpp>
10 #include <stk_util/parallel/ParallelReduce.hpp>
11 #include <stk_util/unit_test_support/stk_utest_macros.hpp>
12 
13 #include <stk_mesh/base/Types.hpp>
14 #include <stk_mesh/base/MetaData.hpp>
15 #include <stk_mesh/base/BulkData.hpp>
16 #include <stk_mesh/base/FieldData.hpp>
17 #include <stk_mesh/base/GetEntities.hpp>
18 
19 #include <stk_mesh/fem/CreateAdjacentEntities.hpp>
20 #include <stk_mesh/fem/TopologyDimensions.hpp>
21 #include <stk_mesh/fem/FEMMetaData.hpp>
22 #include <stk_mesh/fem/FEMHelpers.hpp>
23 
24 #include <stk_rebalance/Rebalance.hpp>
25 #include <stk_rebalance/Partition.hpp>
26 #include <stk_rebalance/ZoltanPartition.hpp>
27 
28 #include <stk_rebalance_utils/RebalanceUtils.hpp>
29 
30 static const size_t NODE_RANK = stk_classic::mesh::fem::FEMMetaData::NODE_RANK;
31 
32 typedef stk_classic::mesh::Field<double> ScalarField ;
33 typedef stk_classic::mesh::Field<double, stk_classic::mesh::Cartesian> VectorField ;
34 
35 enum { nx = 2, ny = 2 };
36 
37 //STKUNIT_UNIT_TEST(UnitTestZoltanGraph, testUnit)
38 void disabled_unit_test()
39 {
40 #ifdef STK_HAS_MPI
41  stk_classic::ParallelMachine comm(MPI_COMM_WORLD);
42 #else
43  stk_classic::ParallelMachine comm(0);
44 #endif
45 
46  unsigned spatial_dimension = 2;
47  std::vector<std::string> rank_names = stk_classic::mesh::fem::entity_rank_names(spatial_dimension);
48  const stk_classic::mesh::EntityRank constraint_rank = rank_names.size();
49  rank_names.push_back("Constraint");
50 
51  stk_classic::mesh::fem::FEMMetaData fem_meta;
52  fem_meta.FEM_initialize(spatial_dimension, rank_names);
53  stk_classic::mesh::MetaData & meta_data = stk_classic::mesh::fem::FEMMetaData::get_meta_data(fem_meta);
54  stk_classic::mesh::BulkData bulk_data( meta_data , comm , 100 );
55  const stk_classic::mesh::EntityRank element_rank = fem_meta.element_rank();
56 
57  stk_classic::mesh::fem::CellTopology quad_top(shards::getCellTopologyData<shards::Quadrilateral<4> >());
58  stk_classic::mesh::Part & quad_part( fem_meta.declare_part("quad", quad_top ) );
59  VectorField & coord_field( fem_meta.declare_field< VectorField >( "coordinates" ) );
60 
61  stk_classic::mesh::put_field( coord_field , NODE_RANK , fem_meta.universal_part() );
62 
63  fem_meta.commit();
64 
65  const unsigned p_size = bulk_data.parallel_size();
66  const unsigned p_rank = bulk_data.parallel_rank();
67 
68  bulk_data.modification_begin();
69 
70  if ( p_rank == 0 ) {
71 
72  std::vector<std::vector<stk_classic::mesh::Entity*> > quads(nx);
73  for ( unsigned ix = 0 ; ix < nx ; ++ix ) quads[ix].resize(ny);
74 
75  const unsigned nnx = nx + 1 ;
76  const unsigned nny = ny + 1 ;
77  for ( unsigned iy = 0 ; iy < ny ; ++iy ) {
78  for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
79  stk_classic::mesh::EntityId elem = 1 + ix + iy * nx ;
80  stk_classic::mesh::EntityId nodes[4] ;
81  nodes[0] = 1 + ix + iy * nnx ;
82  nodes[1] = 2 + ix + iy * nnx ;
83  nodes[2] = 2 + ix + ( iy + 1 ) * nnx ;
84  nodes[3] = 1 + ix + ( iy + 1 ) * nnx ;
85 
86  stk_classic::mesh::Entity &q = stk_classic::mesh::fem::declare_element( bulk_data , quad_part , elem , nodes );
87  quads[ix][iy] = &q;
88  }
89  }
90 
91  for ( unsigned iy = 0 ; iy <= ny ; ++iy ) {
92  for ( unsigned ix = 0 ; ix <= nx ; ++ix ) {
93  stk_classic::mesh::EntityId nid = 1 + ix + iy * nnx ;
94  stk_classic::mesh::Entity * n = bulk_data.get_entity( NODE_RANK, nid );
95  double * const coord = stk_classic::mesh::field_data( coord_field , *n );
96  coord[0] = .1*ix;
97  coord[1] = .1*iy;
98  coord[2] = 0;
99  }
100  }
101 
102  {
103  const unsigned iy_left = 0;
104  const unsigned iy_right = ny;
105  stk_classic::mesh::PartVector add(1, &fem_meta.locally_owned_part());
106  for ( unsigned ix = 0 ; ix <= nx ; ++ix ) {
107  stk_classic::mesh::EntityId nid_left = 1 + ix + iy_left * nnx ;
108  stk_classic::mesh::EntityId nid_right = 1 + ix + iy_right * nnx ;
109  stk_classic::mesh::Entity * n_left = bulk_data.get_entity( NODE_RANK, nid_left );
110  stk_classic::mesh::Entity * n_right = bulk_data.get_entity( NODE_RANK, nid_right );
111  const stk_classic::mesh::EntityId constraint_entity_id = 1 + ix + nny * nnx;
112  stk_classic::mesh::Entity & c = bulk_data.declare_entity( constraint_rank, constraint_entity_id, add );
113  bulk_data.declare_relation( c , *n_left , 0 );
114  bulk_data.declare_relation( c , *n_right , 1 );
115  }
116  }
117 
118  }
119 
120  // Only P0 has any nodes or elements
121  if ( p_rank == 0 ) {
122  STKUNIT_ASSERT( ! bulk_data.buckets( NODE_RANK ).empty() );
123  STKUNIT_ASSERT( ! bulk_data.buckets( element_rank ).empty() );
124  }
125  else {
126  STKUNIT_ASSERT( bulk_data.buckets( NODE_RANK ).empty() );
127  STKUNIT_ASSERT( bulk_data.buckets( element_rank ).empty() );
128  }
129 
130 
131  bulk_data.modification_end();
132 
133  // create some sides and faces to rebalance.
134  stk_classic::mesh::PartVector add_parts;
135  stk_classic::mesh::create_adjacent_entities(bulk_data, add_parts);
136 
137  // Zoltan partition is specialized form a virtual base class, stk_classic::rebalance::Partition.
138  // Other specializations are possible.
139  // Configure Zoltan to use graph-based partitioning
140  Teuchos::ParameterList graph;
141  Teuchos::ParameterList lb_method;
142  lb_method.set("LOAD BALANCING METHOD" , "4");
143  graph.sublist(stk_classic::rebalance::Zoltan::default_parameters_name())=lb_method;
144  stk_classic::rebalance::Zoltan zoltan_partition(comm, spatial_dimension, graph);
145  // end configure snippet
146  stk_classic::mesh::Selector selector(fem_meta.universal_part());
147 
148  // Coordinates are passed to support geometric-based load balancing algorithms
149  stk_classic::rebalance::rebalance(bulk_data, selector, &coord_field, NULL, zoltan_partition, fem_meta.node_rank());
150 
151  const double imbalance_threshold = stk_classic::rebalance::check_balance(bulk_data, NULL, fem_meta.node_rank(), &selector);
152  const bool do_rebal = 1.5 < imbalance_threshold;
153 
154  // Check that we satisfy our threshhold
155  STKUNIT_ASSERT( !do_rebal );
156  if( (2 == p_size) || (4 == p_size) )
157  {
158  STKUNIT_ASSERT_NEAR(imbalance_threshold, 1.0, .1);
159  }
160  else
161  {
162  STKUNIT_ASSERT_LE(imbalance_threshold, 1.5);
163  }
164 
165  // And verify that all dependent entities are on the same proc as their parent element
166  {
167  stk_classic::mesh::EntityVector entities;
168  stk_classic::mesh::Selector selector1 = fem_meta.universal_part();
169 
170  get_selected_entities(selector1, bulk_data.buckets(element_rank), entities);
171  bool result = stk_classic::rebalance::verify_dependent_ownership(NODE_RANK, entities);
172  STKUNIT_ASSERT( result );
173  }
174 }
175 
stk_classic::mesh::put_field
field_type & put_field(field_type &field, EntityRank entity_rank, const Part &part, const void *init_value=NULL)
Declare a field to exist for a given entity type and Part.
stk_classic::mesh::field_data
FieldTraits< field_type >::data_type * field_data(const field_type &f, const Bucket::iterator i)
Pointer to the field data array.
Definition: FieldData.hpp:116
stk_classic::mesh::Field< double >
stk_classic::mesh::Part
An application-defined subset of a problem domain.
Definition: Part.hpp:49
stk_classic::mesh::fem::declare_element
Entity & declare_element(BulkData &mesh, Part &part, const EntityId elem_id, const EntityId node_id[])
Declare an element member of a Part with a CellTopology and nodes conformal to that topology.
Definition: FEMHelpers.cpp:72
stk_classic::rebalance::rebalance
bool rebalance(mesh::BulkData &bulk_data, const mesh::Selector &selector, const VectorField *coord_ref, const ScalarField *elem_weight_ref, Partition &partition, const stk_classic::mesh::EntityRank rank=stk_classic::mesh::InvalidEntityRank)
Rebalance with a Partition object.
Definition: Rebalance.cpp:164
stk_classic::mesh::fem::FEMMetaData::locally_owned_part
Part & locally_owned_part() const
Subset for the problem domain that is owned by the local process. Ghost entities are not members of t...
Definition: FEMMetaData.hpp:277
stk_classic::ParallelMachine
MPI_Comm ParallelMachine
Definition: Parallel.hpp:32
stk_classic::mesh::get_selected_entities
void get_selected_entities(const Selector &selector, const std::vector< Bucket * > &input_buckets, std::vector< Entity * > &entities)
Get entities in selected buckets (selected by the given selector instance), and sorted by ID.
Definition: GetEntities.cpp:77
stk_classic::mesh::fem::FEMMetaData::get_meta_data
static MetaData & get_meta_data(FEMMetaData &fem_meta)
Getter for MetaData off of a FEMMetaData object.
Definition: FEMMetaData.hpp:113
stk_classic::mesh::fem::FEMMetaData::declare_part
Part & declare_part(const std::string &name, fem::CellTopology cell_topology)
Declare a part with a given cell topology.
Definition: FEMMetaData.hpp:240
stk_classic::mesh::fem::FEMMetaData::commit
void commit()
Commit the part and field declarations so that the meta data manager can be used to create mesh bulk ...
Definition: FEMMetaData.hpp:466
stk_classic::mesh::MetaData
The manager of an integrated collection of parts and fields.
Definition: MetaData.hpp:56
stk_classic::mesh::PartVector
std::vector< Part * > PartVector
Collections of parts are frequently maintained as a vector of Part pointers.
Definition: Types.hpp:31
stk_classic::mesh::Selector
This is a class for selecting buckets based on a set of meshparts and set logic.
Definition: Selector.hpp:112
Rebalance.hpp
Static functions for dynamic load balancing.
stk_classic::mesh::fem::FEMMetaData::node_rank
EntityRank node_rank() const
Returns the node rank, which is always zero.
Definition: FEMMetaData.hpp:125
stk_classic::mesh::fem::FEMMetaData::declare_field
field_type & declare_field(const std::string &name, unsigned number_of_states=1)
Declare a field of the given field_type, test name, and number of states.
Definition: FEMMetaData.hpp:427
Partition.hpp
For partitioning of mesh entities over a processing grid.
stk_classic::mesh::fem::FEMMetaData::FEM_initialize
void FEM_initialize(size_t spatial_dimension, const std::vector< std::string > &in_entity_rank_names=std::vector< std::string >())
Initialize the spatial dimension and an optional list of entity rank names associated with each rank.
Definition: FEMMetaData.cpp:61
ZoltanPartition.hpp
stk_classic::mesh::fem::FEMMetaData::element_rank
EntityRank element_rank() const
Returns the element rank which is always equal to spatial dimension.
Definition: FEMMetaData.hpp:160
stk_classic::mesh::fem::FEMMetaData
FEMMetaData is a class that implements a Finite Element Method skin on top of the Sierra Tool Kit Met...
Definition: FEMMetaData.hpp:54
stk_classic::mesh::Entity
A fundamental unit within the discretization of a problem domain, including but not limited to nodes,...
Definition: Entity.hpp:120
stk_classic::mesh::fem::FEMMetaData::universal_part
Part & universal_part() const
Universal subset for the problem domain. All other parts are a subset of the universal part.
Definition: FEMMetaData.hpp:272
stk_classic::mesh::BulkData
Manager for an integrated collection of entities, entity relations, and buckets of field data.
Definition: BulkData.hpp:49