ViewVC Help
View File | Revision Log | Show Annotations | View Changeset | Root Listing
root/group/trunk/OOPSE/libmdtools/mpiSimulation.cpp
(Generate patch)

Comparing trunk/OOPSE/libmdtools/mpiSimulation.cpp (file contents):
Revision 404 by chuckv, Wed Mar 26 16:12:53 2003 UTC vs.
Revision 441 by chuckv, Tue Apr 1 16:50:14 2003 UTC

# Line 1 | Line 1
1   #ifdef IS_MPI
2 <
2 > #include <iostream>
3   #include <cstdlib>
4   #include <cstring>
5 + #include <cmath>
6   #include <mpi.h>
7   #include <mpi++.h>
8  
9   #include "mpiSimulation.hpp"
10   #include "simError.h"
11   #include "fortranWrappers.hpp"
12 + #include "randomSPRNG.hpp"
13  
14 + #define BASE_SEED 123456789
15  
13 –
14 –
16   mpiSimulation* mpiSim;
17  
18   mpiSimulation::mpiSimulation(SimInfo* the_entryPlug)
# Line 21 | Line 22 | mpiSimulation::mpiSimulation(SimInfo* the_entryPlug)
22    
23    mpiPlug->numberProcessors = MPI::COMM_WORLD.Get_size();
24    mpiPlug->myNode = worldRank;
25 <  
25 >
26 >  MolToProcMap = new int[entryPlug->n_mol];
27 >  MolComponentType = new int[entryPlug->n_mol];
28 >  AtomToProcMap = new int[entryPlug->n_atoms];
29 >
30    mpiSim = this;
31    wrapMeSimParallel( this );
32   }
# Line 29 | Line 34 | mpiSimulation::~mpiSimulation(){
34  
35   mpiSimulation::~mpiSimulation(){
36    
37 +  delete[] MolToProcMap;
38 +  delete[] MolComponentType;
39 +  delete[] AtomToProcMap;
40 +
41    delete mpiPlug;
42    // perhaps we should let fortran know the party is over.
43    
44   }
45  
37 –
38 –
46   int* mpiSimulation::divideLabor( void ){
47  
48    int* globalIndex;
49  
50    int nComponents;
51    MoleculeStamp** compStamps;
52 +  randomSPRNG *myRandom;
53    int* componentsNmol;
54 +  int* AtomsPerProc;
55  
56    double numerator;
57    double denominator;
58    double precast;
59 +  double x, y, a;
60 +  int old_atoms, add_atoms, new_atoms;
61  
62    int nTarget;
63    int molIndex, atomIndex, compIndex, compStart;
64    int done;
65    int nLocal, molLocal;
66 <  int i, index;
66 >  int i, j, loops, which_proc, nmol_local, natoms_local;
67 >  int nmol_global, natoms_global;
68 >  int local_index, index;
69    int smallDiff, bigDiff;
70 +  int baseSeed = BASE_SEED;
71  
72    int testSum;
73  
74    nComponents = entryPlug->nComponents;
75    compStamps = entryPlug->compStamps;
76    componentsNmol = entryPlug->componentsNmol;
77 <
77 >  AtomsPerProc = new int[mpiPlug->numberProcessors];
78 >  
79    mpiPlug->nAtomsGlobal = entryPlug->n_atoms;
80    mpiPlug->nBondsGlobal = entryPlug->n_bonds;
81    mpiPlug->nBendsGlobal = entryPlug->n_bends;
# Line 68 | Line 83 | int* mpiSimulation::divideLabor( void ){
83    mpiPlug->nSRIGlobal = entryPlug->n_SRI;
84    mpiPlug->nMolGlobal = entryPlug->n_mol;
85  
86 +  myRandom = new randomSPRNG( baseSeed );
87  
88 +  a = 3.0 * (double)mpiPlug->nMolGlobal / (double)mpiPlug->nAtomsGlobal;
89  
90 +  // Initialize things that we'll send out later:
91 +  for (i = 0; i < mpiPlug->numberProcessors; i++ ) {
92 +    AtomsPerProc[i] = 0;
93 +  }
94 +  for (i = 0; i < mpiPlug->nMolGlobal; i++ ) {
95 +    // default to an error condition:
96 +    MolToProcMap[i] = -1;
97 +    MolComponentType[i] = -1;
98 +  }
99 +  for (i = 0; i < mpiPlug->nAtomsGlobal; i++ ) {
100 +    // default to an error condition:
101 +    AtomToProcMap[i] = -1;
102 +  }
103 +    
104 +  if (mpiPlug->myNode == 0) {
105 +    numerator = (double) entryPlug->n_atoms;
106 +    denominator = (double) mpiPlug->numberProcessors;
107 +    precast = numerator / denominator;
108 +    nTarget = (int)( precast + 0.5 );
109  
110 +    // Build the array of molecule component types first
111 +    molIndex = 0;
112 +    for (i=0; i < nComponents; i++) {
113 +      for (j=0; j < componentsNmol[i]; j++) {        
114 +        MolComponentType[molIndex] = i;
115 +        molIndex++;
116 +      }
117 +    }
118  
119 +    atomIndex = 0;
120  
121 +    for (i = 0; i < molIndex; i++ ) {
122  
123 +      done = 0;
124 +      loops = 0;
125  
126 +      while( !done ){
127 +        loops++;
128 +        
129 +        // Pick a processor at random
130  
131 +        which_proc = (int) (myRandom->getRandom() * mpiPlug->numberProcessors);
132  
133 +        // How many atoms does this processor have?
134 +        
135 +        old_atoms = AtomsPerProc[which_proc];
136 +        add_atoms = compStamps[MolComponentType[i]]->getNAtoms();
137 +        new_atoms = old_atoms + add_atoms;
138  
139 <
140 <  numerator = (double) entryPlug->n_atoms;
141 <  denominator = (double) mpiPlug->numberProcessors;
142 <  precast = numerator / denominator;
143 <  nTarget = (int)( precast + 0.5 );
144 <  
145 <  molIndex = 0;
146 <  atomIndex = 0;
147 <  compIndex = 0;
148 <  compStart = 0;
149 <  for( i=0; i<(mpiPlug->numberProcessors-1); i++){
139 >        // If we've been through this loop too many times, we need
140 >        // to just give up and assign the molecule to this processor
141 >        // and be done with it.
142 >        
143 >        if (loops > 100) {          
144 >          sprintf( painCave.errMsg,
145 >                   "I've tried 100 times to assign molecule %d to a "
146 >                   " processor, but can't find a good spot.\n"  
147 >                   "I'm assigning it at random to processor %d.\n",
148 >                   i, which_proc);
149 >          painCave.isFatal = 0;
150 >          simError();
151 >          
152 >          MolToProcMap[i] = which_proc;
153 >          AtomsPerProc[which_proc] += add_atoms;
154 >          for (j = 0 ; j < add_atoms; j++ ) {
155 >            AtomToProcMap[atomIndex] = which_proc;
156 >            atomIndex++;
157 >          }
158 >          done = 1;
159 >          continue;
160 >        }
161      
162 <    done = 0;
163 <    nLocal = 0;
95 <    molLocal = 0;
96 <
97 <    if( i == mpiPlug->myNode ){
98 <      mpiPlug->myMolStart = molIndex;
99 <      mpiPlug->myAtomStart = atomIndex;
100 <    }
162 >        // If we can add this molecule to this processor without sending
163 >        // it above nTarget, then go ahead and do it:
164      
165 <    while( !done ){
166 <      
167 <      if( (molIndex-compStart) >= componentsNmol[compIndex] ){
168 <        compStart = molIndex;
169 <        compIndex++;
170 <        continue;
171 <      }
165 >        if (new_atoms <= nTarget) {
166 >          MolToProcMap[i] = which_proc;
167 >          AtomsPerProc[which_proc] += add_atoms;
168 >          for (j = 0 ; j < add_atoms; j++ ) {
169 >            AtomToProcMap[atomIndex] = which_proc;
170 >            atomIndex++;
171 >          }
172 >          done = 1;
173 >          continue;
174 >        }
175  
176 <      nLocal += compStamps[compIndex]->getNAtoms();
177 <      atomIndex += compStamps[compIndex]->getNAtoms();
178 <      molIndex++;
179 <      molLocal++;
176 >
177 >        // The only situation left is when new_atoms > nTarget.  We
178 >        // want to accept this with some probability that dies off the
179 >        // farther we are from nTarget
180 >
181 >        // roughly:  x = new_atoms - nTarget
182 >        //           Pacc(x) = exp(- a * x)
183 >        // where a = penalty / (average atoms per molecule)
184 >
185 >        x = (double) (new_atoms - nTarget);
186 >        y = myRandom->getRandom();
187        
188 <      if ( nLocal == nTarget ) done = 1;
189 <      
190 <      else if( nLocal < nTarget ){
191 <        smallDiff = nTarget - nLocal;
192 <      }
193 <      else if( nLocal > nTarget ){
194 <        bigDiff = nLocal - nTarget;
195 <        
196 <        if( bigDiff < smallDiff ) done = 1;
197 <        else{
198 <          molIndex--;
199 <          molLocal--;
200 <          atomIndex -= compStamps[compIndex]->getNAtoms();
128 <          nLocal -= compStamps[compIndex]->getNAtoms();
129 <          done = 1;
130 <        }
188 >        if (y < exp(- a * x)) {
189 >          MolToProcMap[i] = which_proc;
190 >          AtomsPerProc[which_proc] += add_atoms;
191 >          for (j = 0 ; j < add_atoms; j++ ) {
192 >            AtomToProcMap[atomIndex] = which_proc;
193 >            atomIndex++;
194 >           }
195 >          done = 1;
196 >          continue;
197 >        } else {
198 >          continue;
199 >        }      
200 >        
201        }
202      }
203 +
204 +    // Spray out this nonsense to all other processors:
205 +
206 +    MPI::COMM_WORLD.Bcast(MolToProcMap, mpiPlug->nMolGlobal,
207 +                          MPI_INT, 0);
208 +
209 +    MPI::COMM_WORLD.Bcast(AtomToProcMap, mpiPlug->nAtomsGlobal,
210 +                          MPI_INT, 0);
211 +
212 +    MPI::COMM_WORLD.Bcast(MolComponentType, mpiPlug->nMolGlobal,
213 +                          MPI_INT, 0);
214 +
215 +    MPI::COMM_WORLD.Bcast(AtomsPerProc, mpiPlug->numberProcessors,
216 +                          MPI_INT, 0);    
217 +  } else {
218 +
219 +    // Listen to your marching orders from processor 0:
220      
221 <    if( i == mpiPlug->myNode ){
222 <      mpiPlug->myMolEnd = (molIndex - 1);
136 <      mpiPlug->myAtomEnd = (atomIndex - 1);
137 <      mpiPlug->myNlocal = nLocal;
138 <      mpiPlug->myMol = molLocal;
139 <    }
221 >    MPI::COMM_WORLD.Bcast(MolToProcMap, mpiPlug->nMolGlobal,
222 >                          MPI_INT, 0);
223      
224 <    numerator = (double)( entryPlug->n_atoms - atomIndex );
225 <    denominator = (double)( mpiPlug->numberProcessors - (i+1) );
226 <    precast = numerator / denominator;
227 <    nTarget = (int)( precast + 0.5 );
224 >    MPI::COMM_WORLD.Bcast(AtomToProcMap, mpiPlug->nAtomsGlobal,
225 >                          MPI_INT, 0);
226 >
227 >    MPI::COMM_WORLD.Bcast(MolComponentType, mpiPlug->nMolGlobal,
228 >                          MPI_INT, 0);
229 >    
230 >    MPI::COMM_WORLD.Bcast(AtomsPerProc, mpiPlug->numberProcessors,
231 >                          MPI_INT, 0);
232 >
233 >
234    }
146 –  
147 –  if( mpiPlug->myNode == mpiPlug->numberProcessors-1 ){
148 –      mpiPlug->myMolStart = molIndex;
149 –      mpiPlug->myAtomStart = atomIndex;
235  
151 –      nLocal = 0;
152 –      molLocal = 0;
153 –      while( compIndex < nComponents ){
154 –        
155 –        if( (molIndex-compStart) >= componentsNmol[compIndex] ){
156 –          compStart = molIndex;
157 –          compIndex++;
158 –          continue;
159 –        }
236  
237 <        nLocal += compStamps[compIndex]->getNAtoms();
238 <        atomIndex += compStamps[compIndex]->getNAtoms();
239 <        molIndex++;
240 <        molLocal++;
241 <      }
242 <      
243 <      mpiPlug->myMolEnd = (molIndex - 1);
168 <      mpiPlug->myAtomEnd = (atomIndex - 1);
169 <      mpiPlug->myNlocal = nLocal;  
170 <      mpiPlug->myMol = molLocal;
237 >  // Let's all check for sanity:
238 >
239 >  nmol_local = 0;
240 >  for (i = 0 ; i < mpiPlug->nMolGlobal; i++ ) {
241 >    if (MolToProcMap[i] == mpiPlug->myNode) {
242 >      nmol_local++;
243 >    }
244    }
245  
246 +  natoms_local = 0;
247 +  for (i = 0; i < mpiPlug->nAtomsGlobal; i++) {
248 +    if (AtomToProcMap[i] == mpiPlug->myNode) {
249 +      natoms_local++;      
250 +    }
251 +  }
252  
253 <  MPI_Allreduce( &nLocal, &testSum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
253 >  MPI::COMM_WORLD.Allreduce(&nmol_local,&nmol_global,1,MPI_INT,MPI_SUM);
254 >  MPI::COMM_WORLD.Allreduce(&natoms_local,&natoms_global,1,MPI_INT,MPI_SUM);
255    
256 <  if( mpiPlug->myNode == 0 ){
257 <    if( testSum != entryPlug->n_atoms ){
258 <      sprintf( painCave.errMsg,
259 <               "The summ of all nLocals, %d, did not equal the total number of atoms, %d.\n",
260 <               testSum, entryPlug->n_atoms );
261 <      painCave.isFatal = 1;
262 <      simError();
183 <    }
256 >  if( nmol_global != entryPlug->n_mol ){
257 >    sprintf( painCave.errMsg,
258 >             "The sum of all nmol_local, %d, did not equal the "
259 >             "total number of molecules, %d.\n",
260 >             nmol_global, entryPlug->n_mol );
261 >    painCave.isFatal = 1;
262 >    simError();
263    }
264 +  
265 +  if( natoms_global != entryPlug->n_atoms ){
266 +    sprintf( painCave.errMsg,
267 +             "The sum of all natoms_local, %d, did not equal the "
268 +             "total number of atoms, %d.\n",
269 +             natoms_global, entryPlug->n_atoms );
270 +    painCave.isFatal = 1;
271 +    simError();
272 +  }
273  
274    sprintf( checkPointMsg,
275             "Successfully divided the molecules among the processors.\n" );
276    MPIcheckPoint();
277  
278 <  // lets create the identity array
278 >  mpiPlug->myNMol = nmol_local;
279 >  mpiPlug->myNlocal = natoms_local;
280  
281    globalIndex = new int[mpiPlug->myNlocal];
282 <  index = mpiPlug->myAtomStart;
283 <  for( i=0; i<mpiPlug->myNlocal; i++){
284 <    globalIndex[i] = index;
285 <    index++;
282 >  local_index = 0;
283 >  for (i = 0; i < mpiPlug->nAtomsGlobal; i++) {
284 >    if (AtomToProcMap[i] == mpiPlug->myNode) {
285 >      globalIndex[local_index] = i;
286 >      local_index++;
287 >    }
288    }
289 <
289 >  
290    return globalIndex;
291   }
292  
# Line 205 | Line 296 | void mpiSimulation::mpiRefresh( void ){
296    int isError, i;
297    int *globalIndex = new int[mpiPlug->myNlocal];
298  
299 <  for(i=0; i<mpiPlug->myNlocal; i++) globalIndex[i] = entryPlug->atoms[i]->getGlobalIndex();
299 >  // Fortran indexing needs to be increased by 1 in order to get the 2 languages to
300 >  // not barf
301  
302 +  for(i=0; i<mpiPlug->myNlocal; i++) globalIndex[i] = entryPlug->atoms[i]->getGlobalIndex()+1;
303 +
304    
305    isError = 0;
306    setFsimParallel( mpiPlug, &(entryPlug->n_atoms), globalIndex, &isError );

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines