OOPSE/libmdtools/mpiSimulation.cpp

#ifdef IS_MPI

#include <cstdlib>
#include <cstring>
#include <cmath>
#include <mpi.h>
#include <mpi++.h>

#include "mpiSimulation.hpp"
#include "simError.h"
#include "fortranWrappers.hpp"
#include "randomSPRNG.hpp"

#define BASE_SEED 123456789

mpiSimulation* mpiSim;

mpiSimulation::mpiSimulation(SimInfo* the_entryPlug)
{
  entryPlug = the_entryPlug;
  mpiPlug = new mpiSimData;
  
  mpiPlug->numberProcessors = MPI::COMM_WORLD.Get_size();
  mpiPlug->myNode = worldRank;

  MolToProcMap = new int[entryPlug->n_mol];
  MolComponentType = new int[entryPlug->n_mol];

  AtomToProcMap = new int[entryPlug->n_atoms];

  mpiSim = this;
  wrapMeSimParallel( this );
}


mpiSimulation::~mpiSimulation(){
  
  delete[] MolToProcMap;
  delete[] MolComponentType;
  delete[] AtomToProcMap;

  delete mpiPlug;
  // perhaps we should let fortran know the party is over.
  
}

int* mpiSimulation::divideLabor( void ){

  int* globalIndex;

  int nComponents;
  MoleculeStamp** compStamps;
  randomSPRNG *myRandom;
  int* componentsNmol;
  int* AtomsPerProc;

  double numerator;
  double denominator;
  double precast;
  double x, y, a;
  int old_atoms, add_atoms, new_atoms;

  int nTarget;
  int molIndex, atomIndex, compIndex, compStart;
  int done;
  int nLocal, molLocal;
  int i, j, loops, which_proc, nmol_local, natoms_local;
  int nmol_global, natoms_global;
  int local_index, index;
  int smallDiff, bigDiff;
  int baseSeed = BASE_SEED;

  int testSum;

  nComponents = entryPlug->nComponents;
  compStamps = entryPlug->compStamps;
  componentsNmol = entryPlug->componentsNmol;
  AtomsPerProc = new int[mpiPlug->numberProcessors];
  
  mpiPlug->nAtomsGlobal = entryPlug->n_atoms;
  mpiPlug->nBondsGlobal = entryPlug->n_bonds;
  mpiPlug->nBendsGlobal = entryPlug->n_bends;
  mpiPlug->nTorsionsGlobal = entryPlug->n_torsions;
  mpiPlug->nSRIGlobal = entryPlug->n_SRI;
  mpiPlug->nMolGlobal = entryPlug->n_mol;

  myRandom = new randomSPRNG( baseSeed );

  a = (double)mpiPlug->nMolGlobal / (double)mpiPlug->nAtomsGlobal;

  // Initialize things that we'll send out later:
  for (i = 0; i < mpiPlug->numberProcessors; i++ ) {
    AtomsPerProc[i] = 0;
  }
  for (i = 0; i < mpiPlug->nMolGlobal; i++ ) {
    // default to an error condition:
    MolToProcMap[i] = -1;
    MolComponentType[i] = -1;
  }
  for (i = 0; i < mpiPlug->nAtomsGlobal; i++ ) {
    // default to an error condition:
    AtomToProcMap[i] = -1;
  }
    
  if (mpiPlug->myNode == 0) {
    numerator = (double) entryPlug->n_atoms;
    denominator = (double) mpiPlug->numberProcessors;
    precast = numerator / denominator;
    nTarget = (int)( precast + 0.5 );

    // Build the array of molecule component types first
    molIndex = 0;
    for (i=0; i < nComponents; i++) {
      for (j=0; j < componentsNmol[i]; j++) {        
        MolComponentType[molIndex] = i;
        molIndex++;
      }
    }

    atomIndex = 0;

    for (i = 0; i < molIndex; i++ ) {

      done = 0;
      loops = 0;

      while( !done ){
        loops++;
        
        // Pick a processor at random

        which_proc = (int) (myRandom->getRandom() * mpiPlug->numberProcessors);

        // How many atoms does this processor have?
        
        old_atoms = AtomsPerProc[which_proc];

        // If the processor already had too many atoms, just skip this
        // processor and try again.

        if (old_atoms >= nTarget) continue;

        add_atoms = compStamps[MolComponentType[i]]->getNAtoms();
        new_atoms = old_atoms + add_atoms;
    
        // If we can add this molecule to this processor without sending
        // it above nTarget, then go ahead and do it:
    
        if (new_atoms <= nTarget) {
          MolToProcMap[i] = which_proc;
          AtomsPerProc[which_proc] += add_atoms;
          for (j = 0 ; j < add_atoms; j++ ) {
            atomIndex++;
            AtomToProcMap[atomIndex] = which_proc;
          }
          done = 1;
          continue;
        }

        // If we've been through this loop too many times, we need
        // to just give up and assign the molecule to this processor
        // and be done with it. 
        
        if (loops > 100) {          
          sprintf( painCave.errMsg,
                   "I've tried 100 times to assign molecule %d to a "
                   " processor, but can't find a good spot.\n"  
                   "I'm assigning it at random to processor %d.\n",
                   i, which_proc);
          painCave.isFatal = 0;
          simError();
          
          MolToProcMap[i] = which_proc;
          AtomsPerProc[which_proc] += add_atoms;
          for (j = 0 ; j < add_atoms; j++ ) {
            atomIndex++;
            AtomToProcMap[atomIndex] = which_proc;
          }
          done = 1;
          continue;
        }

        // The only situation left is where old_atoms < nTarget, but
        // new_atoms > nTarget.   We want to accept this with some
        // probability that dies off the farther we are from nTarget

        // roughly:  x = new_atoms - nTarget
        //           Pacc(x) = exp(- a * x)
        // where a = 1 / (average atoms per molecule)

        x = (double) (new_atoms - nTarget);
        y = myRandom->getRandom();
        
        if (exp(- a * x) > y) {
          MolToProcMap[i] = which_proc;
          AtomsPerProc[which_proc] += add_atoms;
          for (j = 0 ; j < add_atoms; j++ ) {
            atomIndex++;
            AtomToProcMap[atomIndex] = which_proc;
          }
          done = 1;
          continue;
        } else {
          continue;
        }       
        
      }
    }

    // Spray out this nonsense to all other processors:

    MPI::COMM_WORLD.Bcast(MolToProcMap, mpiPlug->nMolGlobal, 
                          MPI_INT, 0);

    MPI::COMM_WORLD.Bcast(AtomToProcMap, mpiPlug->nAtomsGlobal, 
                          MPI_INT, 0);

    MPI::COMM_WORLD.Bcast(MolComponentType, mpiPlug->nMolGlobal, 
                          MPI_INT, 0);

    MPI::COMM_WORLD.Bcast(AtomsPerProc, mpiPlug->numberProcessors,
                          MPI_INT, 0);    
  } else {

    // Listen to your marching orders from processor 0:
    
    MPI::COMM_WORLD.Bcast(MolToProcMap, mpiPlug->nMolGlobal, 
                          MPI_INT, 0);
    
    MPI::COMM_WORLD.Bcast(AtomToProcMap, mpiPlug->nAtomsGlobal, 
                          MPI_INT, 0);

    MPI::COMM_WORLD.Bcast(MolComponentType, mpiPlug->nMolGlobal, 
                          MPI_INT, 0);
    
    MPI::COMM_WORLD.Bcast(AtomsPerProc, mpiPlug->numberProcessors,
                          MPI_INT, 0);
  }


  // Let's all check for sanity:

  nmol_local = 0;
  for (i = 0 ; i < mpiPlug->nMolGlobal; i++ ) {
    if (MolToProcMap[i] == mpiPlug->myNode) {
      nmol_local++;
    }
  }

  natoms_local = 0;
  for (i = 0; i < mpiPlug->nAtomsGlobal; i++) {
    if (AtomToProcMap[i] == mpiPlug->myNode) {
      natoms_local++;      
    }
  }

  MPI::COMM_WORLD.Allreduce(&nmol_local,&nmol_global,1,MPI_INT,MPI_SUM);
  MPI::COMM_WORLD.Allreduce(&natoms_local,&natoms_global,1,MPI_INT,MPI_SUM);
  
  if( nmol_global != entryPlug->n_mol ){
    sprintf( painCave.errMsg,
             "The sum of all nmol_local, %d, did not equal the "
             "total number of molecules, %d.\n",
             nmol_global, entryPlug->n_mol );
    painCave.isFatal = 1;
    simError();
  }
  
  if( natoms_global != entryPlug->n_atoms ){
    sprintf( painCave.errMsg,
             "The sum of all natoms_local, %d, did not equal the "
             "total number of atoms, %d.\n",
             natoms_global, entryPlug->n_atoms );
    painCave.isFatal = 1;
    simError();
  }

  sprintf( checkPointMsg,
           "Successfully divided the molecules among the processors.\n" );
  MPIcheckPoint();

  mpiPlug->myNMol = nmol_local;
  mpiPlug->myNlocal = natoms_local;

  globalIndex = new int[mpiPlug->myNlocal];
  local_index = 0;
  for (i = 0; i < mpiPlug->nAtomsGlobal; i++) {
    if (AtomToProcMap[i] == mpiPlug->myNode) {
      local_index++;
      globalIndex[local_index] = i;
    }
  }
 
  return globalIndex;
}


void mpiSimulation::mpiRefresh( void ){

  int isError, i;
  int *globalIndex = new int[mpiPlug->myNlocal];

  for(i=0; i<mpiPlug->myNlocal; i++) globalIndex[i] = entryPlug->atoms[i]->getGlobalIndex();

  
  isError = 0;
  setFsimParallel( mpiPlug, &(entryPlug->n_atoms), globalIndex, &isError );
  if( isError ){

    sprintf( painCave.errMsg,
             "mpiRefresh errror: fortran didn't like something we gave it.\n" );
    painCave.isFatal = 1;
    simError();
  }

  delete[] globalIndex;

  sprintf( checkPointMsg,
           " mpiRefresh successful.\n" );
  MPIcheckPoint();
}
 

#endif // is_mpi
Revision:	418
Committed:	Thu Mar 27 14:30:24 2003 UTC (21 years, 3 months ago) by mmeineke
File size:	8649 byte(s)
Log Message:	little bug fixes here and there.
#	User	Rev	Content
1	mmeineke	377	#ifdef IS_MPI
2
3			#include <cstdlib>
4			#include <cstring>
5	gezelter	416	#include <cmath>
6	mmeineke	377	#include <mpi.h>
7			#include <mpi++.h>
8
9			#include "mpiSimulation.hpp"
10			#include "simError.h"
11			#include "fortranWrappers.hpp"
12	gezelter	405	#include "randomSPRNG.hpp"
13	mmeineke	377
14	gezelter	416	#define BASE_SEED 123456789
15	mmeineke	377
16			mpiSimulation* mpiSim;
17
18			mpiSimulation::mpiSimulation(SimInfo* the_entryPlug)
19			{
20			entryPlug = the_entryPlug;
21			mpiPlug = new mpiSimData;
22
23			mpiPlug->numberProcessors = MPI::COMM_WORLD.Get_size();
24			mpiPlug->myNode = worldRank;
25	gezelter	405
26			MolToProcMap = new int[entryPlug->n_mol];
27			MolComponentType = new int[entryPlug->n_mol];
28
29			AtomToProcMap = new int[entryPlug->n_atoms];
30
31	mmeineke	377	mpiSim = this;
32			wrapMeSimParallel( this );
33			}
34
35
36			mpiSimulation::~mpiSimulation(){
37
38	mmeineke	418	delete[] MolToProcMap;
39			delete[] MolComponentType;
40			delete[] AtomToProcMap;
41
42	mmeineke	377	delete mpiPlug;
43			// perhaps we should let fortran know the party is over.
44
45			}
46
47			int* mpiSimulation::divideLabor( void ){
48
49			int* globalIndex;
50
51			int nComponents;
52			MoleculeStamp** compStamps;
53	gezelter	416	randomSPRNG *myRandom;
54	mmeineke	377	int* componentsNmol;
55	gezelter	405	int* AtomsPerProc;
56	mmeineke	377
57			double numerator;
58			double denominator;
59			double precast;
60	gezelter	405	double x, y, a;
61			int old_atoms, add_atoms, new_atoms;
62	mmeineke	377
63			int nTarget;
64			int molIndex, atomIndex, compIndex, compStart;
65			int done;
66			int nLocal, molLocal;
67	gezelter	416	int i, j, loops, which_proc, nmol_local, natoms_local;
68			int nmol_global, natoms_global;
69			int local_index, index;
70	mmeineke	377	int smallDiff, bigDiff;
71	gezelter	416	int baseSeed = BASE_SEED;
72	mmeineke	377
73			int testSum;
74
75			nComponents = entryPlug->nComponents;
76			compStamps = entryPlug->compStamps;
77			componentsNmol = entryPlug->componentsNmol;
78	gezelter	405	AtomsPerProc = new int[mpiPlug->numberProcessors];
79
80	mmeineke	377	mpiPlug->nAtomsGlobal = entryPlug->n_atoms;
81			mpiPlug->nBondsGlobal = entryPlug->n_bonds;
82			mpiPlug->nBendsGlobal = entryPlug->n_bends;
83			mpiPlug->nTorsionsGlobal = entryPlug->n_torsions;
84			mpiPlug->nSRIGlobal = entryPlug->n_SRI;
85			mpiPlug->nMolGlobal = entryPlug->n_mol;
86
87	gezelter	416	myRandom = new randomSPRNG( baseSeed );
88	chuckv	404
89	gezelter	405	a = (double)mpiPlug->nMolGlobal / (double)mpiPlug->nAtomsGlobal;
90	chuckv	404
91	gezelter	405	// Initialize things that we'll send out later:
92			for (i = 0; i < mpiPlug->numberProcessors; i++ ) {
93			AtomsPerProc[i] = 0;
94			}
95			for (i = 0; i < mpiPlug->nMolGlobal; i++ ) {
96			// default to an error condition:
97			MolToProcMap[i] = -1;
98			MolComponentType[i] = -1;
99			}
100			for (i = 0; i < mpiPlug->nAtomsGlobal; i++ ) {
101			// default to an error condition:
102			AtomToProcMap[i] = -1;
103			}
104
105			if (mpiPlug->myNode == 0) {
106			numerator = (double) entryPlug->n_atoms;
107			denominator = (double) mpiPlug->numberProcessors;
108			precast = numerator / denominator;
109			nTarget = (int)( precast + 0.5 );
110	chuckv	404
111	gezelter	405	// Build the array of molecule component types first
112			molIndex = 0;
113			for (i=0; i < nComponents; i++) {
114			for (j=0; j < componentsNmol[i]; j++) {
115			MolComponentType[molIndex] = i;
116			molIndex++;
117			}
118			}
119	chuckv	404
120	gezelter	405	atomIndex = 0;
121	chuckv	404
122	gezelter	405	for (i = 0; i < molIndex; i++ ) {
123	chuckv	404
124	gezelter	405	done = 0;
125			loops = 0;
126	chuckv	404
127	gezelter	405	while( !done ){
128			loops++;
129
130			// Pick a processor at random
131	chuckv	404
132	gezelter	416	which_proc = (int) (myRandom->getRandom() * mpiPlug->numberProcessors);
133	chuckv	404
134	gezelter	405	// How many atoms does this processor have?
135
136			old_atoms = AtomsPerProc[which_proc];
137	chuckv	404
138	gezelter	405	// If the processor already had too many atoms, just skip this
139			// processor and try again.
140	chuckv	404
141	gezelter	405	if (old_atoms >= nTarget) continue;
142	mmeineke	377
143	gezelter	416	add_atoms = compStamps[MolComponentType[i]]->getNAtoms();
144	gezelter	405	new_atoms = old_atoms + add_atoms;
145	mmeineke	377
146	gezelter	405	// If we can add this molecule to this processor without sending
147			// it above nTarget, then go ahead and do it:
148
149			if (new_atoms <= nTarget) {
150			MolToProcMap[i] = which_proc;
151			AtomsPerProc[which_proc] += add_atoms;
152			for (j = 0 ; j < add_atoms; j++ ) {
153			atomIndex++;
154			AtomToProcMap[atomIndex] = which_proc;
155			}
156			done = 1;
157			continue;
158			}
159	mmeineke	377
160	gezelter	405	// If we've been through this loop too many times, we need
161			// to just give up and assign the molecule to this processor
162			// and be done with it.
163
164			if (loops > 100) {
165			sprintf( painCave.errMsg,
166			"I've tried 100 times to assign molecule %d to a "
167			" processor, but can't find a good spot.\n"
168			"I'm assigning it at random to processor %d.\n",
169			i, which_proc);
170			painCave.isFatal = 0;
171			simError();
172
173			MolToProcMap[i] = which_proc;
174			AtomsPerProc[which_proc] += add_atoms;
175			for (j = 0 ; j < add_atoms; j++ ) {
176			atomIndex++;
177			AtomToProcMap[atomIndex] = which_proc;
178			}
179			done = 1;
180			continue;
181			}
182
183			// The only situation left is where old_atoms < nTarget, but
184			// new_atoms > nTarget. We want to accept this with some
185			// probability that dies off the farther we are from nTarget
186
187			// roughly: x = new_atoms - nTarget
188			// Pacc(x) = exp(- a * x)
189			// where a = 1 / (average atoms per molecule)
190
191			x = (double) (new_atoms - nTarget);
192	gezelter	416	y = myRandom->getRandom();
193	gezelter	405
194			if (exp(- a * x) > y) {
195			MolToProcMap[i] = which_proc;
196			AtomsPerProc[which_proc] += add_atoms;
197			for (j = 0 ; j < add_atoms; j++ ) {
198			atomIndex++;
199			AtomToProcMap[atomIndex] = which_proc;
200			}
201			done = 1;
202			continue;
203			} else {
204			continue;
205			}
206
207	mmeineke	377	}
208			}
209	gezelter	405
210			// Spray out this nonsense to all other processors:
211
212	mmeineke	418	MPI::COMM_WORLD.Bcast(MolToProcMap, mpiPlug->nMolGlobal,
213	gezelter	405	MPI_INT, 0);
214
215	mmeineke	418	MPI::COMM_WORLD.Bcast(AtomToProcMap, mpiPlug->nAtomsGlobal,
216	gezelter	405	MPI_INT, 0);
217
218	mmeineke	418	MPI::COMM_WORLD.Bcast(MolComponentType, mpiPlug->nMolGlobal,
219	gezelter	405	MPI_INT, 0);
220
221	mmeineke	418	MPI::COMM_WORLD.Bcast(AtomsPerProc, mpiPlug->numberProcessors,
222	gezelter	405	MPI_INT, 0);
223			} else {
224
225			// Listen to your marching orders from processor 0:
226	mmeineke	377
227	mmeineke	418	MPI::COMM_WORLD.Bcast(MolToProcMap, mpiPlug->nMolGlobal,
228	gezelter	405	MPI_INT, 0);
229	mmeineke	377
230	mmeineke	418	MPI::COMM_WORLD.Bcast(AtomToProcMap, mpiPlug->nAtomsGlobal,
231	gezelter	405	MPI_INT, 0);
232
233	mmeineke	418	MPI::COMM_WORLD.Bcast(MolComponentType, mpiPlug->nMolGlobal,
234	gezelter	405	MPI_INT, 0);
235
236	mmeineke	418	MPI::COMM_WORLD.Bcast(AtomsPerProc, mpiPlug->numberProcessors,
237	gezelter	405	MPI_INT, 0);
238	mmeineke	377	}
239
240
241	gezelter	405	// Let's all check for sanity:
242
243			nmol_local = 0;
244			for (i = 0 ; i < mpiPlug->nMolGlobal; i++ ) {
245			if (MolToProcMap[i] == mpiPlug->myNode) {
246			nmol_local++;
247			}
248	mmeineke	377	}
249
250	gezelter	405	natoms_local = 0;
251			for (i = 0; i < mpiPlug->nAtomsGlobal; i++) {
252			if (AtomToProcMap[i] == mpiPlug->myNode) {
253			natoms_local++;
254			}
255			}
256	mmeineke	377
257	gezelter	405	MPI::COMM_WORLD.Allreduce(&nmol_local,&nmol_global,1,MPI_INT,MPI_SUM);
258			MPI::COMM_WORLD.Allreduce(&natoms_local,&natoms_global,1,MPI_INT,MPI_SUM);
259	mmeineke	377
260	gezelter	405	if( nmol_global != entryPlug->n_mol ){
261			sprintf( painCave.errMsg,
262			"The sum of all nmol_local, %d, did not equal the "
263			"total number of molecules, %d.\n",
264			nmol_global, entryPlug->n_mol );
265			painCave.isFatal = 1;
266			simError();
267	mmeineke	377	}
268	gezelter	405
269			if( natoms_global != entryPlug->n_atoms ){
270			sprintf( painCave.errMsg,
271			"The sum of all natoms_local, %d, did not equal the "
272			"total number of atoms, %d.\n",
273			natoms_global, entryPlug->n_atoms );
274			painCave.isFatal = 1;
275			simError();
276			}
277	mmeineke	377
278			sprintf( checkPointMsg,
279			"Successfully divided the molecules among the processors.\n" );
280			MPIcheckPoint();
281
282	gezelter	405	mpiPlug->myNMol = nmol_local;
283			mpiPlug->myNlocal = natoms_local;
284	mmeineke	377
285			globalIndex = new int[mpiPlug->myNlocal];
286	gezelter	405	local_index = 0;
287			for (i = 0; i < mpiPlug->nAtomsGlobal; i++) {
288			if (AtomToProcMap[i] == mpiPlug->myNode) {
289	chuckv	406	local_index++;
290			globalIndex[local_index] = i;
291	gezelter	405	}
292	mmeineke	377	}
293	gezelter	416
294			return globalIndex;
295	mmeineke	377	}
296
297
298			void mpiSimulation::mpiRefresh( void ){
299
300			int isError, i;
301			int *globalIndex = new int[mpiPlug->myNlocal];
302
303			for(i=0; i<mpiPlug->myNlocal; i++) globalIndex[i] = entryPlug->atoms[i]->getGlobalIndex();
304
305
306			isError = 0;
307			setFsimParallel( mpiPlug, &(entryPlug->n_atoms), globalIndex, &isError );
308			if( isError ){
309
310			sprintf( painCave.errMsg,
311			"mpiRefresh errror: fortran didn't like something we gave it.\n" );
312			painCave.isFatal = 1;
313			simError();
314			}
315
316			delete[] globalIndex;
317
318			sprintf( checkPointMsg,
319			" mpiRefresh successful.\n" );
320			MPIcheckPoint();
321			}
322
323
324			#endif // is_mpi