mdtools/mpi_implementation/mpiSimulation.cpp

#include <cstdlib>
#include <cstring>
#include <mpi.h>

#include "mpiSimulation.hpp"
#include "simError.h"

extern "C"{
  void wrapsimparallelmod_( void (*wrapFunction)(void (*fSub)( mpiSimData*,
                                                               int*, int*, 
                                                               int*)));
}

void wrapSimParallel(void (*fSub)(mpiSimData*, int*, int*, int*));


mpiSimulation* mpiSim;

mpiSimulation::mpiSimulation(SimInfo* the_entryPlug)
{
  entryPlug = the_entryPlug;
  mpiPlug = new mpiSimData;
  
  mpiPlug->numberProcessors = MPI::COMM_WORLD.Get_size();
  mpiPlug->myNode = worldRank;
  
  mpiSim = this;
  wrapMe();

}


mpiSimulation::~mpiSimulation(){
  
  delete mpiPlug;
  // perhaps we should let fortran know the party is over.
  
}

void mpiSimulation::wrapMe(){

  wrapsimparallelmod_( wrapSimParallel );
}


int* mpiSimulation::divideLabor( void ){

  int* globalIndex;

  int nComponents;
  MoleculeStamp** compStamps;
  int* componentsNmol;

  double numerator;
  double denominator;
  double precast;

  int nTarget;
  int molIndex, atomIndex, compIndex, compStart;
  int done;
  int nLocal, molLocal;
  int i, index;
  int smallDiff, bigDiff;

  int testSum;

  nComponents = entryPlug->nComponents;
  compStamps = entryPlug->compStamps;
  componentsNmol = entryPlug->componentsNmol;

  mpiPlug->nAtomsGlobal = entryPlug->n_atoms;
  mpiPlug->nBondsGlobal = entryPlug->n_bonds;
  mpiPlug->nBendsGlobal = entryPlug->n_bends;
  mpiPlug->nTorsionsGlobal = entryPlug->n_torsions;
  mpiPlug->nSRIGlobal = entryPlug->n_SRI;
  mpiPlug->nMolGlobal = entryPlug->n_mol;

  numerator = (double) entryPlug->n_atoms;
  denominator = (double) mpiPlug->numberProcessors;
  precast = numerator / denominator;
  nTarget = (int)( precast + 0.5 );
  
  molIndex = 0;
  atomIndex = 0;
  compIndex = 0;
  compStart = 0;
  for( i=0; i<(mpiPlug->numberProcessors-1); i++){
    
    done = 0;
    nLocal = 0;
    molLocal = 0;

    if( i == mpiPlug->myNode ){
      mpiPlug->myMolStart = molIndex;
      mpiPlug->myAtomStart = atomIndex;
    }
    
    while( !done ){
      
      if( (molIndex-compStart) >= componentsNmol[compIndex] ){
        compStart = molIndex;
        compIndex++;
        continue;
      }

      nLocal += compStamps[compIndex]->getNAtoms();
      atomIndex += compStamps[compIndex]->getNAtoms();
      molIndex++;
      molLocal++;
      
      if ( nLocal == nTarget ) done = 1;
      
      else if( nLocal < nTarget ){
        smallDiff = nTarget - nLocal;
      }
      else if( nLocal > nTarget ){
        bigDiff = nLocal - nTarget;
        
        if( bigDiff < smallDiff ) done = 1;
        else{
          molIndex--;
          molLocal--;
          atomIndex -= compStamps[compIndex]->getNAtoms();
          nLocal -= compStamps[compIndex]->getNAtoms();
          done = 1;
        }
      }
    }
    
    if( i == mpiPlug->myNode ){
      mpiPlug->myMolEnd = (molIndex - 1);
      mpiPlug->myAtomEnd = (atomIndex - 1);
      mpiPlug->myNlocal = nLocal;
      mpiPlug->myMol = molLocal;
    }
    
    numerator = (double)( entryPlug->n_atoms - atomIndex );
    denominator = (double)( mpiPlug->numberProcessors - (i+1) );
    precast = numerator / denominator;
    nTarget = (int)( precast + 0.5 );
  }
  
  if( mpiPlug->myNode == mpiPlug->numberProcessors-1 ){
      mpiPlug->myMolStart = molIndex;
      mpiPlug->myAtomStart = atomIndex;

      nLocal = 0;
      molLocal = 0;
      while( compIndex < nComponents ){
        
        if( (molIndex-compStart) >= componentsNmol[compIndex] ){
          compStart = molIndex;
          compIndex++;
          continue;
        }

        nLocal += compStamps[compIndex]->getNAtoms();
        atomIndex += compStamps[compIndex]->getNAtoms();
        molIndex++;
        molLocal++;
      }
      
      mpiPlug->myMolEnd = (molIndex - 1);
      mpiPlug->myAtomEnd = (atomIndex - 1);
      mpiPlug->myNlocal = nLocal;  
      mpiPlug->myMol = molLocal;
  }


  MPI_Allreduce( &nLocal, &testSum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
  
  if( mpiPlug->myNode == 0 ){
    if( testSum != entryPlug->n_atoms ){
      sprintf( painCave.errMsg,
               "The summ of all nLocals, %d, did not equal the total number of atoms, %d.\n",
               testSum, entryPlug->n_atoms );
      painCave.isFatal = 1;
      simError();
    }
  }

  sprintf( checkPointMsg,
           "Successfully divided the molecules among the processors.\n" );
  MPIcheckPoint();

  // lets create the identity array

  globalIndex = new int[mpiPlug->myNlocal];
  index = mpiPlug->myAtomStart;
  for( i=0; i<mpiPlug->myNlocal; i++){
    globalIndex[i] = index;
    index++;
  }

  return globalIndex;
}


void wrapSimParallel(void (*fSub)(mpiSimData*, int*, int*, int*)){
  
  mpiSim->setInternal( fSub );
}


void mpiSimulation::mpiRefresh( void ){

  int isError, i;
  int *globalIndex = new int[mpiPlug->myNlocal];

  for(i=0; i<mpiPlug->myNlocal; i++) globalIndex[i] = entryPlug->atoms[i]->getGlobalIndex();

  
  isError = 0;
  setFsimParallel( mpiPlug, &(entryPlug->n_atoms), globalIndex, &isError );
  if( isError ){

    sprintf( painCave.errMsg,
             "mpiRefresh errror: fortran didn't like something we gave it.\n" );
    painCave.isFatal = 1;
    simError();
  }

  delete[] globalIndex;

  sprintf( checkPointMsg,
           " mpiRefresh successful.\n" );
  MPIcheckPoint();
}
  
Revision:	253
Committed:	Thu Jan 30 15:20:21 2003 UTC (21 years, 5 months ago) by chuckv
File size:	5113 byte(s)
Log Message:	Added a generic util code directory and moved Linux_ifc_machdep to it. MPI changes to compile MPI modules.
#	User	Rev	Content
1	chuckv	253	#include <cstdlib>
2	chuckv	194	#include <cstring>
3	chuckv	132	#include <mpi.h>
4	chuckv	122
5	chuckv	194	#include "mpiSimulation.hpp"
6			#include "simError.h"
7
8	mmeineke	245	extern "C"{
9			void wrapsimparallelmod_( void (wrapFunction)(void (fSub)( mpiSimData*,
10			int, int,
11			int*)));
12			}
13
14	chuckv	252	void wrapSimParallel(void (fSub)(mpiSimData, int, int, int*));
15	mmeineke	245
16
17	chuckv	215	mpiSimulation* mpiSim;
18	chuckv	194
19			mpiSimulation::mpiSimulation(SimInfo* the_entryPlug)
20	chuckv	122	{
21	chuckv	195	entryPlug = the_entryPlug;
22	chuckv	253	mpiPlug = new mpiSimData;
23	chuckv	195
24	chuckv	215	mpiPlug->numberProcessors = MPI::COMM_WORLD.Get_size();
25			mpiPlug->myNode = worldRank;
26	chuckv	195
27	mmeineke	245	mpiSim = this;
28			wrapMe();
29	chuckv	215
30	chuckv	122	}
31
32
33	chuckv	194	mpiSimulation::~mpiSimulation(){
34	chuckv	195
35	chuckv	215	delete mpiPlug;
36			// perhaps we should let fortran know the party is over.
37	chuckv	195
38	chuckv	194	}
39
40	mmeineke	245	void mpiSimulation::wrapMe(){
41	chuckv	194
42	mmeineke	245	wrapsimparallelmod_( wrapSimParallel );
43			}
44
45
46
47	chuckv	253	int* mpiSimulation::divideLabor( void ){
48	mmeineke	200
49	chuckv	253	int* globalIndex;
50
51	mmeineke	200	int nComponents;
52			MoleculeStamp** compStamps;
53			int* componentsNmol;
54
55	chuckv	195	double numerator;
56			double denominator;
57			double precast;
58
59			int nTarget;
60			int molIndex, atomIndex, compIndex, compStart;
61			int done;
62	chuckv	196	int nLocal, molLocal;
63	chuckv	253	int i, index;
64	chuckv	195	int smallDiff, bigDiff;
65
66			int testSum;
67
68	mmeineke	200	nComponents = entryPlug->nComponents;
69			compStamps = entryPlug->compStamps;
70			componentsNmol = entryPlug->componentsNmol;
71
72	chuckv	215	mpiPlug->nAtomsGlobal = entryPlug->n_atoms;
73			mpiPlug->nBondsGlobal = entryPlug->n_bonds;
74			mpiPlug->nBendsGlobal = entryPlug->n_bends;
75			mpiPlug->nTorsionsGlobal = entryPlug->n_torsions;
76			mpiPlug->nSRIGlobal = entryPlug->n_SRI;
77	chuckv	253	mpiPlug->nMolGlobal = entryPlug->n_mol;
78	mmeineke	202
79	chuckv	195	numerator = (double) entryPlug->n_atoms;
80	chuckv	215	denominator = (double) mpiPlug->numberProcessors;
81	chuckv	195	precast = numerator / denominator;
82			nTarget = (int)( precast + 0.5 );
83
84			molIndex = 0;
85			atomIndex = 0;
86			compIndex = 0;
87			compStart = 0;
88	chuckv	215	for( i=0; i<(mpiPlug->numberProcessors-1); i++){
89	chuckv	195
90			done = 0;
91			nLocal = 0;
92	chuckv	196	molLocal = 0;
93	chuckv	195
94	chuckv	215	if( i == mpiPlug->myNode ){
95			mpiPlug->myMolStart = molIndex;
96			mpiPlug->myAtomStart = atomIndex;
97	chuckv	195	}
98
99			while( !done ){
100
101			if( (molIndex-compStart) >= componentsNmol[compIndex] ){
102			compStart = molIndex;
103			compIndex++;
104			continue;
105			}
106
107			nLocal += compStamps[compIndex]->getNAtoms();
108			atomIndex += compStamps[compIndex]->getNAtoms();
109			molIndex++;
110	chuckv	196	molLocal++;
111	chuckv	195
112			if ( nLocal == nTarget ) done = 1;
113
114			else if( nLocal < nTarget ){
115			smallDiff = nTarget - nLocal;
116			}
117			else if( nLocal > nTarget ){
118			bigDiff = nLocal - nTarget;
119
120			if( bigDiff < smallDiff ) done = 1;
121			else{
122			molIndex--;
123	chuckv	196	molLocal--;
124	chuckv	195	atomIndex -= compStamps[compIndex]->getNAtoms();
125			nLocal -= compStamps[compIndex]->getNAtoms();
126			done = 1;
127			}
128			}
129			}
130
131	chuckv	215	if( i == mpiPlug->myNode ){
132			mpiPlug->myMolEnd = (molIndex - 1);
133			mpiPlug->myAtomEnd = (atomIndex - 1);
134			mpiPlug->myNlocal = nLocal;
135			mpiPlug->myMol = molLocal;
136	chuckv	195	}
137
138			numerator = (double)( entryPlug->n_atoms - atomIndex );
139	chuckv	215	denominator = (double)( mpiPlug->numberProcessors - (i+1) );
140	chuckv	195	precast = numerator / denominator;
141			nTarget = (int)( precast + 0.5 );
142			}
143
144	chuckv	215	if( mpiPlug->myNode == mpiPlug->numberProcessors-1 ){
145			mpiPlug->myMolStart = molIndex;
146			mpiPlug->myAtomStart = atomIndex;
147	chuckv	195
148			nLocal = 0;
149	chuckv	196	molLocal = 0;
150	chuckv	195	while( compIndex < nComponents ){
151
152			if( (molIndex-compStart) >= componentsNmol[compIndex] ){
153			compStart = molIndex;
154			compIndex++;
155			continue;
156			}
157
158			nLocal += compStamps[compIndex]->getNAtoms();
159			atomIndex += compStamps[compIndex]->getNAtoms();
160			molIndex++;
161	chuckv	196	molLocal++;
162	chuckv	195	}
163
164	chuckv	215	mpiPlug->myMolEnd = (molIndex - 1);
165			mpiPlug->myAtomEnd = (atomIndex - 1);
166			mpiPlug->myNlocal = nLocal;
167			mpiPlug->myMol = molLocal;
168	chuckv	195	}
169
170
171	chuckv	215	MPI_Allreduce( &nLocal, &testSum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
172	chuckv	195
173	chuckv	215	if( mpiPlug->myNode == 0 ){
174	chuckv	195	if( testSum != entryPlug->n_atoms ){
175			sprintf( painCave.errMsg,
176			"The summ of all nLocals, %d, did not equal the total number of atoms, %d.\n",
177			testSum, entryPlug->n_atoms );
178			painCave.isFatal = 1;
179			simError();
180			}
181			}
182
183			sprintf( checkPointMsg,
184			"Successfully divided the molecules among the processors.\n" );
185			MPIcheckPoint();
186	chuckv	196
187			// lets create the identity array
188	chuckv	253
189			globalIndex = new int[mpiPlug->myNlocal];
190			index = mpiPlug->myAtomStart;
191			for( i=0; i<mpiPlug->myNlocal; i++){
192			globalIndex[i] = index;
193			index++;
194			}
195
196			return globalIndex;
197	chuckv	195	}
198	mmeineke	245
199
200	chuckv	252	void wrapSimParallel(void (fSub)(mpiSimData, int, int, int*)){
201	mmeineke	245
202			mpiSim->setInternal( fSub );
203			}
204	chuckv	253
205
206			void mpiSimulation::mpiRefresh( void ){
207
208			int isError, i;
209			int *globalIndex = new int[mpiPlug->myNlocal];
210
211			for(i=0; i<mpiPlug->myNlocal; i++) globalIndex[i] = entryPlug->atoms[i]->getGlobalIndex();
212
213
214			isError = 0;
215			setFsimParallel( mpiPlug, &(entryPlug->n_atoms), globalIndex, &isError );
216			if( isError ){
217
218			sprintf( painCave.errMsg,
219			"mpiRefresh errror: fortran didn't like something we gave it.\n" );
220			painCave.isFatal = 1;
221			simError();
222			}
223
224			delete[] globalIndex;
225
226			sprintf( checkPointMsg,
227			" mpiRefresh successful.\n" );
228			MPIcheckPoint();
229			}
230