# | Line 1 | Line 1 | |
---|---|---|
1 | + | #define _FILE_OFFSET_BITS 64 |
2 | + | |
3 | #include <cstring> | |
4 | #include <iostream> | |
5 | #include <fstream> | |
# | Line 5 | Line 7 | |
7 | #ifdef IS_MPI | |
8 | #include <mpi.h> | |
9 | #include "mpiSimulation.hpp" | |
10 | < | #define TAKE_THIS_TAG 0 |
10 | > | #define TAKE_THIS_TAG_CHAR 1 |
11 | > | #define TAKE_THIS_TAG_INT 2 |
12 | > | |
13 | > | namespace dWrite{ |
14 | > | void nodeZeroError( void ); |
15 | > | void anonymousNodeDie( void ); |
16 | > | } |
17 | > | |
18 | > | using namespace dWrite; |
19 | #endif //is_mpi | |
20 | ||
21 | #include "ReadWrite.hpp" | |
22 | #include "simError.h" | |
23 | ||
14 | – | |
15 | – | |
16 | – | |
17 | – | |
24 | DumpWriter::DumpWriter( SimInfo* the_entry_plug ){ | |
25 | ||
26 | entry_plug = the_entry_plug; | |
# | Line 23 | Line 29 | DumpWriter::DumpWriter( SimInfo* the_entry_plug ){ | |
29 | if(worldRank == 0 ){ | |
30 | #endif // is_mpi | |
31 | ||
26 | – | |
27 | – | |
32 | strcpy( outName, entry_plug->sampleName ); | |
33 | ||
34 | outFile.open(outName, ios::out | ios::trunc ); | |
# | Line 37 | Line 41 | DumpWriter::DumpWriter( SimInfo* the_entry_plug ){ | |
41 | painCave.isFatal = 1; | |
42 | simError(); | |
43 | } | |
44 | < | |
44 | > | |
45 | //outFile.setf( ios::scientific ); | |
46 | ||
47 | #ifdef IS_MPI | |
# | Line 69 | Line 73 | void DumpWriter::writeDump( double currentTime ){ | |
73 | char writeLine[BUFFERSIZE]; | |
74 | ||
75 | int i; | |
76 | + | #ifdef IS_MPI |
77 | + | int j, which_node, done, which_atom, local_index; |
78 | + | #else //is_mpi |
79 | + | int nAtoms = entry_plug->n_atoms; |
80 | + | #endif //is_mpi |
81 | + | |
82 | double q[4]; | |
83 | DirectionalAtom* dAtom; | |
74 | – | int nAtoms = entry_plug->n_atoms; |
84 | Atom** atoms = entry_plug->atoms; | |
85 | + | double pos[3], vel[3]; |
86 | ||
87 | ||
88 | #ifndef IS_MPI | |
89 | ||
90 | outFile << nAtoms << "\n"; | |
91 | ||
92 | < | outFile << currentTime << "\t" |
93 | < | << entry_plug->box_x << "\t" |
94 | < | << entry_plug->box_y << "\t" |
95 | < | << entry_plug->box_z << "\n"; |
92 | > | outFile << currentTime << ";\t" |
93 | > | << entry_plug->Hmat[0][0] << "\t" |
94 | > | << entry_plug->Hmat[1][0] << "\t" |
95 | > | << entry_plug->Hmat[2][0] << ";\t" |
96 | > | |
97 | > | << entry_plug->Hmat[0][1] << "\t" |
98 | > | << entry_plug->Hmat[1][1] << "\t" |
99 | > | << entry_plug->Hmat[2][1] << ";\t" |
100 | > | |
101 | > | << entry_plug->Hmat[0][2] << "\t" |
102 | > | << entry_plug->Hmat[1][2] << "\t" |
103 | > | << entry_plug->Hmat[2][2] << ";\n"; |
104 | ||
105 | for( i=0; i<nAtoms; i++ ){ | |
106 | ||
107 | + | atoms[i]->getPos(pos); |
108 | + | atoms[i]->getVel(vel); |
109 | ||
110 | sprintf( tempBuffer, | |
111 | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", | |
112 | atoms[i]->getType(), | |
113 | < | atoms[i]->getX(), |
114 | < | atoms[i]->getY(), |
115 | < | atoms[i]->getZ(), |
116 | < | atoms[i]->get_vx(), |
117 | < | atoms[i]->get_vy(), |
118 | < | atoms[i]->get_vz()); |
113 | > | pos[0], |
114 | > | pos[1], |
115 | > | pos[2], |
116 | > | vel[0], |
117 | > | vel[1], |
118 | > | vel[2]); |
119 | strcpy( writeLine, tempBuffer ); | |
120 | ||
121 | if( atoms[i]->isDirectional() ){ | |
# | Line 123 | Line 143 | void DumpWriter::writeDump( double currentTime ){ | |
143 | ||
144 | #else // is_mpi | |
145 | ||
146 | < | int masterIndex; |
147 | < | int nodeAtomsStart; |
128 | < | int nodeAtomsEnd; |
129 | < | int mpiErr; |
130 | < | int sendError; |
131 | < | int procIndex; |
132 | < | |
133 | < | MPI_Status istatus[MPI_STATUS_SIZE]; |
146 | > | // first thing first, suspend fatalities. |
147 | > | painCave.isEventLoop = 1; |
148 | ||
149 | < | |
150 | < | // write out header and node 0's coordinates |
149 | > | int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone |
150 | > | int haveError; |
151 | ||
152 | + | MPI_Status istatus; |
153 | + | int *AtomToProcMap = mpiSim->getAtomToProcMap(); |
154 | + | |
155 | + | // write out header and node 0's coordinates |
156 | + | |
157 | if( worldRank == 0 ){ | |
158 | outFile << mpiSim->getTotAtoms() << "\n"; | |
159 | + | |
160 | + | outFile << currentTime << ";\t" |
161 | + | << entry_plug->Hmat[0][0] << "\t" |
162 | + | << entry_plug->Hmat[1][0] << "\t" |
163 | + | << entry_plug->Hmat[2][0] << ";\t" |
164 | ||
165 | < | outFile << currentTime << "\t" |
166 | < | << entry_plug->box_x << "\t" |
167 | < | << entry_plug->box_y << "\t" |
144 | < | << entry_plug->box_z << "\n"; |
145 | < | |
146 | < | masterIndex = 0; |
147 | < | for( i=0; i<nAtoms; i++ ){ |
165 | > | << entry_plug->Hmat[0][1] << "\t" |
166 | > | << entry_plug->Hmat[1][1] << "\t" |
167 | > | << entry_plug->Hmat[2][1] << ";\t" |
168 | ||
169 | < | sprintf( tempBuffer, |
170 | < | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", |
171 | < | atoms[i]->getType(), |
172 | < | atoms[i]->getX(), |
173 | < | atoms[i]->getY(), |
174 | < | atoms[i]->getZ(), |
175 | < | atoms[i]->get_vx(), |
176 | < | atoms[i]->get_vy(), |
177 | < | atoms[i]->get_vz()); |
178 | < | strcpy( writeLine, tempBuffer ); |
169 | > | << entry_plug->Hmat[0][2] << "\t" |
170 | > | << entry_plug->Hmat[1][2] << "\t" |
171 | > | << entry_plug->Hmat[2][2] << ";\n"; |
172 | > | |
173 | > | outFile.flush(); |
174 | > | for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) { |
175 | > | // Get the Node number which has this atom; |
176 | > | |
177 | > | which_node = AtomToProcMap[i]; |
178 | > | |
179 | > | if (which_node == 0 ) { |
180 | ||
181 | < | if( atoms[i]->isDirectional() ){ |
181 | > | haveError = 0; |
182 | > | which_atom = i; |
183 | > | local_index=-1; |
184 | > | for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) { |
185 | > | if (atoms[j]->getGlobalIndex() == which_atom) local_index = j; |
186 | > | } |
187 | > | if (local_index != -1) { |
188 | > | //format the line |
189 | ||
190 | < | dAtom = (DirectionalAtom *)atoms[i]; |
191 | < | dAtom->getQ( q ); |
190 | > | atoms[local_index]->getPos(pos); |
191 | > | atoms[local_index]->getVel(vel); |
192 | > | |
193 | > | sprintf( tempBuffer, |
194 | > | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", |
195 | > | atoms[local_index]->getType(), |
196 | > | pos[0], |
197 | > | pos[1], |
198 | > | pos[2], |
199 | > | vel[0], |
200 | > | vel[1], |
201 | > | vel[2]); // check here. |
202 | > | strcpy( writeLine, tempBuffer ); |
203 | ||
204 | < | sprintf( tempBuffer, |
205 | < | "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", |
206 | < | q[0], |
207 | < | q[1], |
208 | < | q[2], |
209 | < | q[3], |
210 | < | dAtom->getJx(), |
211 | < | dAtom->getJy(), |
212 | < | dAtom->getJz()); |
213 | < | strcat( writeLine, tempBuffer ); |
204 | > | if( atoms[local_index]->isDirectional() ){ |
205 | > | |
206 | > | dAtom = (DirectionalAtom *)atoms[local_index]; |
207 | > | dAtom->getQ( q ); |
208 | > | |
209 | > | sprintf( tempBuffer, |
210 | > | "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", |
211 | > | q[0], |
212 | > | q[1], |
213 | > | q[2], |
214 | > | q[3], |
215 | > | dAtom->getJx(), |
216 | > | dAtom->getJy(), |
217 | > | dAtom->getJz()); |
218 | > | strcat( writeLine, tempBuffer ); |
219 | > | |
220 | > | } |
221 | > | else |
222 | > | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); |
223 | > | } |
224 | > | else { |
225 | > | sprintf(painCave.errMsg, |
226 | > | "Atom %d not found on processor %d\n", |
227 | > | i, worldRank ); |
228 | > | haveError= 1; |
229 | > | simError(); |
230 | > | } |
231 | > | |
232 | > | if(haveError) nodeZeroError(); |
233 | > | |
234 | } | |
235 | < | else |
236 | < | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); |
235 | > | else { |
236 | > | myStatus = 1; |
237 | > | MPI_Send(&myStatus, 1, MPI_INT, which_node, |
238 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
239 | > | MPI_Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG_INT, |
240 | > | MPI_COMM_WORLD); |
241 | > | MPI_Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node, |
242 | > | TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD, &istatus); |
243 | > | MPI_Recv(&myStatus, 1, MPI_INT, which_node, |
244 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
245 | ||
246 | + | if(!myStatus) nodeZeroError(); |
247 | + | |
248 | + | } |
249 | + | |
250 | outFile << writeLine; | |
251 | < | masterIndex++; |
251 | > | outFile.flush(); |
252 | } | |
253 | < | outFile.flush(); |
254 | < | } |
253 | > | |
254 | > | // kill everyone off: |
255 | > | myStatus = -1; |
256 | > | for (j = 0; j < mpiSim->getNumberProcessors(); j++) { |
257 | > | MPI_Send(&myStatus, 1, MPI_INT, j, |
258 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
259 | > | } |
260 | ||
261 | < | sprintf( checkPointMsg, |
186 | < | "Sucessfully wrote node 0's dump configuration.\n"); |
187 | < | MPIcheckPoint(); |
261 | > | } else { |
262 | ||
263 | < | for (procIndex = 1; procIndex < mpiSim->getNumberProcessors(); |
264 | < | procIndex++){ |
263 | > | done = 0; |
264 | > | while (!done) { |
265 | > | |
266 | > | MPI_Recv(&myStatus, 1, MPI_INT, 0, |
267 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
268 | ||
269 | < | if( worldRank == 0 ){ |
269 | > | if(!myStatus) anonymousNodeDie(); |
270 | ||
271 | < | mpiErr = MPI_Recv(&nodeAtomsStart,1,MPI_INT,procIndex, |
272 | < | TAKE_THIS_TAG,MPI_COMM_WORLD,istatus); |
271 | > | if(myStatus < 0) break; |
272 | > | |
273 | > | MPI_Recv(&which_atom, 1, MPI_INT, 0, |
274 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
275 | ||
276 | < | mpiErr = MPI_Recv(&nodeAtomsEnd,1,MPI_INT,procIndex, |
277 | < | TAKE_THIS_TAG,MPI_COMM_WORLD, istatus); |
278 | < | |
279 | < | // Make sure where node 0 is writing to, matches where the |
201 | < | // receiving node expects it to be. |
202 | < | |
203 | < | if (masterIndex != nodeAtomsStart){ |
204 | < | sendError = 1; |
205 | < | mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,TAKE_THIS_TAG, |
206 | < | MPI_COMM_WORLD); |
207 | < | sprintf(painCave.errMsg, |
208 | < | "DumpWriter error: atoms start index (%d) for " |
209 | < | "node %d not equal to master index (%d)", |
210 | < | nodeAtomsStart,procIndex,masterIndex ); |
211 | < | painCave.isFatal = 1; |
212 | < | simError(); |
276 | > | myStatus = 1; |
277 | > | local_index=-1; |
278 | > | for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) { |
279 | > | if (atoms[j]->getGlobalIndex() == which_atom) local_index = j; |
280 | } | |
281 | < | |
282 | < | sendError = 0; |
216 | < | mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,TAKE_THIS_TAG, |
217 | < | MPI_COMM_WORLD); |
218 | < | |
219 | < | // recieve the nodes writeLines |
220 | < | |
221 | < | for ( i = nodeAtomsStart; i <= nodeAtomsEnd; i++){ |
222 | < | |
223 | < | mpiErr = MPI_Recv(writeLine,BUFFERSIZE,MPI_CHAR,procIndex, |
224 | < | TAKE_THIS_TAG,MPI_COMM_WORLD,istatus ); |
225 | < | |
226 | < | outFile << writeLine; |
227 | < | masterIndex++; |
228 | < | } |
229 | < | } |
281 | > | if (local_index != -1) { |
282 | > | //format the line |
283 | ||
284 | < | else if( worldRank == procIndex ){ |
284 | > | atoms[local_index]->getPos(pos); |
285 | > | atoms[local_index]->getVel(vel); |
286 | ||
233 | – | nodeAtomsStart = mpiSim->getMyAtomStart(); |
234 | – | nodeAtomsEnd = mpiSim->getMyAtomEnd(); |
235 | – | |
236 | – | mpiErr = MPI_Send(&nodeAtomsStart,1,MPI_INT,0,TAKE_THIS_TAG, |
237 | – | MPI_COMM_WORLD); |
238 | – | mpiErr = MPI_Send(&nodeAtomsEnd,1,MPI_INT,0,TAKE_THIS_TAG, |
239 | – | MPI_COMM_WORLD); |
240 | – | |
241 | – | sendError = -1; |
242 | – | mpiErr = MPI_Recv(&sendError,1,MPI_INT,0,TAKE_THIS_TAG, |
243 | – | MPI_COMM_WORLD, istatus); |
244 | – | |
245 | – | if (sendError) MPIcheckPoint(); |
246 | – | |
247 | – | // send current node's configuration line by line. |
248 | – | |
249 | – | for( i=0; i<nAtoms; i++ ){ |
250 | – | |
287 | sprintf( tempBuffer, | |
288 | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", | |
289 | < | atoms[i]->getType(), |
290 | < | atoms[i]->getX(), |
291 | < | atoms[i]->getY(), |
292 | < | atoms[i]->getZ(), |
293 | < | atoms[i]->get_vx(), |
294 | < | atoms[i]->get_vy(), |
295 | < | atoms[i]->get_vz()); // check here. |
289 | > | atoms[local_index]->getType(), |
290 | > | pos[0], |
291 | > | pos[1], |
292 | > | pos[2], |
293 | > | vel[0], |
294 | > | vel[1], |
295 | > | vel[2]); // check here. |
296 | strcpy( writeLine, tempBuffer ); | |
297 | + | |
298 | + | if( atoms[local_index]->isDirectional() ){ |
299 | ||
300 | < | if( atoms[i]->isDirectional() ){ |
263 | < | |
264 | < | dAtom = (DirectionalAtom *)atoms[i]; |
300 | > | dAtom = (DirectionalAtom *)atoms[local_index]; |
301 | dAtom->getQ( q ); | |
302 | < | |
302 | > | |
303 | sprintf( tempBuffer, | |
304 | "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", | |
305 | q[0], | |
# | Line 275 | Line 311 | void DumpWriter::writeDump( double currentTime ){ | |
311 | dAtom->getJz()); | |
312 | strcat( writeLine, tempBuffer ); | |
313 | } | |
314 | < | else |
314 | > | else{ |
315 | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); | |
316 | < | |
317 | < | mpiErr = MPI_Send(writeLine,BUFFERSIZE,MPI_CHAR,0,TAKE_THIS_TAG, |
318 | < | MPI_COMM_WORLD); |
316 | > | } |
317 | > | } |
318 | > | else { |
319 | > | sprintf(painCave.errMsg, |
320 | > | "Atom %d not found on processor %d\n", |
321 | > | which_atom, worldRank ); |
322 | > | myStatus = 0; |
323 | > | simError(); |
324 | > | |
325 | > | strcpy( writeLine, "Hello, I'm an error.\n"); |
326 | } | |
327 | + | |
328 | + | MPI_Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, |
329 | + | TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD); |
330 | + | MPI_Send( &myStatus, 1, MPI_INT, 0, |
331 | + | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
332 | } | |
333 | < | |
334 | < | sprintf(checkPointMsg,"Node %d sent dump configuration.", |
335 | < | procIndex); |
336 | < | MPIcheckPoint(); |
337 | < | } |
338 | < | |
333 | > | } |
334 | > | outFile.flush(); |
335 | > | sprintf( checkPointMsg, |
336 | > | "Sucessfully took a dump.\n"); |
337 | > | MPIcheckPoint(); |
338 | > | |
339 | > | // last thing last, enable fatalities. |
340 | > | painCave.isEventLoop = 0; |
341 | > | |
342 | #endif // is_mpi | |
343 | } | |
344 | ||
345 | + | void DumpWriter::writeFinal(double finalTime){ |
346 | ||
347 | + | char finalName[500]; |
348 | + | ofstream finalOut; |
349 | ||
296 | – | void DumpWriter::writeFinal(){ |
297 | – | |
298 | – | |
350 | const int BUFFERSIZE = 2000; | |
351 | < | char tempBuffer[500]; |
352 | < | char writeLine[BUFFERSIZE]; |
302 | < | |
303 | < | char finalName[500]; |
351 | > | char tempBuffer[BUFFERSIZE]; |
352 | > | char writeLine[BUFFERSIZE]; |
353 | ||
305 | – | int i; |
354 | double q[4]; | |
355 | DirectionalAtom* dAtom; | |
308 | – | int nAtoms = entry_plug->n_atoms; |
356 | Atom** atoms = entry_plug->atoms; | |
357 | + | int i; |
358 | + | #ifdef IS_MPI |
359 | + | int j, which_node, done, which_atom, local_index; |
360 | + | #else //is_mpi |
361 | + | int nAtoms = entry_plug->n_atoms; |
362 | + | #endif //is_mpi |
363 | ||
364 | < | ofstream finalOut; |
364 | > | double pos[3], vel[3]; |
365 | ||
366 | #ifdef IS_MPI | |
367 | if(worldRank == 0 ){ | |
# | Line 335 | Line 388 | void DumpWriter::writeFinal(){ | |
388 | ||
389 | #endif //is_mpi | |
390 | ||
391 | < | |
339 | < | |
391 | > | |
392 | #ifndef IS_MPI | |
393 | ||
394 | finalOut << nAtoms << "\n"; | |
395 | ||
396 | < | finalOut << entry_plug->box_x << "\t" |
397 | < | << entry_plug->box_y << "\t" |
398 | < | << entry_plug->box_z << "\n"; |
396 | > | finalOut << finalTime << ";\t" |
397 | > | << entry_plug->Hmat[0][0] << "\t" |
398 | > | << entry_plug->Hmat[1][0] << "\t" |
399 | > | << entry_plug->Hmat[2][0] << ";\t" |
400 | ||
401 | + | << entry_plug->Hmat[0][1] << "\t" |
402 | + | << entry_plug->Hmat[1][1] << "\t" |
403 | + | << entry_plug->Hmat[2][1] << ";\t" |
404 | + | |
405 | + | << entry_plug->Hmat[0][2] << "\t" |
406 | + | << entry_plug->Hmat[1][2] << "\t" |
407 | + | << entry_plug->Hmat[2][2] << ";\n"; |
408 | + | |
409 | for( i=0; i<nAtoms; i++ ){ | |
410 | ||
411 | + | atoms[i]->getPos(pos); |
412 | + | atoms[i]->getVel(vel); |
413 | + | |
414 | sprintf( tempBuffer, | |
415 | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", | |
416 | atoms[i]->getType(), | |
417 | < | atoms[i]->getX(), |
418 | < | atoms[i]->getY(), |
419 | < | atoms[i]->getZ(), |
420 | < | atoms[i]->get_vx(), |
421 | < | atoms[i]->get_vy(), |
422 | < | atoms[i]->get_vz()); |
417 | > | pos[0], |
418 | > | pos[1], |
419 | > | pos[2], |
420 | > | vel[0], |
421 | > | vel[1], |
422 | > | vel[2]); |
423 | strcpy( writeLine, tempBuffer ); | |
424 | ||
425 | if( atoms[i]->isDirectional() ){ | |
# | Line 380 | Line 444 | void DumpWriter::writeFinal(){ | |
444 | finalOut << writeLine; | |
445 | } | |
446 | finalOut.flush(); | |
447 | + | finalOut.close(); |
448 | ||
449 | #else // is_mpi | |
450 | + | |
451 | + | // first thing first, suspend fatalities. |
452 | + | painCave.isEventLoop = 1; |
453 | ||
454 | < | int masterIndex; |
455 | < | int nodeAtomsStart; |
388 | < | int nodeAtomsEnd; |
389 | < | int mpiErr; |
390 | < | int sendError; |
391 | < | int procIndex; |
392 | < | |
393 | < | MPI_Status istatus[MPI_STATUS_SIZE]; |
454 | > | int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone |
455 | > | int haveError; |
456 | ||
457 | < | |
458 | < | // write out header and node 0's coordinates |
457 | > | MPI_Status istatus; |
458 | > | int *AtomToProcMap = mpiSim->getAtomToProcMap(); |
459 | ||
460 | + | // write out header and node 0's coordinates |
461 | + | |
462 | + | haveError = 0; |
463 | if( worldRank == 0 ){ | |
464 | finalOut << mpiSim->getTotAtoms() << "\n"; | |
465 | + | |
466 | + | finalOut << finalTime << ";\t" |
467 | + | << entry_plug->Hmat[0][0] << "\t" |
468 | + | << entry_plug->Hmat[1][0] << "\t" |
469 | + | << entry_plug->Hmat[2][0] << ";\t" |
470 | ||
471 | < | finalOut << entry_plug->box_x << "\t" |
472 | < | << entry_plug->box_y << "\t" |
473 | < | << entry_plug->box_z << "\n"; |
471 | > | << entry_plug->Hmat[0][1] << "\t" |
472 | > | << entry_plug->Hmat[1][1] << "\t" |
473 | > | << entry_plug->Hmat[2][1] << ";\t" |
474 | > | |
475 | > | << entry_plug->Hmat[0][2] << "\t" |
476 | > | << entry_plug->Hmat[1][2] << "\t" |
477 | > | << entry_plug->Hmat[2][2] << ";\n"; |
478 | ||
479 | < | masterIndex = 0; |
480 | < | |
407 | < | for( i=0; i<nAtoms; i++ ){ |
479 | > | for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) { |
480 | > | // Get the Node number which has this molecule: |
481 | ||
482 | < | sprintf( tempBuffer, |
483 | < | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", |
484 | < | atoms[i]->getType(), |
485 | < | atoms[i]->getX(), |
486 | < | atoms[i]->getY(), |
487 | < | atoms[i]->getZ(), |
488 | < | atoms[i]->get_vx(), |
489 | < | atoms[i]->get_vy(), |
490 | < | atoms[i]->get_vz()); |
491 | < | strcpy( writeLine, tempBuffer ); |
492 | < | |
493 | < | if( atoms[i]->isDirectional() ){ |
482 | > | which_node = AtomToProcMap[i]; |
483 | > | |
484 | > | if (which_node == mpiSim->getMyNode()) { |
485 | > | |
486 | > | which_atom = i; |
487 | > | local_index=-1; |
488 | > | for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) { |
489 | > | if (atoms[j]->getGlobalIndex() == which_atom) local_index = j; |
490 | > | } |
491 | > | if (local_index != -1) { |
492 | > | |
493 | > | atoms[local_index]->getPos(pos); |
494 | > | atoms[local_index]->getVel(vel); |
495 | ||
496 | < | dAtom = (DirectionalAtom *)atoms[i]; |
497 | < | dAtom->getQ( q ); |
496 | > | sprintf( tempBuffer, |
497 | > | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", |
498 | > | atoms[local_index]->getType(), |
499 | > | pos[0], |
500 | > | pos[1], |
501 | > | pos[2], |
502 | > | vel[0], |
503 | > | vel[1], |
504 | > | vel[2]); |
505 | > | strcpy( writeLine, tempBuffer ); |
506 | ||
507 | < | sprintf( tempBuffer, |
508 | < | "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", |
509 | < | q[0], |
510 | < | q[1], |
511 | < | q[2], |
512 | < | q[3], |
513 | < | dAtom->getJx(), |
514 | < | dAtom->getJy(), |
515 | < | dAtom->getJz()); |
516 | < | strcat( writeLine, tempBuffer ); |
517 | < | } |
518 | < | else |
519 | < | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); |
507 | > | if( atoms[local_index]->isDirectional() ){ |
508 | > | |
509 | > | dAtom = (DirectionalAtom *)atoms[local_index]; |
510 | > | dAtom->getQ( q ); |
511 | > | |
512 | > | sprintf( tempBuffer, |
513 | > | "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", |
514 | > | q[0], |
515 | > | q[1], |
516 | > | q[2], |
517 | > | q[3], |
518 | > | dAtom->getJx(), |
519 | > | dAtom->getJy(), |
520 | > | dAtom->getJz()); |
521 | > | strcat( writeLine, tempBuffer ); |
522 | > | } |
523 | > | else |
524 | > | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); |
525 | > | } |
526 | > | else { |
527 | > | sprintf(painCave.errMsg, |
528 | > | "Atom %d not found on processor %d\n", |
529 | > | i, worldRank ); |
530 | > | haveError= 1; |
531 | > | simError(); |
532 | > | } |
533 | > | |
534 | > | if(haveError) nodeZeroError(); |
535 | > | |
536 | > | } |
537 | > | else { |
538 | > | |
539 | > | myStatus = 1; |
540 | > | MPI_Send(&myStatus, 1, MPI_INT, which_node, |
541 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
542 | > | MPI_Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG_INT, |
543 | > | MPI_COMM_WORLD); |
544 | > | MPI_Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node, |
545 | > | TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD, &istatus); |
546 | > | MPI_Recv(&myStatus, 1, MPI_INT, which_node, |
547 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
548 | ||
549 | + | if(!myStatus) nodeZeroError(); |
550 | + | } |
551 | + | |
552 | finalOut << writeLine; | |
440 | – | masterIndex++; |
553 | } | |
442 | – | finalOut.flush(); |
443 | – | } |
554 | ||
555 | < | for (procIndex = 1; procIndex < mpiSim->getNumberProcessors(); |
556 | < | procIndex++){ |
555 | > | // kill everyone off: |
556 | > | myStatus = -1; |
557 | > | for (j = 0; j < mpiSim->getNumberProcessors(); j++) { |
558 | > | MPI_Send(&myStatus, 1, MPI_INT, j, |
559 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
560 | > | } |
561 | ||
562 | < | if( worldRank == 0 ){ |
563 | < | |
564 | < | mpiErr = MPI_Recv(&nodeAtomsStart,1,MPI_INT,procIndex, |
565 | < | TAKE_THIS_TAG,MPI_COMM_WORLD,istatus); |
562 | > | } else { |
563 | > | |
564 | > | done = 0; |
565 | > | while (!done) { |
566 | ||
567 | < | mpiErr = MPI_Recv(&nodeAtomsEnd,1,MPI_INT,procIndex, |
568 | < | TAKE_THIS_TAG,MPI_COMM_WORLD, istatus); |
569 | < | |
570 | < | // Make sure where node 0 is writing to, matches where the |
571 | < | // receiving node expects it to be. |
572 | < | |
573 | < | if (masterIndex != nodeAtomsStart){ |
574 | < | sendError = 1; |
575 | < | mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,TAKE_THIS_TAG, |
576 | < | MPI_COMM_WORLD); |
577 | < | sprintf(painCave.errMsg, |
578 | < | "DumpWriter error: atoms start index (%d) for " |
579 | < | "node %d not equal to master index (%d)", |
580 | < | nodeAtomsStart,procIndex,masterIndex ); |
467 | < | painCave.isFatal = 1; |
468 | < | simError(); |
567 | > | MPI_Recv(&myStatus, 1, MPI_INT, 0, |
568 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
569 | > | |
570 | > | if(!myStatus) anonymousNodeDie(); |
571 | > | |
572 | > | if(myStatus < 0) break; |
573 | > | |
574 | > | MPI_Recv(&which_atom, 1, MPI_INT, 0, |
575 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
576 | > | |
577 | > | myStatus = 1; |
578 | > | local_index=-1; |
579 | > | for (j=0; j < mpiSim->getMyNlocal(); j++) { |
580 | > | if (atoms[j]->getGlobalIndex() == which_atom) local_index = j; |
581 | } | |
582 | < | |
471 | < | sendError = 0; |
472 | < | mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,TAKE_THIS_TAG, |
473 | < | MPI_COMM_WORLD); |
582 | > | if (local_index != -1) { |
583 | ||
584 | < | // recieve the nodes writeLines |
584 | > | atoms[local_index]->getPos(pos); |
585 | > | atoms[local_index]->getVel(vel); |
586 | ||
587 | < | for ( i = nodeAtomsStart; i <= nodeAtomsEnd; i++){ |
478 | < | |
479 | < | mpiErr = MPI_Recv(writeLine,BUFFERSIZE,MPI_CHAR,procIndex, |
480 | < | TAKE_THIS_TAG,MPI_COMM_WORLD,istatus ); |
481 | < | |
482 | < | finalOut << writeLine; |
483 | < | masterIndex++; |
484 | < | } |
485 | < | |
486 | < | finalOut.flush(); |
487 | < | } |
488 | < | |
489 | < | else if( worldRank == procIndex ){ |
490 | < | |
491 | < | nodeAtomsStart = mpiSim->getMyAtomStart(); |
492 | < | nodeAtomsEnd = mpiSim->getMyAtomEnd(); |
493 | < | |
494 | < | mpiErr = MPI_Send(&nodeAtomsStart,1,MPI_INT,0,TAKE_THIS_TAG, |
495 | < | MPI_COMM_WORLD); |
496 | < | mpiErr = MPI_Send(&nodeAtomsEnd,1,MPI_INT,0,TAKE_THIS_TAG, |
497 | < | MPI_COMM_WORLD); |
498 | < | |
499 | < | mpiErr = MPI_Recv(&sendError,1,MPI_INT,0,TAKE_THIS_TAG, |
500 | < | MPI_COMM_WORLD, istatus); |
501 | < | if (sendError) MPIcheckPoint(); |
502 | < | |
503 | < | // send current node's configuration line by line. |
504 | < | |
505 | < | for( i=0; i<nAtoms; i++ ){ |
506 | < | |
587 | > | //format the line |
588 | sprintf( tempBuffer, | |
589 | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", | |
590 | < | atoms[i]->getType(), |
591 | < | atoms[i]->getX(), |
592 | < | atoms[i]->getY(), |
593 | < | atoms[i]->getZ(), |
594 | < | atoms[i]->get_vx(), |
595 | < | atoms[i]->get_vy(), |
596 | < | atoms[i]->get_vz()); |
590 | > | atoms[local_index]->getType(), |
591 | > | pos[0], |
592 | > | pos[1], |
593 | > | pos[2], |
594 | > | vel[0], |
595 | > | vel[1], |
596 | > | vel[2]); // check here. |
597 | strcpy( writeLine, tempBuffer ); | |
598 | + | |
599 | + | if( atoms[local_index]->isDirectional() ){ |
600 | ||
601 | < | if( atoms[i]->isDirectional() ){ |
519 | < | |
520 | < | dAtom = (DirectionalAtom *)atoms[i]; |
601 | > | dAtom = (DirectionalAtom *)atoms[local_index]; |
602 | dAtom->getQ( q ); | |
603 | < | |
603 | > | |
604 | sprintf( tempBuffer, | |
605 | "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", | |
606 | q[0], | |
# | Line 531 | Line 612 | void DumpWriter::writeFinal(){ | |
612 | dAtom->getJz()); | |
613 | strcat( writeLine, tempBuffer ); | |
614 | } | |
615 | < | else |
615 | > | else{ |
616 | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); | |
617 | < | |
618 | < | mpiErr = MPI_Send(writeLine,BUFFERSIZE,MPI_CHAR,0,TAKE_THIS_TAG, |
619 | < | MPI_COMM_WORLD); |
617 | > | } |
618 | > | } |
619 | > | else { |
620 | > | sprintf(painCave.errMsg, |
621 | > | "Atom %d not found on processor %d\n", |
622 | > | which_atom, worldRank ); |
623 | > | myStatus = 0; |
624 | > | simError(); |
625 | > | |
626 | > | strcpy( writeLine, "Hello, I'm an error.\n"); |
627 | } | |
628 | + | |
629 | + | MPI_Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, |
630 | + | TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD); |
631 | + | MPI_Send( &myStatus, 1, MPI_INT, 0, |
632 | + | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
633 | } | |
541 | – | |
542 | – | sprintf(checkPointMsg,"Node %d sent dump configuration.", |
543 | – | procIndex); |
544 | – | MPIcheckPoint(); |
634 | } | |
635 | + | finalOut.flush(); |
636 | + | sprintf( checkPointMsg, |
637 | + | "Sucessfully took a dump.\n"); |
638 | + | MPIcheckPoint(); |
639 | + | |
640 | + | if( worldRank == 0 ) finalOut.close(); |
641 | + | #endif // is_mpi |
642 | + | } |
643 | ||
547 | – | if( worldRank == 0 ) finalOut.close(); |
644 | ||
645 | < | |
646 | < | #endif // is_mpi |
645 | > | |
646 | > | #ifdef IS_MPI |
647 | > | |
648 | > | // a couple of functions to let us escape the write loop |
649 | > | |
650 | > | void dWrite::nodeZeroError( void ){ |
651 | > | int j, myStatus; |
652 | > | |
653 | > | myStatus = 0; |
654 | > | for (j = 0; j < mpiSim->getNumberProcessors(); j++) { |
655 | > | MPI_Send( &myStatus, 1, MPI_INT, j, |
656 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
657 | > | } |
658 | > | |
659 | > | |
660 | > | MPI_Finalize(); |
661 | > | exit (0); |
662 | > | |
663 | } | |
664 | + | |
665 | + | void dWrite::anonymousNodeDie( void ){ |
666 | + | |
667 | + | MPI_Finalize(); |
668 | + | exit (0); |
669 | + | } |
670 | + | |
671 | + | #endif //is_mpi |
– | Removed lines |
+ | Added lines |
< | Changed lines |
> | Changed lines |