# | Line 4 | Line 4 | |
---|---|---|
4 | ||
5 | #ifdef IS_MPI | |
6 | #include <mpi.h> | |
7 | + | #include <mpi++.h> |
8 | #include "mpiSimulation.hpp" | |
9 | < | #define TAKE_THIS_TAG 0 |
9 | > | #define TAKE_THIS_TAG_CHAR 1 |
10 | > | #define TAKE_THIS_TAG_INT 2 |
11 | #endif //is_mpi | |
12 | ||
13 | #include "ReadWrite.hpp" | |
14 | #include "simError.h" | |
15 | ||
14 | – | #define GAME_OVER -1 |
15 | – | |
16 | DumpWriter::DumpWriter( SimInfo* the_entry_plug ){ | |
17 | ||
18 | entry_plug = the_entry_plug; | |
# | Line 66 | Line 66 | void DumpWriter::writeDump( double currentTime ){ | |
66 | char tempBuffer[BUFFERSIZE]; | |
67 | char writeLine[BUFFERSIZE]; | |
68 | ||
69 | < | int i; |
69 | > | int i, j, which_node, done, game_over, which_atom, local_index; |
70 | double q[4]; | |
71 | DirectionalAtom* dAtom; | |
72 | int nAtoms = entry_plug->n_atoms; | |
# | Line 120 | Line 120 | void DumpWriter::writeDump( double currentTime ){ | |
120 | outFile.flush(); | |
121 | ||
122 | #else // is_mpi | |
123 | + | |
124 | + | MPI::Status istatus; |
125 | + | int *AtomToProcMap = mpiSim->getAtomToProcMap(); |
126 | ||
124 | – | MPI_Status istatus[MPI_STATUS_SIZE]; |
125 | – | |
127 | // write out header and node 0's coordinates | |
128 | ||
129 | if( worldRank == 0 ){ | |
# | Line 132 | Line 133 | void DumpWriter::writeDump( double currentTime ){ | |
133 | << entry_plug->box_x << "\t" | |
134 | << entry_plug->box_y << "\t" | |
135 | << entry_plug->box_z << "\n"; | |
136 | < | |
137 | < | for (i = 0 ; i < mpiPlug->nAtomsGlobal; i++ ) { |
138 | < | // Get the Node number which has this molecule: |
136 | > | outFile.flush(); |
137 | > | for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) { |
138 | > | // Get the Node number which has this atom; |
139 | ||
140 | which_node = AtomToProcMap[i]; | |
141 | ||
142 | < | if (which_node == mpiPlug->myNode) { |
142 | < | |
143 | < | sprintf( tempBuffer, |
144 | < | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", |
145 | < | atoms[i]->getType(), |
146 | < | atoms[i]->getX(), |
147 | < | atoms[i]->getY(), |
148 | < | atoms[i]->getZ(), |
149 | < | atoms[i]->get_vx(), |
150 | < | atoms[i]->get_vy(), |
151 | < | atoms[i]->get_vz()); |
152 | < | strcpy( writeLine, tempBuffer ); |
142 | > | if (which_node == 0 ) { |
143 | ||
144 | < | if( atoms[i]->isDirectional() ){ |
145 | < | |
146 | < | dAtom = (DirectionalAtom *)atoms[i]; |
147 | < | dAtom->getQ( q ); |
158 | < | |
159 | < | sprintf( tempBuffer, |
160 | < | "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", |
161 | < | q[0], |
162 | < | q[1], |
163 | < | q[2], |
164 | < | q[3], |
165 | < | dAtom->getJx(), |
166 | < | dAtom->getJy(), |
167 | < | dAtom->getJz()); |
168 | < | strcat( writeLine, tempBuffer ); |
144 | > | which_atom = i; |
145 | > | local_index=-1; |
146 | > | for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) { |
147 | > | if (atoms[j]->getGlobalIndex() == which_atom) local_index = j; |
148 | } | |
149 | < | else |
150 | < | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); |
149 | > | if (local_index != -1) { |
150 | > | //format the line |
151 | > | sprintf( tempBuffer, |
152 | > | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", |
153 | > | atoms[local_index]->getType(), |
154 | > | atoms[local_index]->getX(), |
155 | > | atoms[local_index]->getY(), |
156 | > | atoms[local_index]->getZ(), |
157 | > | atoms[local_index]->get_vx(), |
158 | > | atoms[local_index]->get_vy(), |
159 | > | atoms[local_index]->get_vz()); // check here. |
160 | > | strcpy( writeLine, tempBuffer ); |
161 | > | |
162 | > | if( atoms[local_index]->isDirectional() ){ |
163 | > | |
164 | > | dAtom = (DirectionalAtom *)atoms[local_index]; |
165 | > | dAtom->getQ( q ); |
166 | > | |
167 | > | sprintf( tempBuffer, |
168 | > | "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", |
169 | > | q[0], |
170 | > | q[1], |
171 | > | q[2], |
172 | > | q[3], |
173 | > | dAtom->getJx(), |
174 | > | dAtom->getJy(), |
175 | > | dAtom->getJz()); |
176 | > | strcat( writeLine, tempBuffer ); |
177 | > | |
178 | > | } |
179 | > | else |
180 | > | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); |
181 | > | } |
182 | > | else { |
183 | > | strcpy( writeLine, "ATOM NOT FOUND ON THIS PROCESSOR"); |
184 | > | } |
185 | > | } |
186 | > | else { |
187 | ||
188 | < | } else { |
189 | < | |
190 | < | MPI::COMM_WORLD.Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG); |
191 | < | MPI::COMM_WORLD.Receive(writeLine, BUFFERSIZE, MPI_CHAR, which_node, |
192 | < | TAKE_THIS_TAG, istatus); |
188 | > | //std::cerr << "node 0: sending node " << which_node << " request for atom " << i << "\n"; |
189 | > | MPI::COMM_WORLD.Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG_INT); |
190 | > | //std::cerr << "node 0: sent!\n"; |
191 | > | MPI::COMM_WORLD.Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node, |
192 | > | TAKE_THIS_TAG_CHAR, istatus); |
193 | > | //std::cerr << "node 0: got this line: " << writeLine; |
194 | } | |
195 | ||
196 | outFile << writeLine; | |
197 | + | outFile.flush(); |
198 | } | |
199 | ||
200 | // kill everyone off: | |
201 | < | for (j = 0; j < mpiPlug->numberProcessors; j++) { |
202 | < | MPI::COMM_WORLD.Send(GAME_OVER, 1, MPI_INT, j, TAKE_THIS_TAG); |
201 | > | game_over = -1; |
202 | > | for (j = 0; j < mpiSim->getNumberProcessors(); j++) { |
203 | > | MPI::COMM_WORLD.Send(&game_over, 1, MPI_INT, j, TAKE_THIS_TAG_INT); |
204 | } | |
205 | ||
206 | } else { | |
207 | ||
208 | done = 0; | |
209 | while (!done) { | |
210 | < | MPI::COMM_WORLD.Receive(&which_atom, 1, MPI_INT, 0, |
211 | < | TAKE_THIS_TAG, istatus); |
212 | < | |
213 | < | if (which_atom == GAME_OVER) { |
210 | > | //std::cerr << "node: " << mpiSim->getMyNode() << " Waiting for receive \n"; |
211 | > | MPI::COMM_WORLD.Recv(&which_atom, 1, MPI_INT, 0, |
212 | > | TAKE_THIS_TAG_INT, istatus); |
213 | > | //std::cerr << "node: " << mpiSim->getMyNode() << " got request for atom " << which_atom << "\n"; |
214 | > | if (which_atom == -1) { |
215 | done=1; | |
216 | continue; | |
217 | } else { | |
218 | < | |
219 | < | //format the line |
220 | < | sprintf( tempBuffer, |
221 | < | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", |
222 | < | atoms[which_atom]->getType(), |
223 | < | atoms[which_atom]->getX(), |
224 | < | atoms[which_atom]->getY(), |
225 | < | atoms[which_atom]->getZ(), |
226 | < | atoms[which_atom]->get_vx(), |
227 | < | atoms[which_atom]->get_vy(), |
228 | < | atoms[which_atom]->get_vz()); // check here. |
229 | < | strcpy( writeLine, tempBuffer ); |
218 | > | local_index=-1; |
219 | > | for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) { |
220 | > | if (atoms[j]->getGlobalIndex() == which_atom) local_index = j; |
221 | > | } |
222 | > | if (local_index != -1) { |
223 | > | //format the line |
224 | > | sprintf( tempBuffer, |
225 | > | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", |
226 | > | atoms[local_index]->getType(), |
227 | > | atoms[local_index]->getX(), |
228 | > | atoms[local_index]->getY(), |
229 | > | atoms[local_index]->getZ(), |
230 | > | atoms[local_index]->get_vx(), |
231 | > | atoms[local_index]->get_vy(), |
232 | > | atoms[local_index]->get_vz()); // check here. |
233 | > | strcpy( writeLine, tempBuffer ); |
234 | ||
235 | < | if( atoms[which_atom]->isDirectional() ){ |
236 | < | |
237 | < | dAtom = (DirectionalAtom *)atoms[which_atom]; |
238 | < | dAtom->getQ( q ); |
239 | < | |
240 | < | sprintf( tempBuffer, |
241 | < | "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", |
242 | < | q[0], |
243 | < | q[1], |
244 | < | q[2], |
245 | < | q[3], |
246 | < | dAtom->getJx(), |
247 | < | dAtom->getJy(), |
248 | < | dAtom->getJz()); |
249 | < | strcat( writeLine, tempBuffer ); |
250 | < | } |
251 | < | else |
252 | < | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); |
253 | < | |
254 | < | MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, |
255 | < | TAKE_THIS_TAG); |
235 | > | if( atoms[local_index]->isDirectional() ){ |
236 | > | |
237 | > | dAtom = (DirectionalAtom *)atoms[local_index]; |
238 | > | dAtom->getQ( q ); |
239 | > | |
240 | > | sprintf( tempBuffer, |
241 | > | "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", |
242 | > | q[0], |
243 | > | q[1], |
244 | > | q[2], |
245 | > | q[3], |
246 | > | dAtom->getJx(), |
247 | > | dAtom->getJy(), |
248 | > | dAtom->getJz()); |
249 | > | strcat( writeLine, tempBuffer ); |
250 | > | } |
251 | > | else |
252 | > | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); |
253 | > | // std::cerr << "node: " << mpiSim->getMyNode() << " sending this line" << writeLine; |
254 | > | MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, |
255 | > | TAKE_THIS_TAG_CHAR); |
256 | > | } else { |
257 | > | strcpy( writeLine, "ATOM NOT FOUND ON THIS PROCESSOR"); |
258 | > | MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, |
259 | > | TAKE_THIS_TAG_CHAR); |
260 | > | } |
261 | } | |
262 | } | |
263 | } | |
# | Line 241 | Line 269 | void DumpWriter::writeFinal(){ | |
269 | } | |
270 | ||
271 | void DumpWriter::writeFinal(){ | |
272 | < | |
272 | > | |
273 | char finalName[500]; | |
274 | ofstream finalOut; | |
275 | + | |
276 | + | const int BUFFERSIZE = 2000; |
277 | + | char tempBuffer[BUFFERSIZE]; |
278 | + | char writeLine[BUFFERSIZE]; |
279 | + | |
280 | + | double q[4]; |
281 | + | DirectionalAtom* dAtom; |
282 | + | int nAtoms = entry_plug->n_atoms; |
283 | + | Atom** atoms = entry_plug->atoms; |
284 | + | int i, j, which_node, done, game_over, which_atom, local_index; |
285 | ||
286 | + | |
287 | #ifdef IS_MPI | |
288 | if(worldRank == 0 ){ | |
289 | #endif // is_mpi | |
# | Line 278 | Line 317 | void DumpWriter::writeFinal(){ | |
317 | finalOut << entry_plug->box_x << "\t" | |
318 | << entry_plug->box_y << "\t" | |
319 | << entry_plug->box_z << "\n"; | |
320 | < | |
320 | > | |
321 | for( i=0; i<nAtoms; i++ ){ | |
322 | ||
323 | sprintf( tempBuffer, | |
# | Line 318 | Line 357 | void DumpWriter::writeFinal(){ | |
357 | ||
358 | #else // is_mpi | |
359 | ||
360 | < | MPI_Status istatus[MPI_STATUS_SIZE]; |
361 | < | |
360 | > | MPI::Status istatus; |
361 | > | int *AtomToProcMap = mpiSim->getAtomToProcMap(); |
362 | > | |
363 | // write out header and node 0's coordinates | |
364 | ||
365 | if( worldRank == 0 ){ | |
# | Line 329 | Line 369 | void DumpWriter::writeFinal(){ | |
369 | << entry_plug->box_y << "\t" | |
370 | << entry_plug->box_z << "\n"; | |
371 | ||
372 | < | for (i = 0 ; i < mpiPlug->nAtomsGlobal; i++ ) { |
372 | > | for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) { |
373 | // Get the Node number which has this molecule: | |
374 | ||
375 | which_node = AtomToProcMap[i]; | |
376 | ||
377 | < | if (which_node == mpiPlug->myNode) { |
378 | < | |
379 | < | sprintf( tempBuffer, |
380 | < | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", |
381 | < | atoms[i]->getType(), |
382 | < | atoms[i]->getX(), |
343 | < | atoms[i]->getY(), |
344 | < | atoms[i]->getZ(), |
345 | < | atoms[i]->get_vx(), |
346 | < | atoms[i]->get_vy(), |
347 | < | atoms[i]->get_vz()); |
348 | < | strcpy( writeLine, tempBuffer ); |
349 | < | |
350 | < | if( atoms[i]->isDirectional() ){ |
351 | < | |
352 | < | dAtom = (DirectionalAtom *)atoms[i]; |
353 | < | dAtom->getQ( q ); |
354 | < | |
355 | < | sprintf( tempBuffer, |
356 | < | "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", |
357 | < | q[0], |
358 | < | q[1], |
359 | < | q[2], |
360 | < | q[3], |
361 | < | dAtom->getJx(), |
362 | < | dAtom->getJy(), |
363 | < | dAtom->getJz()); |
364 | < | strcat( writeLine, tempBuffer ); |
377 | > | if (which_node == mpiSim->getMyNode()) { |
378 | > | |
379 | > | which_atom = i; |
380 | > | local_index=-1; |
381 | > | for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) { |
382 | > | if (atoms[j]->getGlobalIndex() == which_atom) local_index = j; |
383 | } | |
384 | < | else |
385 | < | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); |
386 | < | |
384 | > | if (local_index != -1) { |
385 | > | sprintf( tempBuffer, |
386 | > | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", |
387 | > | atoms[local_index]->getType(), |
388 | > | atoms[local_index]->getX(), |
389 | > | atoms[local_index]->getY(), |
390 | > | atoms[local_index]->getZ(), |
391 | > | atoms[local_index]->get_vx(), |
392 | > | atoms[local_index]->get_vy(), |
393 | > | atoms[local_index]->get_vz()); |
394 | > | strcpy( writeLine, tempBuffer ); |
395 | > | |
396 | > | if( atoms[local_index]->isDirectional() ){ |
397 | > | |
398 | > | dAtom = (DirectionalAtom *)atoms[local_index]; |
399 | > | dAtom->getQ( q ); |
400 | > | |
401 | > | sprintf( tempBuffer, |
402 | > | "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", |
403 | > | q[0], |
404 | > | q[1], |
405 | > | q[2], |
406 | > | q[3], |
407 | > | dAtom->getJx(), |
408 | > | dAtom->getJy(), |
409 | > | dAtom->getJz()); |
410 | > | strcat( writeLine, tempBuffer ); |
411 | > | } |
412 | > | else |
413 | > | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); |
414 | > | } |
415 | > | else { |
416 | > | strcpy( writeLine, "ATOM NOT FOUND ON THIS PROCESSOR"); |
417 | > | } |
418 | > | |
419 | } else { | |
420 | ||
421 | < | MPI::COMM_WORLD.Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG); |
422 | < | MPI::COMM_WORLD.Receive(writeLine, BUFFERSIZE, MPI_CHAR, which_node, |
423 | < | TAKE_THIS_TAG, istatus); |
421 | > | MPI::COMM_WORLD.Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG_INT); |
422 | > | MPI::COMM_WORLD.Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node, |
423 | > | TAKE_THIS_TAG_CHAR, istatus); |
424 | } | |
425 | ||
426 | finalOut << writeLine; | |
427 | } | |
428 | ||
429 | // kill everyone off: | |
430 | < | for (j = 0; j < mpiPlug->numberProcessors; j++) { |
431 | < | MPI::COMM_WORLD.Send(GAME_OVER, 1, MPI_INT, j, TAKE_THIS_TAG); |
430 | > | game_over = -1; |
431 | > | for (j = 0; j < mpiSim->getNumberProcessors(); j++) { |
432 | > | MPI::COMM_WORLD.Send(&game_over, 1, MPI_INT, j, TAKE_THIS_TAG_INT); |
433 | } | |
434 | ||
435 | } else { | |
436 | ||
437 | done = 0; | |
438 | while (!done) { | |
439 | < | MPI::COMM_WORLD.Receive(&which_atom, 1, MPI_INT, 0, |
440 | < | TAKE_THIS_TAG, istatus); |
439 | > | MPI::COMM_WORLD.Recv(&which_atom, 1, MPI_INT, 0, |
440 | > | TAKE_THIS_TAG_INT, istatus); |
441 | ||
442 | < | if (which_atom == GAME_OVER) { |
442 | > | if (which_atom == -1) { |
443 | done=1; | |
444 | continue; | |
445 | } else { | |
446 | ||
447 | < | //format the line |
448 | < | sprintf( tempBuffer, |
449 | < | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", |
450 | < | atoms[which_atom]->getType(), |
451 | < | atoms[which_atom]->getX(), |
452 | < | atoms[which_atom]->getY(), |
453 | < | atoms[which_atom]->getZ(), |
454 | < | atoms[which_atom]->get_vx(), |
455 | < | atoms[which_atom]->get_vy(), |
456 | < | atoms[which_atom]->get_vz()); // check here. |
457 | < | strcpy( writeLine, tempBuffer ); |
447 | > | local_index=-1; |
448 | > | for (j=0; j < mpiSim->getMyNlocal(); j++) { |
449 | > | if (atoms[j]->getGlobalIndex() == which_atom) local_index = j; |
450 | > | } |
451 | > | if (local_index != -1) { |
452 | > | |
453 | > | //format the line |
454 | > | sprintf( tempBuffer, |
455 | > | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", |
456 | > | atoms[local_index]->getType(), |
457 | > | atoms[local_index]->getX(), |
458 | > | atoms[local_index]->getY(), |
459 | > | atoms[local_index]->getZ(), |
460 | > | atoms[local_index]->get_vx(), |
461 | > | atoms[local_index]->get_vy(), |
462 | > | atoms[local_index]->get_vz()); // check here. |
463 | > | strcpy( writeLine, tempBuffer ); |
464 | ||
465 | < | if( atoms[which_atom]->isDirectional() ){ |
465 | > | if( atoms[local_index]->isDirectional() ){ |
466 | > | |
467 | > | dAtom = (DirectionalAtom *)atoms[local_index]; |
468 | > | dAtom->getQ( q ); |
469 | ||
470 | < | dAtom = (DirectionalAtom *)atoms[which_atom]; |
471 | < | dAtom->getQ( q ); |
472 | < | |
473 | < | sprintf( tempBuffer, |
474 | < | "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", |
475 | < | q[0], |
476 | < | q[1], |
477 | < | q[2], |
478 | < | q[3], |
479 | < | dAtom->getJx(), |
480 | < | dAtom->getJy(), |
481 | < | dAtom->getJz()); |
482 | < | strcat( writeLine, tempBuffer ); |
483 | < | } |
484 | < | else |
485 | < | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); |
486 | < | |
487 | < | MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, |
488 | < | TAKE_THIS_TAG); |
470 | > | sprintf( tempBuffer, |
471 | > | "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", |
472 | > | q[0], |
473 | > | q[1], |
474 | > | q[2], |
475 | > | q[3], |
476 | > | dAtom->getJx(), |
477 | > | dAtom->getJy(), |
478 | > | dAtom->getJz()); |
479 | > | strcat( writeLine, tempBuffer ); |
480 | > | } |
481 | > | else |
482 | > | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); |
483 | > | |
484 | > | MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, |
485 | > | TAKE_THIS_TAG_CHAR); |
486 | > | } else { |
487 | > | strcpy( writeLine, "ATOM NOT FOUND ON THIS PROCESSOR"); |
488 | > | MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, |
489 | > | TAKE_THIS_TAG_CHAR); |
490 | > | } |
491 | } | |
492 | } | |
493 | < | } |
493 | > | } |
494 | finalOut.flush(); | |
495 | sprintf( checkPointMsg, | |
496 | "Sucessfully took a dump.\n"); |
– | Removed lines |
+ | Added lines |
< | Changed lines |
> | Changed lines |