ViewVC Help
View File | Revision Log | Show Annotations | View Changeset | Root Listing
root/group/trunk/OOPSE-2.0/src/io/DumpWriter.cpp
(Generate patch)

Comparing trunk/OOPSE-2.0/src/io/DumpWriter.cpp (file contents):
Revision 1490 by gezelter, Fri Sep 24 04:16:43 2004 UTC vs.
Revision 2204 by gezelter, Fri Apr 15 22:04:00 2005 UTC

# Line 1 | Line 1
1 < #define _LARGEFILE_SOURCE64
2 < #define _FILE_OFFSET_BITS 64
3 <
4 < #include <string.h>
5 < #include <iostream>
6 < #include <fstream>
7 < #include <algorithm>
8 < #include <utility>
9 <
1 > /*
2 > * Copyright (c) 2005 The University of Notre Dame. All Rights Reserved.
3 > *
4 > * The University of Notre Dame grants you ("Licensee") a
5 > * non-exclusive, royalty free, license to use, modify and
6 > * redistribute this software in source and binary code form, provided
7 > * that the following conditions are met:
8 > *
9 > * 1. Acknowledgement of the program authors must be made in any
10 > *    publication of scientific results based in part on use of the
11 > *    program.  An acceptable form of acknowledgement is citation of
12 > *    the article in which the program was described (Matthew
13 > *    A. Meineke, Charles F. Vardeman II, Teng Lin, Christopher
14 > *    J. Fennell and J. Daniel Gezelter, "OOPSE: An Object-Oriented
15 > *    Parallel Simulation Engine for Molecular Dynamics,"
16 > *    J. Comput. Chem. 26, pp. 252-271 (2005))
17 > *
18 > * 2. Redistributions of source code must retain the above copyright
19 > *    notice, this list of conditions and the following disclaimer.
20 > *
21 > * 3. Redistributions in binary form must reproduce the above copyright
22 > *    notice, this list of conditions and the following disclaimer in the
23 > *    documentation and/or other materials provided with the
24 > *    distribution.
25 > *
26 > * This software is provided "AS IS," without a warranty of any
27 > * kind. All express or implied conditions, representations and
28 > * warranties, including any implied warranty of merchantability,
29 > * fitness for a particular purpose or non-infringement, are hereby
30 > * excluded.  The University of Notre Dame and its licensors shall not
31 > * be liable for any damages suffered by licensee as a result of
32 > * using, modifying or distributing the software or its
33 > * derivatives. In no event will the University of Notre Dame or its
34 > * licensors be liable for any lost revenue, profit or data, or for
35 > * direct, indirect, special, consequential, incidental or punitive
36 > * damages, however caused and regardless of the theory of liability,
37 > * arising out of the use of or inability to use software, even if the
38 > * University of Notre Dame has been advised of the possibility of
39 > * such damages.
40 > */
41 >
42 > #include "io/DumpWriter.hpp"
43 > #include "primitives/Molecule.hpp"
44 > #include "utils/simError.h"
45 > #include "io/basic_teebuf.hpp"
46   #ifdef IS_MPI
47   #include <mpi.h>
48 < #include "mpiSimulation.hpp"
48 > #endif //is_mpi
49  
50 < namespace dWrite{
15 <  void DieDieDie( void );
16 < }
50 > namespace oopse {
51  
52 < using namespace dWrite;
53 < #endif //is_mpi
52 >  DumpWriter::DumpWriter(SimInfo* info)
53 >    : info_(info), filename_(info->getDumpFileName()), eorFilename_(info->getFinalConfigFileName()){
54 > #ifdef IS_MPI
55  
56 < #include "ReadWrite.hpp"
57 < #include "simError.h"
56 >      if (worldRank == 0) {
57 > #endif // is_mpi
58  
59 < DumpWriter::DumpWriter( SimInfo* the_entry_plug ){
59 >        dumpFile_.open(filename_.c_str(), std::ios::out | std::ios::trunc);
60  
61 <  entry_plug = the_entry_plug;
61 >        if (!dumpFile_) {
62 >          sprintf(painCave.errMsg, "Could not open \"%s\" for dump output.\n",
63 >                  filename_.c_str());
64 >          painCave.isFatal = 1;
65 >          simError();
66 >        }
67  
68   #ifdef IS_MPI
29 –  if(worldRank == 0 ){
30 – #endif // is_mpi
69  
70 <    dumpFile.open(entry_plug->sampleName.c_str(), ios::out | ios::trunc );
70 >      }
71  
72 <    if( !dumpFile ){
72 >      sprintf(checkPointMsg, "Sucessfully opened output file for dumping.\n");
73 >      MPIcheckPoint();
74  
75 <      sprintf( painCave.errMsg,
76 <               "Could not open \"%s\" for dump output.\n",
38 <               entry_plug->sampleName.c_str());
39 <      painCave.isFatal = 1;
40 <      simError();
75 > #endif // is_mpi
76 >
77      }
78  
79 +
80 +  DumpWriter::DumpWriter(SimInfo* info, const std::string& filename)
81 +    : info_(info), filename_(filename){
82   #ifdef IS_MPI
44 –  }
83  
84 <  //sort the local atoms by global index
47 <  sortByGlobalIndex();
48 <  
49 <  sprintf( checkPointMsg,
50 <           "Sucessfully opened output file for dumping.\n");
51 <  MPIcheckPoint();
84 >      if (worldRank == 0) {
85   #endif // is_mpi
53 – }
86  
87 < DumpWriter::~DumpWriter( ){
87 >        eorFilename_ = filename_.substr(0, filename_.rfind(".")) + ".eor";
88 >        dumpFile_.open(filename_.c_str(), std::ios::out | std::ios::trunc);
89  
90 < #ifdef IS_MPI
91 <  if(worldRank == 0 ){
92 < #endif // is_mpi
90 >        if (!dumpFile_) {
91 >          sprintf(painCave.errMsg, "Could not open \"%s\" for dump output.\n",
92 >                  filename_.c_str());
93 >          painCave.isFatal = 1;
94 >          simError();
95 >        }
96  
61 –    dumpFile.close();
62 –
97   #ifdef IS_MPI
64 –  }
65 – #endif // is_mpi
66 – }
98  
99 < #ifdef IS_MPI
99 >      }
100  
101 < /**
102 < * A hook function to load balancing
72 < */
101 >      sprintf(checkPointMsg, "Sucessfully opened output file for dumping.\n");
102 >      MPIcheckPoint();
103  
104 < void DumpWriter::update(){
75 <  sortByGlobalIndex();          
76 < }
77 <  
78 < /**
79 < * Auxiliary sorting function
80 < */
81 <
82 < bool indexSortingCriterion(const pair<int, int>& p1, const pair<int, int>& p2){
83 <  return p1.second < p2.second;
84 < }
104 > #endif // is_mpi
105  
106 < /**
87 < * Sorting the local index by global index
88 < */
89 <
90 < void DumpWriter::sortByGlobalIndex(){
91 <  Molecule* mols = entry_plug->molecules;  
92 <  indexArray.clear();
93 <  
94 <  for(int i = 0; i < entry_plug->n_mol;i++)
95 <    indexArray.push_back(make_pair(i, mols[i].getGlobalIndex()));
96 <  
97 <  sort(indexArray.begin(), indexArray.end(), indexSortingCriterion);    
98 < }
106 >    }
107  
108 < #endif
108 >  DumpWriter::~DumpWriter() {
109  
110 < void DumpWriter::writeDump(double currentTime){
110 > #ifdef IS_MPI
111  
112 <  ofstream finalOut;
113 <  vector<ofstream*> fileStreams;
112 >    if (worldRank == 0) {
113 > #endif // is_mpi
114  
115 +      dumpFile_.close();
116 +
117   #ifdef IS_MPI
118 <  if(worldRank == 0 ){
109 < #endif    
110 <    finalOut.open( entry_plug->finalName.c_str(), ios::out | ios::trunc );
111 <    if( !finalOut ){
112 <      sprintf( painCave.errMsg,
113 <               "Could not open \"%s\" for final dump output.\n",
114 <               entry_plug->finalName.c_str() );
115 <      painCave.isFatal = 1;
116 <      simError();
118 >
119      }
120 < #ifdef IS_MPI
119 <  }
120 >
121   #endif // is_mpi
122  
123 <  fileStreams.push_back(&finalOut);
123 <  fileStreams.push_back(&dumpFile);
123 >  }
124  
125 <  writeFrame(fileStreams, currentTime);
125 >  void DumpWriter::writeCommentLine(std::ostream& os, Snapshot* s) {
126  
127 < #ifdef IS_MPI
128 <  finalOut.close();
129 < #endif
130 <        
131 < }
127 >    double currentTime;
128 >    Mat3x3d hmat;
129 >    double chi;
130 >    double integralOfChiDt;
131 >    Mat3x3d eta;
132 >    
133 >    currentTime = s->getTime();
134 >    hmat = s->getHmat();
135 >    chi = s->getChi();
136 >    integralOfChiDt = s->getIntegralOfChiDt();
137 >    eta = s->getEta();
138 >    
139 >    os << currentTime << ";\t"
140 >       << hmat(0, 0) << "\t" << hmat(1, 0) << "\t" << hmat(2, 0) << ";\t"
141 >       << hmat(0, 1) << "\t" << hmat(1, 1) << "\t" << hmat(2, 1) << ";\t"
142 >       << hmat(0, 2) << "\t" << hmat(1, 2) << "\t" << hmat(2, 2) << ";\t";
143  
144 < void DumpWriter::writeFinal(double currentTime){
144 >    //write out additional parameters, such as chi and eta
145  
146 <  ofstream finalOut;
136 <  vector<ofstream*> fileStreams;
146 >    os << chi << "\t" << integralOfChiDt << "\t;";
147  
148 < #ifdef IS_MPI
149 <  if(worldRank == 0 ){
150 < #endif // is_mpi
148 >    os << eta(0, 0) << "\t" << eta(1, 0) << "\t" << eta(2, 0) << ";\t"
149 >       << eta(0, 1) << "\t" << eta(1, 1) << "\t" << eta(2, 1) << ";\t"
150 >       << eta(0, 2) << "\t" << eta(1, 2) << "\t" << eta(2, 2) << ";";
151 >        
152 >    os << "\n";
153 >  }
154  
155 <    finalOut.open( entry_plug->finalName.c_str(), ios::out | ios::trunc );
155 >  void DumpWriter::writeFrame(std::ostream& os) {
156 >    const int BUFFERSIZE = 2000;
157 >    const int MINIBUFFERSIZE = 100;
158  
159 <    if( !finalOut ){
160 <      sprintf( painCave.errMsg,
146 <               "Could not open \"%s\" for final dump output.\n",
147 <               entry_plug->finalName.c_str() );
148 <      painCave.isFatal = 1;
149 <      simError();
150 <    }
159 >    char tempBuffer[BUFFERSIZE];
160 >    char writeLine[BUFFERSIZE];
161  
162 < #ifdef IS_MPI
163 <  }
164 < #endif // is_mpi
165 <  
156 <  fileStreams.push_back(&finalOut);  
157 <  writeFrame(fileStreams, currentTime);
162 >    Quat4d q;
163 >    Vector3d ji;
164 >    Vector3d pos;
165 >    Vector3d vel;
166  
167 < #ifdef IS_MPI
168 <  finalOut.close();
169 < #endif
167 >    Molecule* mol;
168 >    StuntDouble* integrableObject;
169 >    SimInfo::MoleculeIterator mi;
170 >    Molecule::IntegrableObjectIterator ii;
171    
172 < }
172 >    int nTotObjects;    
173 >    nTotObjects = info_->getNGlobalIntegrableObjects();
174  
175 < void DumpWriter::writeFrame( vector<ofstream*>& outFile, double currentTime ){
175 > #ifndef IS_MPI
176  
167 –  const int BUFFERSIZE = 2000;
168 –  const int MINIBUFFERSIZE = 100;
177  
178 <  char tempBuffer[BUFFERSIZE];  
179 <  char writeLine[BUFFERSIZE];
178 >    os << nTotObjects << "\n";
179 >        
180 >    writeCommentLine(os, info_->getSnapshotManager()->getCurrentSnapshot());
181  
182 <  int i;
174 <  unsigned int k;
182 >    for (mol = info_->beginMolecule(mi); mol != NULL; mol = info_->nextMolecule(mi)) {
183  
184 < #ifdef IS_MPI
185 <  
186 <  /*********************************************************************
179 <   * Documentation?  You want DOCUMENTATION?
180 <   *
181 <   * Why all the potatoes below?  
182 <   *
183 <   * To make a long story short, the original version of DumpWriter
184 <   * worked in the most inefficient way possible.  Node 0 would
185 <   * poke each of the node for an individual atom's formatted data
186 <   * as node 0 worked its way down the global index. This was particularly
187 <   * inefficient since the method blocked all processors at every atom
188 <   * (and did it twice!).
189 <   *
190 <   * An intermediate version of DumpWriter could be described from Node
191 <   * zero's perspective as follows:
192 <   *
193 <   *  1) Have 100 of your friends stand in a circle.
194 <   *  2) When you say go, have all of them start tossing potatoes at
195 <   *     you (one at a time).
196 <   *  3) Catch the potatoes.
197 <   *
198 <   * It was an improvement, but MPI has buffers and caches that could
199 <   * best be described in this analogy as "potato nets", so there's no
200 <   * need to block the processors atom-by-atom.
201 <   *
202 <   * This new and improved DumpWriter works in an even more efficient
203 <   * way:
204 <   *
205 <   *  1) Have 100 of your friend stand in a circle.
206 <   *  2) When you say go, have them start tossing 5-pound bags of
207 <   *     potatoes at you.
208 <   *  3) Once you've caught a friend's bag of potatoes,
209 <   *     toss them a spud to let them know they can toss another bag.
210 <   *
211 <   * How's THAT for documentation?
212 <   *
213 <   *********************************************************************/
184 >      for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL;
185 >           integrableObject = mol->nextIntegrableObject(ii)) {
186 >                
187  
188 <  int *potatoes;
189 <  int myPotato;
188 >        pos = integrableObject->getPos();
189 >        vel = integrableObject->getVel();
190  
191 <  int nProc;
192 <  int j, which_node, done, which_atom, local_index, currentIndex;
193 <  double atomData[13];
194 <  int isDirectional;
195 <  char* atomTypeString;
196 <  char MPIatomTypeString[MINIBUFFERSIZE];
224 <  int nObjects;
225 <  int msgLen; // the length of message actually recieved at master nodes
226 < #endif //is_mpi
191 >        sprintf(tempBuffer, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
192 >                integrableObject->getType().c_str(),
193 >                pos[0], pos[1], pos[2],
194 >                vel[0], vel[1], vel[2]);
195 >
196 >        strcpy(writeLine, tempBuffer);
197  
198 <  double q[4], ji[3];
199 <  DirectionalAtom* dAtom;
200 <  double pos[3], vel[3];
231 <  int nTotObjects;
232 <  StuntDouble* sd;
233 <  char* molName;
234 <  vector<StuntDouble*> integrableObjects;
235 <  vector<StuntDouble*>::iterator iter;
236 <  nTotObjects = entry_plug->getTotIntegrableObjects();
237 < #ifndef IS_MPI
238 <  
239 <  for(k = 0; k < outFile.size(); k++){
240 <    *outFile[k] << nTotObjects << "\n";
198 >        if (integrableObject->isDirectional()) {
199 >          q = integrableObject->getQ();
200 >          ji = integrableObject->getJ();
201  
202 <    *outFile[k] << currentTime << ";\t"
203 <               << entry_plug->Hmat[0][0] << "\t"
204 <                     << entry_plug->Hmat[1][0] << "\t"
205 <                     << entry_plug->Hmat[2][0] << ";\t"
206 <              
207 <               << entry_plug->Hmat[0][1] << "\t"
208 <                     << entry_plug->Hmat[1][1] << "\t"
249 <                     << entry_plug->Hmat[2][1] << ";\t"
202 >          sprintf(tempBuffer, "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
203 >                  q[0], q[1], q[2], q[3],
204 >                  ji[0], ji[1], ji[2]);
205 >          strcat(writeLine, tempBuffer);
206 >        } else {
207 >          strcat(writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
208 >        }
209  
210 <                     << entry_plug->Hmat[0][2] << "\t"
252 <                     << entry_plug->Hmat[1][2] << "\t"
253 <                     << entry_plug->Hmat[2][2] << ";";
210 >        os << writeLine;
211  
255 –    //write out additional parameters, such as chi and eta
256 –    *outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
257 –  }
258 –  
259 –  for( i=0; i< entry_plug->n_mol; i++ ){
260 –
261 –    integrableObjects = entry_plug->molecules[i].getIntegrableObjects();
262 –    molName = (entry_plug->compStamps[entry_plug->molecules[i].getStampID()])->getID();
263 –    
264 –    for( iter = integrableObjects.begin();iter !=  integrableObjects.end(); ++iter){
265 –      sd = *iter;
266 –      sd->getPos(pos);
267 –      sd->getVel(vel);
268 –
269 –      sprintf( tempBuffer,
270 –             "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
271 –             sd->getType(),
272 –             pos[0],
273 –             pos[1],
274 –             pos[2],
275 –             vel[0],
276 –             vel[1],
277 –             vel[2]);
278 –      strcpy( writeLine, tempBuffer );
279 –
280 –      if( sd->isDirectional() ){
281 –
282 –        sd->getQ( q );
283 –        sd->getJ( ji );
284 –
285 –        sprintf( tempBuffer,
286 –               "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
287 –               q[0],
288 –               q[1],
289 –               q[2],
290 –               q[3],
291 –                 ji[0],
292 –                 ji[1],
293 –                 ji[2]);
294 –        strcat( writeLine, tempBuffer );
212        }
296 –      else
297 –        strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
298 –    
299 –      for(k = 0; k < outFile.size(); k++)
300 –        *outFile[k] << writeLine;      
213      }
214  
215 < }
304 <
215 >    os.flush();
216   #else // is_mpi
217 +    /*********************************************************************
218 +     * Documentation?  You want DOCUMENTATION?
219 +     *
220 +     * Why all the potatoes below?  
221 +     *
222 +     * To make a long story short, the original version of DumpWriter
223 +     * worked in the most inefficient way possible.  Node 0 would
224 +     * poke each of the node for an individual atom's formatted data
225 +     * as node 0 worked its way down the global index. This was particularly
226 +     * inefficient since the method blocked all processors at every atom
227 +     * (and did it twice!).
228 +     *
229 +     * An intermediate version of DumpWriter could be described from Node
230 +     * zero's perspective as follows:
231 +     *
232 +     *  1) Have 100 of your friends stand in a circle.
233 +     *  2) When you say go, have all of them start tossing potatoes at
234 +     *     you (one at a time).
235 +     *  3) Catch the potatoes.
236 +     *
237 +     * It was an improvement, but MPI has buffers and caches that could
238 +     * best be described in this analogy as "potato nets", so there's no
239 +     * need to block the processors atom-by-atom.
240 +     *
241 +     * This new and improved DumpWriter works in an even more efficient
242 +     * way:
243 +     *
244 +     *  1) Have 100 of your friend stand in a circle.
245 +     *  2) When you say go, have them start tossing 5-pound bags of
246 +     *     potatoes at you.
247 +     *  3) Once you've caught a friend's bag of potatoes,
248 +     *     toss them a spud to let them know they can toss another bag.
249 +     *
250 +     * How's THAT for documentation?
251 +     *
252 +     *********************************************************************/
253 +    const int masterNode = 0;
254  
255 <  /* code to find maximum tag value */
256 <  
257 <  int *tagub, flag, MAXTAG;
258 <  MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);
259 <  if (flag) {
260 <    MAXTAG = *tagub;
261 <  } else {
262 <    MAXTAG = 32767;
263 <  }  
255 >    int * potatoes;
256 >    int myPotato;
257 >    int nProc;
258 >    int which_node;
259 >    double atomData[13];
260 >    int isDirectional;
261 >    char MPIatomTypeString[MINIBUFFERSIZE];
262 >    int msgLen; // the length of message actually recieved at master nodes
263 >    int haveError;
264 >    MPI_Status istatus;
265 >    int nCurObj;
266 >    
267 >    // code to find maximum tag value
268 >    int * tagub;
269 >    int flag;
270 >    int MAXTAG;
271 >    MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);
272  
273 <  int haveError;
273 >    if (flag) {
274 >      MAXTAG = *tagub;
275 >    } else {
276 >      MAXTAG = 32767;
277 >    }
278  
279 <  MPI_Status istatus;
320 <  int nCurObj;
321 <  int *MolToProcMap = mpiSim->getMolToProcMap();
279 >    if (worldRank == masterNode) { //master node (node 0) is responsible for writing the dump file
280  
281 <  // write out header and node 0's coordinates
281 >      // Node 0 needs a list of the magic potatoes for each processor;
282  
283 <  if( worldRank == 0 ){
283 >      MPI_Comm_size(MPI_COMM_WORLD, &nProc);
284 >      potatoes = new int[nProc];
285  
286 <    // Node 0 needs a list of the magic potatoes for each processor;
286 >      //write out the comment lines
287 >      for(int i = 0; i < nProc; i++) {
288 >        potatoes[i] = 0;
289 >      }
290  
329 –    nProc = mpiSim->getNProcessors();
330 –    potatoes = new int[nProc];
291  
292 <    //write out the comment lines
293 <    for (i = 0; i < nProc; i++)
334 <      potatoes[i] = 0;
335 <    
336 <      for(k = 0; k < outFile.size(); k++){
337 <        *outFile[k] << nTotObjects << "\n";
292 >      os << nTotObjects << "\n";
293 >      writeCommentLine(os, info_->getSnapshotManager()->getCurrentSnapshot());
294  
295 <        *outFile[k] << currentTime << ";\t"
340 <                         << entry_plug->Hmat[0][0] << "\t"
341 <                         << entry_plug->Hmat[1][0] << "\t"
342 <                         << entry_plug->Hmat[2][0] << ";\t"
295 >      for(int i = 0; i < info_->getNGlobalMolecules(); i++) {
296  
297 <                         << entry_plug->Hmat[0][1] << "\t"
345 <                         << entry_plug->Hmat[1][1] << "\t"
346 <                         << entry_plug->Hmat[2][1] << ";\t"
297 >        // Get the Node number which has this atom;
298  
299 <                         << entry_plug->Hmat[0][2] << "\t"
349 <                         << entry_plug->Hmat[1][2] << "\t"
350 <                         << entry_plug->Hmat[2][2] << ";";
351 <  
352 <        *outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
353 <    }
299 >        which_node = info_->getMolToProc(i);
300  
301 <    currentIndex = 0;
301 >        if (which_node != masterNode) { //current molecule is in slave node
302 >          if (potatoes[which_node] + 1 >= MAXTAG) {
303 >            // The potato was going to exceed the maximum value,
304 >            // so wrap this processor potato back to 0:        
305  
306 <    for (i = 0 ; i < mpiSim->getNMolGlobal(); i++ ) {
307 <      
308 <      // Get the Node number which has this atom;
309 <      
361 <      which_node = MolToProcMap[i];
362 <      
363 <      if (which_node != 0) {
364 <        
365 <        if (potatoes[which_node] + 1 >= MAXTAG) {
366 <          // The potato was going to exceed the maximum value,
367 <          // so wrap this processor potato back to 0:        
306 >            potatoes[which_node] = 0;
307 >            MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0,
308 >                     MPI_COMM_WORLD);
309 >          }
310  
311 <          potatoes[which_node] = 0;          
370 <          MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0, MPI_COMM_WORLD);
371 <          
372 <        }
311 >          myPotato = potatoes[which_node];
312  
313 <        myPotato = potatoes[which_node];        
313 >          //recieve the number of integrableObject in current molecule
314 >          MPI_Recv(&nCurObj, 1, MPI_INT, which_node, myPotato,
315 >                   MPI_COMM_WORLD, &istatus);
316 >          myPotato++;
317  
318 <        //recieve the number of integrableObject in current molecule
319 <        MPI_Recv(&nCurObj, 1, MPI_INT, which_node,
320 <                 myPotato, MPI_COMM_WORLD, &istatus);
321 <        myPotato++;
380 <        
381 <        for(int l = 0; l < nCurObj; l++){
318 >          for(int l = 0; l < nCurObj; l++) {
319 >            if (potatoes[which_node] + 2 >= MAXTAG) {
320 >              // The potato was going to exceed the maximum value,
321 >              // so wrap this processor potato back to 0:        
322  
323 <          if (potatoes[which_node] + 2 >= MAXTAG) {
324 <            // The potato was going to exceed the maximum value,
325 <            // so wrap this processor potato back to 0:        
323 >              potatoes[which_node] = 0;
324 >              MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node,
325 >                       0, MPI_COMM_WORLD);
326 >            }
327  
328 <            potatoes[which_node] = 0;          
329 <            MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0, MPI_COMM_WORLD);
330 <            
390 <          }
328 >            MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR,
329 >                     which_node, myPotato, MPI_COMM_WORLD,
330 >                     &istatus);
331  
332 <          MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, which_node,
393 <          myPotato, MPI_COMM_WORLD, &istatus);
332 >            myPotato++;
333  
334 <          atomTypeString = MPIatomTypeString;
334 >            MPI_Recv(atomData, 13, MPI_DOUBLE, which_node, myPotato,
335 >                     MPI_COMM_WORLD, &istatus);
336 >            myPotato++;
337  
338 <          myPotato++;
338 >            MPI_Get_count(&istatus, MPI_DOUBLE, &msgLen);
339  
340 <          MPI_Recv(atomData, 13, MPI_DOUBLE, which_node, myPotato, MPI_COMM_WORLD, &istatus);
341 <          myPotato++;
342 <
343 <          MPI_Get_count(&istatus, MPI_DOUBLE, &msgLen);
340 >            if (msgLen == 13)
341 >              isDirectional = 1;
342 >            else
343 >              isDirectional = 0;
344  
345 <          if(msgLen  == 13)
405 <            isDirectional = 1;
406 <          else
407 <            isDirectional = 0;
408 <          
409 <          // If we've survived to here, format the line:
410 <            
411 <          if (!isDirectional) {
412 <        
413 <            sprintf( writeLine,
414 <                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
415 <                 atomTypeString,
416 <                 atomData[0],
417 <                 atomData[1],
418 <                 atomData[2],
419 <                 atomData[3],
420 <                 atomData[4],
421 <                 atomData[5]);
422 <        
423 <           strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
424 <        
425 <          }
426 <          else {
427 <        
428 <                sprintf( writeLine,
429 <                         "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
430 <                         atomTypeString,
431 <                         atomData[0],
432 <                         atomData[1],
433 <                         atomData[2],
434 <                         atomData[3],
435 <                         atomData[4],
436 <                         atomData[5],
437 <                         atomData[6],
438 <                         atomData[7],
439 <                         atomData[8],
440 <                         atomData[9],
441 <                         atomData[10],
442 <                         atomData[11],
443 <                         atomData[12]);
444 <            
445 <          }
446 <          
447 <          for(k = 0; k < outFile.size(); k++)
448 <            *outFile[k] << writeLine;            
345 >            // If we've survived to here, format the line:
346  
347 <        }// end for(int l =0)
348 <        potatoes[which_node] = myPotato;
347 >            if (!isDirectional) {
348 >              sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
349 >                      MPIatomTypeString, atomData[0],
350 >                      atomData[1], atomData[2],
351 >                      atomData[3], atomData[4],
352 >                      atomData[5]);
353  
354 <      }
355 <      else {
356 <        
357 <        haveError = 0;
358 <        
359 <            local_index = indexArray[currentIndex].first;        
354 >              strcat(writeLine,
355 >                     "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
356 >            } else {
357 >              sprintf(writeLine,
358 >                      "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
359 >                      MPIatomTypeString,
360 >                      atomData[0],
361 >                      atomData[1],
362 >                      atomData[2],
363 >                      atomData[3],
364 >                      atomData[4],
365 >                      atomData[5],
366 >                      atomData[6],
367 >                      atomData[7],
368 >                      atomData[8],
369 >                      atomData[9],
370 >                      atomData[10],
371 >                      atomData[11],
372 >                      atomData[12]);
373 >            }
374  
375 <        integrableObjects = (entry_plug->molecules[local_index]).getIntegrableObjects();
375 >            os << writeLine;
376  
377 <        for(iter= integrableObjects.begin(); iter != integrableObjects.end(); ++iter){    
463 <                sd = *iter;
464 <            atomTypeString = sd->getType();
465 <            
466 <            sd->getPos(pos);
467 <            sd->getVel(vel);          
468 <          
469 <            atomData[0] = pos[0];
470 <            atomData[1] = pos[1];
471 <            atomData[2] = pos[2];
377 >          } // end for(int l =0)
378  
379 <            atomData[3] = vel[0];
380 <            atomData[4] = vel[1];
475 <            atomData[5] = vel[2];
476 <              
477 <            isDirectional = 0;
379 >          potatoes[which_node] = myPotato;
380 >        } else { //master node has current molecule
381  
382 <            if( sd->isDirectional() ){
382 >          mol = info_->getMoleculeByGlobalIndex(i);
383  
384 <              isDirectional = 1;
384 >          if (mol == NULL) {
385 >            sprintf(painCave.errMsg, "Molecule not found on node %d!", worldRank);
386 >            painCave.isFatal = 1;
387 >            simError();
388 >          }
389                  
390 <              sd->getQ( q );
391 <              sd->getJ( ji );
390 >          for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL;
391 >               integrableObject = mol->nextIntegrableObject(ii)) {      
392  
393 <              for (int j = 0; j < 6 ; j++)
394 <                atomData[j] = atomData[j];            
488 <              
489 <              atomData[6] = q[0];
490 <              atomData[7] = q[1];
491 <              atomData[8] = q[2];
492 <              atomData[9] = q[3];
493 <              
494 <              atomData[10] = ji[0];
495 <              atomData[11] = ji[1];
496 <              atomData[12] = ji[2];
497 <            }
498 <            
499 <            // If we've survived to here, format the line:
500 <            
501 <            if (!isDirectional) {
502 <        
503 <              sprintf( writeLine,
504 <                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
505 <                 atomTypeString,
506 <                 atomData[0],
507 <                 atomData[1],
508 <                 atomData[2],
509 <                 atomData[3],
510 <                 atomData[4],
511 <                 atomData[5]);
512 <        
513 <             strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
514 <        
515 <            }
516 <            else {
517 <        
518 <                sprintf( writeLine,
519 <                         "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
520 <                         atomTypeString,
521 <                         atomData[0],
522 <                         atomData[1],
523 <                         atomData[2],
524 <                         atomData[3],
525 <                         atomData[4],
526 <                         atomData[5],
527 <                         atomData[6],
528 <                         atomData[7],
529 <                         atomData[8],
530 <                         atomData[9],
531 <                         atomData[10],
532 <                         atomData[11],
533 <                         atomData[12]);
534 <              
535 <            }
536 <            
537 <            for(k = 0; k < outFile.size(); k++)
538 <              *outFile[k] << writeLine;
539 <            
540 <            
541 <        }//end for(iter = integrableObject.begin())
542 <        
543 <      currentIndex++;
544 <      }
393 >            pos = integrableObject->getPos();
394 >            vel = integrableObject->getVel();
395  
396 <    }//end for(i = 0; i < mpiSim->getNmol())
397 <    
398 <    for(k = 0; k < outFile.size(); k++)
549 <      outFile[k]->flush();
550 <    
551 <    sprintf( checkPointMsg,
552 <             "Sucessfully took a dump.\n");
553 <    
554 <    MPIcheckPoint();        
555 <    
556 <    delete[] potatoes;
557 <    
558 <  } else {
396 >            atomData[0] = pos[0];
397 >            atomData[1] = pos[1];
398 >            atomData[2] = pos[2];
399  
400 <    // worldRank != 0, so I'm a remote node.  
400 >            atomData[3] = vel[0];
401 >            atomData[4] = vel[1];
402 >            atomData[5] = vel[2];
403  
404 <    // Set my magic potato to 0:
404 >            isDirectional = 0;
405  
406 <    myPotato = 0;
407 <    currentIndex = 0;
566 <    
567 <    for (i = 0 ; i < mpiSim->getNMolGlobal(); i++ ) {
568 <      
569 <      // Am I the node which has this integrableObject?
570 <      
571 <      if (MolToProcMap[i] == worldRank) {
406 >            if (integrableObject->isDirectional()) {
407 >              isDirectional = 1;
408  
409 +              q = integrableObject->getQ();
410 +              ji = integrableObject->getJ();
411  
412 <        if (myPotato + 1 >= MAXTAG) {
413 <          
414 <          // The potato was going to exceed the maximum value,
577 <          // so wrap this processor potato back to 0 (and block until
578 <          // node 0 says we can go:
579 <          
580 <          MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus);
581 <          
582 <        }
412 >              for(int j = 0; j < 6; j++) {
413 >                atomData[j] = atomData[j];
414 >              }
415  
416 <          local_index = indexArray[currentIndex].first;        
417 <          integrableObjects = entry_plug->molecules[local_index].getIntegrableObjects();
418 <          
419 <          nCurObj = integrableObjects.size();
588 <                      
589 <          MPI_Send(&nCurObj, 1, MPI_INT, 0,
590 <                             myPotato, MPI_COMM_WORLD);
591 <          myPotato++;
416 >              atomData[6] = q[0];
417 >              atomData[7] = q[1];
418 >              atomData[8] = q[2];
419 >              atomData[9] = q[3];
420  
421 <          for( iter = integrableObjects.begin(); iter  != integrableObjects.end(); iter++){
421 >              atomData[10] = ji[0];
422 >              atomData[11] = ji[1];
423 >              atomData[12] = ji[2];
424 >            }
425  
426 <            if (myPotato + 2 >= MAXTAG) {
596 <          
597 <              // The potato was going to exceed the maximum value,
598 <              // so wrap this processor potato back to 0 (and block until
599 <              // node 0 says we can go:
600 <          
601 <              MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus);
602 <              
603 <            }
604 <            
605 <            sd = *iter;
606 <            
607 <            atomTypeString = sd->getType();
426 >            // If we've survived to here, format the line:
427  
428 <            sd->getPos(pos);
429 <            sd->getVel(vel);
428 >            if (!isDirectional) {
429 >              sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
430 >                      integrableObject->getType().c_str(), atomData[0],
431 >                      atomData[1], atomData[2],
432 >                      atomData[3], atomData[4],
433 >                      atomData[5]);
434  
435 <            atomData[0] = pos[0];
436 <            atomData[1] = pos[1];
437 <            atomData[2] = pos[2];
435 >              strcat(writeLine,
436 >                     "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
437 >            } else {
438 >              sprintf(writeLine,
439 >                      "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
440 >                      integrableObject->getType().c_str(),
441 >                      atomData[0],
442 >                      atomData[1],
443 >                      atomData[2],
444 >                      atomData[3],
445 >                      atomData[4],
446 >                      atomData[5],
447 >                      atomData[6],
448 >                      atomData[7],
449 >                      atomData[8],
450 >                      atomData[9],
451 >                      atomData[10],
452 >                      atomData[11],
453 >                      atomData[12]);
454 >            }
455  
616 –            atomData[3] = vel[0];
617 –            atomData[4] = vel[1];
618 –            atomData[5] = vel[2];
619 –              
620 –            isDirectional = 0;
456  
457 <            if( sd->isDirectional() ){
457 >            os << writeLine;
458  
459 <                isDirectional = 1;
459 >          } //end for(iter = integrableObject.begin())
460 >        }
461 >      } //end for(i = 0; i < mpiSim->getNmol())
462 >
463 >      os.flush();
464 >        
465 >      sprintf(checkPointMsg, "Sucessfully took a dump.\n");
466 >      MPIcheckPoint();
467 >
468 >      delete [] potatoes;
469 >    } else {
470 >
471 >      // worldRank != 0, so I'm a remote node.  
472 >
473 >      // Set my magic potato to 0:
474 >
475 >      myPotato = 0;
476 >
477 >      for(int i = 0; i < info_->getNGlobalMolecules(); i++) {
478 >
479 >        // Am I the node which has this integrableObject?
480 >        int whichNode = info_->getMolToProc(i);
481 >        if (whichNode == worldRank) {
482 >          if (myPotato + 1 >= MAXTAG) {
483 >
484 >            // The potato was going to exceed the maximum value,
485 >            // so wrap this processor potato back to 0 (and block until
486 >            // node 0 says we can go:
487 >
488 >            MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD,
489 >                     &istatus);
490 >          }
491 >
492 >          mol = info_->getMoleculeByGlobalIndex(i);
493 >
494                  
495 <                sd->getQ( q );
627 <                sd->getJ( ji );
628 <                
629 <                
630 <                atomData[6] = q[0];
631 <                atomData[7] = q[1];
632 <                atomData[8] = q[2];
633 <                atomData[9] = q[3];
634 <      
635 <                atomData[10] = ji[0];
636 <                atomData[11] = ji[1];
637 <                atomData[12] = ji[2];
638 <              }
495 >          nCurObj = mol->getNIntegrableObjects();
496  
497 <            
498 <            strncpy(MPIatomTypeString, atomTypeString, MINIBUFFERSIZE);
497 >          MPI_Send(&nCurObj, 1, MPI_INT, 0, myPotato, MPI_COMM_WORLD);
498 >          myPotato++;
499  
500 <            // null terminate the string before sending (just in case):
501 <            MPIatomTypeString[MINIBUFFERSIZE-1] = '\0';
500 >          for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL;
501 >               integrableObject = mol->nextIntegrableObject(ii)) {
502  
503 <            MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
647 <                             myPotato, MPI_COMM_WORLD);
648 <            
649 <            myPotato++;
650 <            
651 <            if (isDirectional) {
503 >            if (myPotato + 2 >= MAXTAG) {
504  
505 <              MPI_Send(atomData, 13, MPI_DOUBLE, 0,
506 <                       myPotato, MPI_COMM_WORLD);
507 <              
656 <            } else {
505 >              // The potato was going to exceed the maximum value,
506 >              // so wrap this processor potato back to 0 (and block until
507 >              // node 0 says we can go:
508  
509 <              MPI_Send(atomData, 6, MPI_DOUBLE, 0,
510 <                       myPotato, MPI_COMM_WORLD);
511 <            }
509 >              MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD,
510 >                       &istatus);
511 >            }
512  
513 <            myPotato++;  
513 >            pos = integrableObject->getPos();
514 >            vel = integrableObject->getVel();
515  
516 <          }
516 >            atomData[0] = pos[0];
517 >            atomData[1] = pos[1];
518 >            atomData[2] = pos[2];
519  
520 <          currentIndex++;    
521 <          
522 <        }
523 <      
520 >            atomData[3] = vel[0];
521 >            atomData[4] = vel[1];
522 >            atomData[5] = vel[2];
523 >
524 >            isDirectional = 0;
525 >
526 >            if (integrableObject->isDirectional()) {
527 >              isDirectional = 1;
528 >
529 >              q = integrableObject->getQ();
530 >              ji = integrableObject->getJ();
531 >
532 >              atomData[6] = q[0];
533 >              atomData[7] = q[1];
534 >              atomData[8] = q[2];
535 >              atomData[9] = q[3];
536 >
537 >              atomData[10] = ji[0];
538 >              atomData[11] = ji[1];
539 >              atomData[12] = ji[2];
540 >            }
541 >
542 >            strncpy(MPIatomTypeString, integrableObject->getType().c_str(), MINIBUFFERSIZE);
543 >
544 >            // null terminate the  std::string before sending (just in case):
545 >            MPIatomTypeString[MINIBUFFERSIZE - 1] = '\0';
546 >
547 >            MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
548 >                     myPotato, MPI_COMM_WORLD);
549 >
550 >            myPotato++;
551 >
552 >            if (isDirectional) {
553 >              MPI_Send(atomData, 13, MPI_DOUBLE, 0, myPotato,
554 >                       MPI_COMM_WORLD);
555 >            } else {
556 >              MPI_Send(atomData, 6, MPI_DOUBLE, 0, myPotato,
557 >                       MPI_COMM_WORLD);
558 >            }
559 >
560 >            myPotato++;
561 >          }
562 >                    
563 >        }
564 >            
565        }
566 +      sprintf(checkPointMsg, "Sucessfully took a dump.\n");
567 +      MPIcheckPoint();
568 +    }
569  
570 <    sprintf( checkPointMsg,
571 <             "Sucessfully took a dump.\n");
572 <    MPIcheckPoint();                
570 > #endif // is_mpi
571 >
572 >  }
573 >
574 >  void DumpWriter::writeDump() {
575 >    writeFrame(dumpFile_);
576 >
577 >  }
578 >
579 >  void DumpWriter::writeEor() {
580 >    std::ofstream eorStream;
581      
582 + #ifdef IS_MPI
583 +    if (worldRank == 0) {
584 + #endif // is_mpi
585 +
586 +      eorStream.open(eorFilename_.c_str());
587 +      if (!eorStream.is_open()) {
588 +        sprintf(painCave.errMsg, "DumpWriter : Could not open \"%s\" for writing.\n",
589 +                eorFilename_.c_str());
590 +        painCave.isFatal = 1;
591 +        simError();
592 +      }
593 +
594 + #ifdef IS_MPI
595      }
596 + #endif // is_mpi    
597  
598 +    writeFrame(eorStream);
599 +  }
600  
601 <  
601 >
602 >  void DumpWriter::writeDumpAndEor() {
603 >    std::ofstream eorStream;
604 >    std::vector<std::streambuf*> buffers;
605 > #ifdef IS_MPI
606 >    if (worldRank == 0) {
607   #endif // is_mpi
681 – }
608  
609 +      buffers.push_back(dumpFile_.rdbuf());
610 +
611 +      eorStream.open(eorFilename_.c_str());
612 +      if (!eorStream.is_open()) {
613 +        sprintf(painCave.errMsg, "DumpWriter : Could not open \"%s\" for writing.\n",
614 +                eorFilename_.c_str());
615 +        painCave.isFatal = 1;
616 +        simError();
617 +      }
618 +
619 +      buffers.push_back(eorStream.rdbuf());
620 +        
621   #ifdef IS_MPI
622 +    }
623 + #endif // is_mpi    
624  
625 < // a couple of functions to let us escape the write loop
625 >    TeeBuf tbuf(buffers.begin(), buffers.end());
626 >    std::ostream os(&tbuf);
627  
628 < void dWrite::DieDieDie( void ){
628 >    writeFrame(os);
629 >    
630 >  }
631  
689 –  MPI_Finalize();
690 –  exit (0);
691 – }
632  
633 < #endif //is_mpi
633 >
634 > }//end namespace oopse

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines