src/io/DumpWriter.cpp

/*
 * Copyright (c) 2005 The University of Notre Dame. All Rights Reserved.
 *
 * The University of Notre Dame grants you ("Licensee") a
 * non-exclusive, royalty free, license to use, modify and
 * redistribute this software in source and binary code form, provided
 * that the following conditions are met:
 *
 * 1. Acknowledgement of the program authors must be made in any
 *    publication of scientific results based in part on use of the
 *    program.  An acceptable form of acknowledgement is citation of
 *    the article in which the program was described (Matthew
 *    A. Meineke, Charles F. Vardeman II, Teng Lin, Christopher
 *    J. Fennell and J. Daniel Gezelter, "OOPSE: An Object-Oriented
 *    Parallel Simulation Engine for Molecular Dynamics,"
 *    J. Comput. Chem. 26, pp. 252-271 (2005))
 *
 * 2. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 3. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the
 *    distribution.
 *
 * This software is provided "AS IS," without a warranty of any
 * kind. All express or implied conditions, representations and
 * warranties, including any implied warranty of merchantability,
 * fitness for a particular purpose or non-infringement, are hereby
 * excluded.  The University of Notre Dame and its licensors shall not
 * be liable for any damages suffered by licensee as a result of
 * using, modifying or distributing the software or its
 * derivatives. In no event will the University of Notre Dame or its
 * licensors be liable for any lost revenue, profit or data, or for
 * direct, indirect, special, consequential, incidental or punitive
 * damages, however caused and regardless of the theory of liability,
 * arising out of the use of or inability to use software, even if the
 * University of Notre Dame has been advised of the possibility of
 * such damages.
 */
 
#include "io/DumpWriter.hpp"
#include "primitives/Molecule.hpp"
#include "utils/simError.h"
#include "io/basic_teebuf.hpp"
#include "io/gzstream.hpp"
#include "io/Globals.hpp"

#ifdef IS_MPI
#include <mpi.h>
#endif //is_mpi

namespace oopse {

  DumpWriter::DumpWriter(SimInfo* info) 
    : info_(info), filename_(info->getDumpFileName()), eorFilename_(info->getFinalConfigFileName()){

    Globals* simParams = info->getSimParams();
    needCompression_ = simParams->getCompressDumpFile();

#ifdef HAVE_LIBZ
    if (needCompression_) {
        filename_ += ".gz";
        eorFilename_ += ".gz";
    }
#endif
    
#ifdef IS_MPI

      if (worldRank == 0) {
#endif // is_mpi


        dumpFile_ = createOStream(filename_);

        if (!dumpFile_) {
          sprintf(painCave.errMsg, "Could not open \"%s\" for dump output.\n",
                  filename_.c_str());
          painCave.isFatal = 1;
          simError();
        }

#ifdef IS_MPI

      }

      sprintf(checkPointMsg, "Sucessfully opened output file for dumping.\n");
      MPIcheckPoint();

#endif // is_mpi

    }


  DumpWriter::DumpWriter(SimInfo* info, const std::string& filename) 
    : info_(info), filename_(filename){

    Globals* simParams = info->getSimParams();
    eorFilename_ = filename_.substr(0, filename_.rfind(".")) + ".eor";    

    needCompression_ = simParams->getCompressDumpFile();

#ifdef HAVE_LIBZ
    if (needCompression_) {
        filename_ += ".gz";
        eorFilename_ += ".gz";
    }
#endif
    
#ifdef IS_MPI

      if (worldRank == 0) {
#endif // is_mpi


        dumpFile_ = createOStream(filename_);

        if (!dumpFile_) {
          sprintf(painCave.errMsg, "Could not open \"%s\" for dump output.\n",
                  filename_.c_str());
          painCave.isFatal = 1;
          simError();
        }

#ifdef IS_MPI

      }

      sprintf(checkPointMsg, "Sucessfully opened output file for dumping.\n");
      MPIcheckPoint();

#endif // is_mpi

    }

  DumpWriter::~DumpWriter() {

#ifdef IS_MPI

    if (worldRank == 0) {
#endif // is_mpi

      delete dumpFile_;

#ifdef IS_MPI

    }

#endif // is_mpi

  }

  void DumpWriter::writeCommentLine(std::ostream& os, Snapshot* s) {

    double currentTime;
    Mat3x3d hmat;
    double chi;
    double integralOfChiDt;
    Mat3x3d eta;
    
    currentTime = s->getTime();
    hmat = s->getHmat();
    chi = s->getChi();
    integralOfChiDt = s->getIntegralOfChiDt();
    eta = s->getEta();
    
    os << currentTime << ";\t" 
       << hmat(0, 0) << "\t" << hmat(1, 0) << "\t" << hmat(2, 0) << ";\t" 
       << hmat(0, 1) << "\t" << hmat(1, 1) << "\t" << hmat(2, 1) << ";\t"
       << hmat(0, 2) << "\t" << hmat(1, 2) << "\t" << hmat(2, 2) << ";\t";

    //write out additional parameters, such as chi and eta

    os << chi << "\t" << integralOfChiDt << "\t;";

    os << eta(0, 0) << "\t" << eta(1, 0) << "\t" << eta(2, 0) << ";\t" 
       << eta(0, 1) << "\t" << eta(1, 1) << "\t" << eta(2, 1) << ";\t"
       << eta(0, 2) << "\t" << eta(1, 2) << "\t" << eta(2, 2) << ";";
        
    os << "\n";
  }

  void DumpWriter::writeFrame(std::ostream& os) {
    const int BUFFERSIZE = 2000;
    const int MINIBUFFERSIZE = 100;

    char tempBuffer[BUFFERSIZE];
    char writeLine[BUFFERSIZE];

    Quat4d q;
    Vector3d ji;
    Vector3d pos;
    Vector3d vel;

    Molecule* mol;
    StuntDouble* integrableObject;
    SimInfo::MoleculeIterator mi;
    Molecule::IntegrableObjectIterator ii;
  
    int nTotObjects;    
    nTotObjects = info_->getNGlobalIntegrableObjects();

#ifndef IS_MPI


    os << nTotObjects << "\n";
        
    writeCommentLine(os, info_->getSnapshotManager()->getCurrentSnapshot());

    for (mol = info_->beginMolecule(mi); mol != NULL; mol = info_->nextMolecule(mi)) {

      for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL; 
           integrableObject = mol->nextIntegrableObject(ii)) { 
                

        pos = integrableObject->getPos();
        vel = integrableObject->getVel();

        sprintf(tempBuffer, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                integrableObject->getType().c_str(), 
                pos[0], pos[1], pos[2],
                vel[0], vel[1], vel[2]);

        strcpy(writeLine, tempBuffer);

        if (integrableObject->isDirectional()) {
          q = integrableObject->getQ();
          ji = integrableObject->getJ();

          sprintf(tempBuffer, "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", 
                  q[0], q[1], q[2], q[3],
                  ji[0], ji[1], ji[2]);
          strcat(writeLine, tempBuffer);
        } else {
          strcat(writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
        }

        os << writeLine;

      }
    }

    os.flush();
#else // is_mpi
    /*********************************************************************
     * Documentation?  You want DOCUMENTATION?
     * 
     * Why all the potatoes below?  
     *
     * To make a long story short, the original version of DumpWriter
     * worked in the most inefficient way possible.  Node 0 would 
     * poke each of the node for an individual atom's formatted data 
     * as node 0 worked its way down the global index. This was particularly 
     * inefficient since the method blocked all processors at every atom 
     * (and did it twice!).
     *
     * An intermediate version of DumpWriter could be described from Node
     * zero's perspective as follows:
     * 
     *  1) Have 100 of your friends stand in a circle.
     *  2) When you say go, have all of them start tossing potatoes at
     *     you (one at a time).
     *  3) Catch the potatoes.
     *
     * It was an improvement, but MPI has buffers and caches that could 
     * best be described in this analogy as "potato nets", so there's no 
     * need to block the processors atom-by-atom.
     * 
     * This new and improved DumpWriter works in an even more efficient 
     * way:
     * 
     *  1) Have 100 of your friend stand in a circle.
     *  2) When you say go, have them start tossing 5-pound bags of 
     *     potatoes at you.
     *  3) Once you've caught a friend's bag of potatoes,
     *     toss them a spud to let them know they can toss another bag.
     *
     * How's THAT for documentation?
     *
     *********************************************************************/
    const int masterNode = 0;

    int * potatoes;
    int myPotato;
    int nProc;
    int which_node;
    double atomData[13];
    int isDirectional;
    char MPIatomTypeString[MINIBUFFERSIZE];
    int msgLen; // the length of message actually recieved at master nodes
    int haveError;
    MPI_Status istatus;
    int nCurObj;
    
    // code to find maximum tag value
    int * tagub;
    int flag;
    int MAXTAG;
    MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);

    if (flag) {
      MAXTAG = *tagub;
    } else {
      MAXTAG = 32767;
    }

    if (worldRank == masterNode) { //master node (node 0) is responsible for writing the dump file

      // Node 0 needs a list of the magic potatoes for each processor;

      MPI_Comm_size(MPI_COMM_WORLD, &nProc);
      potatoes = new int[nProc];

      //write out the comment lines
      for(int i = 0; i < nProc; i++) {
        potatoes[i] = 0;
      }


      os << nTotObjects << "\n";
      writeCommentLine(os, info_->getSnapshotManager()->getCurrentSnapshot());

      for(int i = 0; i < info_->getNGlobalMolecules(); i++) {

        // Get the Node number which has this atom;

        which_node = info_->getMolToProc(i);

        if (which_node != masterNode) { //current molecule is in slave node
          if (potatoes[which_node] + 1 >= MAXTAG) {
            // The potato was going to exceed the maximum value, 
            // so wrap this processor potato back to 0:         

            potatoes[which_node] = 0;
            MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0,
                     MPI_COMM_WORLD);
          }

          myPotato = potatoes[which_node];

          //recieve the number of integrableObject in current molecule
          MPI_Recv(&nCurObj, 1, MPI_INT, which_node, myPotato,
                   MPI_COMM_WORLD, &istatus);
          myPotato++;

          for(int l = 0; l < nCurObj; l++) {
            if (potatoes[which_node] + 2 >= MAXTAG) {
              // The potato was going to exceed the maximum value, 
              // so wrap this processor potato back to 0:         

              potatoes[which_node] = 0;
              MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node,
                       0, MPI_COMM_WORLD);
            }

            MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR,
                     which_node, myPotato, MPI_COMM_WORLD,
                     &istatus);

            myPotato++;

            MPI_Recv(atomData, 13, MPI_DOUBLE, which_node, myPotato,
                     MPI_COMM_WORLD, &istatus);
            myPotato++;

            MPI_Get_count(&istatus, MPI_DOUBLE, &msgLen);

            if (msgLen == 13)
              isDirectional = 1;
            else
              isDirectional = 0;

            // If we've survived to here, format the line:

            if (!isDirectional) {
              sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                      MPIatomTypeString, atomData[0],
                      atomData[1], atomData[2],
                      atomData[3], atomData[4],
                      atomData[5]);

              strcat(writeLine,
                     "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
            } else {
              sprintf(writeLine,
                      "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
                      MPIatomTypeString,
                      atomData[0],
                      atomData[1],
                      atomData[2],
                      atomData[3],
                      atomData[4],
                      atomData[5],
                      atomData[6],
                      atomData[7],
                      atomData[8],
                      atomData[9],
                      atomData[10],
                      atomData[11],
                      atomData[12]);
            }

            os << writeLine;

          } // end for(int l =0)

          potatoes[which_node] = myPotato;
        } else { //master node has current molecule

          mol = info_->getMoleculeByGlobalIndex(i);

          if (mol == NULL) {
            sprintf(painCave.errMsg, "Molecule not found on node %d!", worldRank);
            painCave.isFatal = 1;
            simError();
          }
                
          for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL; 
               integrableObject = mol->nextIntegrableObject(ii)) {      

            pos = integrableObject->getPos();
            vel = integrableObject->getVel();

            atomData[0] = pos[0];
            atomData[1] = pos[1];
            atomData[2] = pos[2];

            atomData[3] = vel[0];
            atomData[4] = vel[1];
            atomData[5] = vel[2];

            isDirectional = 0;

            if (integrableObject->isDirectional()) {
              isDirectional = 1;

              q = integrableObject->getQ();
              ji = integrableObject->getJ();

              for(int j = 0; j < 6; j++) {
                atomData[j] = atomData[j];
              }

              atomData[6] = q[0];
              atomData[7] = q[1];
              atomData[8] = q[2];
              atomData[9] = q[3];

              atomData[10] = ji[0];
              atomData[11] = ji[1];
              atomData[12] = ji[2];
            }

            // If we've survived to here, format the line:

            if (!isDirectional) {
              sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                      integrableObject->getType().c_str(), atomData[0],
                      atomData[1], atomData[2],
                      atomData[3], atomData[4],
                      atomData[5]);

              strcat(writeLine,
                     "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
            } else {
              sprintf(writeLine,
                      "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
                      integrableObject->getType().c_str(),
                      atomData[0],
                      atomData[1],
                      atomData[2],
                      atomData[3],
                      atomData[4],
                      atomData[5],
                      atomData[6],
                      atomData[7],
                      atomData[8],
                      atomData[9],
                      atomData[10],
                      atomData[11],
                      atomData[12]);
            }


            os << writeLine;

          } //end for(iter = integrableObject.begin())
        }
      } //end for(i = 0; i < mpiSim->getNmol())

      os.flush();
        
      sprintf(checkPointMsg, "Sucessfully took a dump.\n");
      MPIcheckPoint();

      delete [] potatoes;
    } else {

      // worldRank != 0, so I'm a remote node.  

      // Set my magic potato to 0:

      myPotato = 0;

      for(int i = 0; i < info_->getNGlobalMolecules(); i++) {

        // Am I the node which has this integrableObject?
        int whichNode = info_->getMolToProc(i);
        if (whichNode == worldRank) {
          if (myPotato + 1 >= MAXTAG) {

            // The potato was going to exceed the maximum value, 
            // so wrap this processor potato back to 0 (and block until
            // node 0 says we can go:

            MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD,
                     &istatus);
          }

          mol = info_->getMoleculeByGlobalIndex(i);

                
          nCurObj = mol->getNIntegrableObjects();

          MPI_Send(&nCurObj, 1, MPI_INT, 0, myPotato, MPI_COMM_WORLD);
          myPotato++;

          for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL; 
               integrableObject = mol->nextIntegrableObject(ii)) {

            if (myPotato + 2 >= MAXTAG) {

              // The potato was going to exceed the maximum value, 
              // so wrap this processor potato back to 0 (and block until
              // node 0 says we can go:

              MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD,
                       &istatus);
            }

            pos = integrableObject->getPos();
            vel = integrableObject->getVel();

            atomData[0] = pos[0];
            atomData[1] = pos[1];
            atomData[2] = pos[2];

            atomData[3] = vel[0];
            atomData[4] = vel[1];
            atomData[5] = vel[2];

            isDirectional = 0;

            if (integrableObject->isDirectional()) {
              isDirectional = 1;

              q = integrableObject->getQ();
              ji = integrableObject->getJ();

              atomData[6] = q[0];
              atomData[7] = q[1];
              atomData[8] = q[2];
              atomData[9] = q[3];

              atomData[10] = ji[0];
              atomData[11] = ji[1];
              atomData[12] = ji[2];
            }

            strncpy(MPIatomTypeString, integrableObject->getType().c_str(), MINIBUFFERSIZE);

            // null terminate the  std::string before sending (just in case):
            MPIatomTypeString[MINIBUFFERSIZE - 1] = '\0';

            MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
                     myPotato, MPI_COMM_WORLD);

            myPotato++;

            if (isDirectional) {
              MPI_Send(atomData, 13, MPI_DOUBLE, 0, myPotato,
                       MPI_COMM_WORLD);
            } else {
              MPI_Send(atomData, 6, MPI_DOUBLE, 0, myPotato,
                       MPI_COMM_WORLD);
            }

            myPotato++;
          }
                    
        }
            
      }
      sprintf(checkPointMsg, "Sucessfully took a dump.\n");
      MPIcheckPoint();
    }

#endif // is_mpi

  }

  void DumpWriter::writeDump() {
    writeFrame(*dumpFile_);
  }

  void DumpWriter::writeEor() {
    std::ostream* eorStream;
    
#ifdef IS_MPI
    if (worldRank == 0) {
#endif // is_mpi

      eorStream = createOStream(eorFilename_);

#ifdef IS_MPI
    }
#endif // is_mpi    

    writeFrame(*eorStream);

#ifdef IS_MPI
    if (worldRank == 0) {
#endif // is_mpi
    delete eorStream;

#ifdef IS_MPI
    }
#endif // is_mpi  

  }


  void DumpWriter::writeDumpAndEor() {
    std::vector<std::streambuf*> buffers;
    std::ostream* eorStream;
#ifdef IS_MPI
    if (worldRank == 0) {
#endif // is_mpi

      buffers.push_back(dumpFile_->rdbuf());

      eorStream = createOStream(eorFilename_);

      buffers.push_back(eorStream->rdbuf());
        
#ifdef IS_MPI
    }
#endif // is_mpi    

    TeeBuf tbuf(buffers.begin(), buffers.end());
    std::ostream os(&tbuf);

    writeFrame(os);

#ifdef IS_MPI
    if (worldRank == 0) {
#endif // is_mpi
    delete eorStream;

#ifdef IS_MPI
    }
#endif // is_mpi  
    
  }

std::ostream* DumpWriter::createOStream(const std::string& filename) {

    std::ostream* newOStream;
#ifdef HAVE_LIBZ 
    if (needCompression_) {
        newOStream = new ogzstream(filename.c_str());
    } else {
        newOStream = new std::ofstream(filename.c_str());
    }
#else
    newOStream = new std::ofstream(filename.c_str());
#endif
    return newOStream;
}

}//end namespace oopse
Revision:	2318
Committed:	Wed Sep 21 20:59:31 2005 UTC (18 years, 9 months ago) by tim
File size:	17658 byte(s)
Log Message:	DumpReader using pure c++ io
#	User	Rev	Content
1	gezelter	2204	/*
2	gezelter	1930	* Copyright (c) 2005 The University of Notre Dame. All Rights Reserved.
3			*
4			* The University of Notre Dame grants you ("Licensee") a
5			* non-exclusive, royalty free, license to use, modify and
6			* redistribute this software in source and binary code form, provided
7			* that the following conditions are met:
8			*
9			* 1. Acknowledgement of the program authors must be made in any
10			* publication of scientific results based in part on use of the
11			* program. An acceptable form of acknowledgement is citation of
12			* the article in which the program was described (Matthew
13			* A. Meineke, Charles F. Vardeman II, Teng Lin, Christopher
14			* J. Fennell and J. Daniel Gezelter, "OOPSE: An Object-Oriented
15			* Parallel Simulation Engine for Molecular Dynamics,"
16			* J. Comput. Chem. 26, pp. 252-271 (2005))
17			*
18			* 2. Redistributions of source code must retain the above copyright
19			* notice, this list of conditions and the following disclaimer.
20			*
21			* 3. Redistributions in binary form must reproduce the above copyright
22			* notice, this list of conditions and the following disclaimer in the
23			* documentation and/or other materials provided with the
24			* distribution.
25			*
26			* This software is provided "AS IS," without a warranty of any
27			* kind. All express or implied conditions, representations and
28			* warranties, including any implied warranty of merchantability,
29			* fitness for a particular purpose or non-infringement, are hereby
30			* excluded. The University of Notre Dame and its licensors shall not
31			* be liable for any damages suffered by licensee as a result of
32			* using, modifying or distributing the software or its
33			* derivatives. In no event will the University of Notre Dame or its
34			* licensors be liable for any lost revenue, profit or data, or for
35			* direct, indirect, special, consequential, incidental or punitive
36			* damages, however caused and regardless of the theory of liability,
37			* arising out of the use of or inability to use software, even if the
38			* University of Notre Dame has been advised of the possibility of
39			* such damages.
40			*/
41
42			#include "io/DumpWriter.hpp"
43			#include "primitives/Molecule.hpp"
44			#include "utils/simError.h"
45	tim	2060	#include "io/basic_teebuf.hpp"
46	tim	2314	#include "io/gzstream.hpp"
47			#include "io/Globals.hpp"
48
49	gezelter	1490	#ifdef IS_MPI
50			#include <mpi.h>
51	gezelter	1930	#endif //is_mpi
52	gezelter	1490
53	gezelter	1930	namespace oopse {
54	gezelter	1490
55	gezelter	2204	DumpWriter::DumpWriter(SimInfo* info)
56			: info_(info), filename_(info->getDumpFileName()), eorFilename_(info->getFinalConfigFileName()){
57	tim	2314
58			Globals* simParams = info->getSimParams();
59			needCompression_ = simParams->getCompressDumpFile();
60
61	tim	2318	#ifdef HAVE_LIBZ
62	tim	2314	if (needCompression_) {
63			filename_ += ".gz";
64			eorFilename_ += ".gz";
65			}
66	tim	2318	#endif
67	tim	2314
68	tim	2060	#ifdef IS_MPI
69
70	gezelter	2204	if (worldRank == 0) {
71	tim	2060	#endif // is_mpi
72
73
74	tim	2314	dumpFile_ = createOStream(filename_);
75
76	tim	2060	if (!dumpFile_) {
77	gezelter	2204	sprintf(painCave.errMsg, "Could not open \"%s\" for dump output.\n",
78			filename_.c_str());
79			painCave.isFatal = 1;
80			simError();
81	tim	2060	}
82
83			#ifdef IS_MPI
84
85	gezelter	2204	}
86	tim	2060
87	gezelter	2204	sprintf(checkPointMsg, "Sucessfully opened output file for dumping.\n");
88			MPIcheckPoint();
89	tim	2060
90			#endif // is_mpi
91
92	gezelter	2204	}
93	tim	2060
94
95	gezelter	2204	DumpWriter::DumpWriter(SimInfo* info, const std::string& filename)
96			: info_(info), filename_(filename){
97	tim	2314
98			Globals* simParams = info->getSimParams();
99			eorFilename_ = filename_.substr(0, filename_.rfind(".")) + ".eor";
100
101			needCompression_ = simParams->getCompressDumpFile();
102	tim	2318
103			#ifdef HAVE_LIBZ
104	tim	2314	if (needCompression_) {
105			filename_ += ".gz";
106			eorFilename_ += ".gz";
107			}
108	tim	2318	#endif
109	tim	2314
110	gezelter	1930	#ifdef IS_MPI
111	gezelter	1490
112	gezelter	2204	if (worldRank == 0) {
113	gezelter	1930	#endif // is_mpi
114	gezelter	1490
115
116	tim	2314	dumpFile_ = createOStream(filename_);
117
118	gezelter	1930	if (!dumpFile_) {
119	gezelter	2204	sprintf(painCave.errMsg, "Could not open \"%s\" for dump output.\n",
120			filename_.c_str());
121			painCave.isFatal = 1;
122			simError();
123	gezelter	1930	}
124	gezelter	1490
125			#ifdef IS_MPI
126
127	gezelter	2204	}
128	gezelter	1490
129	gezelter	2204	sprintf(checkPointMsg, "Sucessfully opened output file for dumping.\n");
130			MPIcheckPoint();
131	gezelter	1490
132			#endif // is_mpi
133	gezelter	1930
134	gezelter	2204	}
135	gezelter	1490
136	gezelter	2204	DumpWriter::~DumpWriter() {
137	gezelter	1490
138			#ifdef IS_MPI
139	gezelter	1930
140			if (worldRank == 0) {
141	gezelter	1490	#endif // is_mpi
142
143	tim	2314	delete dumpFile_;
144	gezelter	1490
145			#ifdef IS_MPI
146	gezelter	1930
147			}
148
149	gezelter	1490	#endif // is_mpi
150	gezelter	1930
151	gezelter	2204	}
152	gezelter	1490
153	gezelter	2204	void DumpWriter::writeCommentLine(std::ostream& os, Snapshot* s) {
154	gezelter	1490
155	gezelter	1930	double currentTime;
156			Mat3x3d hmat;
157			double chi;
158			double integralOfChiDt;
159			Mat3x3d eta;
160
161			currentTime = s->getTime();
162			hmat = s->getHmat();
163			chi = s->getChi();
164			integralOfChiDt = s->getIntegralOfChiDt();
165			eta = s->getEta();
166
167			os << currentTime << ";\t"
168	gezelter	2204	<< hmat(0, 0) << "\t" << hmat(1, 0) << "\t" << hmat(2, 0) << ";\t"
169			<< hmat(0, 1) << "\t" << hmat(1, 1) << "\t" << hmat(2, 1) << ";\t"
170			<< hmat(0, 2) << "\t" << hmat(1, 2) << "\t" << hmat(2, 2) << ";\t";
171	gezelter	1490
172	gezelter	1930	//write out additional parameters, such as chi and eta
173
174			os << chi << "\t" << integralOfChiDt << "\t;";
175
176			os << eta(0, 0) << "\t" << eta(1, 0) << "\t" << eta(2, 0) << ";\t"
177	gezelter	2204	<< eta(0, 1) << "\t" << eta(1, 1) << "\t" << eta(2, 1) << ";\t"
178			<< eta(0, 2) << "\t" << eta(1, 2) << "\t" << eta(2, 2) << ";";
179	gezelter	1930
180	tim	2060	os << "\n";
181	gezelter	2204	}
182	gezelter	1490
183	gezelter	2204	void DumpWriter::writeFrame(std::ostream& os) {
184	gezelter	1930	const int BUFFERSIZE = 2000;
185			const int MINIBUFFERSIZE = 100;
186
187			char tempBuffer[BUFFERSIZE];
188			char writeLine[BUFFERSIZE];
189
190			Quat4d q;
191			Vector3d ji;
192			Vector3d pos;
193			Vector3d vel;
194
195			Molecule* mol;
196			StuntDouble* integrableObject;
197			SimInfo::MoleculeIterator mi;
198			Molecule::IntegrableObjectIterator ii;
199	gezelter	1490
200	gezelter	1930	int nTotObjects;
201			nTotObjects = info_->getNGlobalIntegrableObjects();
202	gezelter	1490
203	gezelter	1930	#ifndef IS_MPI
204	gezelter	1490
205
206	gezelter	1930	os << nTotObjects << "\n";
207
208			writeCommentLine(os, info_->getSnapshotManager()->getCurrentSnapshot());
209	gezelter	1490
210	gezelter	1930	for (mol = info_->beginMolecule(mi); mol != NULL; mol = info_->nextMolecule(mi)) {
211	gezelter	1490
212	gezelter	2204	for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL;
213			integrableObject = mol->nextIntegrableObject(ii)) {
214	gezelter	1930
215	gezelter	1490
216	gezelter	2204	pos = integrableObject->getPos();
217			vel = integrableObject->getVel();
218	gezelter	1490
219	gezelter	2204	sprintf(tempBuffer, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
220			integrableObject->getType().c_str(),
221			pos[0], pos[1], pos[2],
222			vel[0], vel[1], vel[2]);
223	gezelter	1490
224	gezelter	2204	strcpy(writeLine, tempBuffer);
225	gezelter	1490
226	gezelter	2204	if (integrableObject->isDirectional()) {
227			q = integrableObject->getQ();
228			ji = integrableObject->getJ();
229	gezelter	1490
230	gezelter	2204	sprintf(tempBuffer, "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
231			q[0], q[1], q[2], q[3],
232			ji[0], ji[1], ji[2]);
233			strcat(writeLine, tempBuffer);
234			} else {
235			strcat(writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
236			}
237	gezelter	1490
238	gezelter	2204	os << writeLine;
239	gezelter	1490
240	gezelter	2204	}
241	gezelter	1490	}
242
243	tim	2008	os.flush();
244	gezelter	1930	#else // is_mpi
245			/*********************************************************************
246			* Documentation? You want DOCUMENTATION?
247			*
248			* Why all the potatoes below?
249			*
250			* To make a long story short, the original version of DumpWriter
251			* worked in the most inefficient way possible. Node 0 would
252			* poke each of the node for an individual atom's formatted data
253			* as node 0 worked its way down the global index. This was particularly
254			* inefficient since the method blocked all processors at every atom
255			* (and did it twice!).
256			*
257			* An intermediate version of DumpWriter could be described from Node
258			* zero's perspective as follows:
259			*
260			* 1) Have 100 of your friends stand in a circle.
261			* 2) When you say go, have all of them start tossing potatoes at
262			* you (one at a time).
263			* 3) Catch the potatoes.
264			*
265			* It was an improvement, but MPI has buffers and caches that could
266			* best be described in this analogy as "potato nets", so there's no
267			* need to block the processors atom-by-atom.
268			*
269			* This new and improved DumpWriter works in an even more efficient
270			* way:
271			*
272			* 1) Have 100 of your friend stand in a circle.
273			* 2) When you say go, have them start tossing 5-pound bags of
274			* potatoes at you.
275			* 3) Once you've caught a friend's bag of potatoes,
276			* toss them a spud to let them know they can toss another bag.
277			*
278			* How's THAT for documentation?
279			*
280			*********************************************************************/
281			const int masterNode = 0;
282	gezelter	1490
283	gezelter	1930	int * potatoes;
284			int myPotato;
285			int nProc;
286			int which_node;
287			double atomData[13];
288			int isDirectional;
289			char MPIatomTypeString[MINIBUFFERSIZE];
290			int msgLen; // the length of message actually recieved at master nodes
291			int haveError;
292			MPI_Status istatus;
293			int nCurObj;
294
295			// code to find maximum tag value
296			int * tagub;
297			int flag;
298			int MAXTAG;
299			MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);
300	gezelter	1490
301	gezelter	1930	if (flag) {
302	gezelter	2204	MAXTAG = *tagub;
303	gezelter	1930	} else {
304	gezelter	2204	MAXTAG = 32767;
305	gezelter	1930	}
306	gezelter	1490
307	gezelter	1930	if (worldRank == masterNode) { //master node (node 0) is responsible for writing the dump file
308	gezelter	1490
309	gezelter	2204	// Node 0 needs a list of the magic potatoes for each processor;
310	gezelter	1490
311	gezelter	2204	MPI_Comm_size(MPI_COMM_WORLD, &nProc);
312			potatoes = new int[nProc];
313	gezelter	1490
314	gezelter	2204	//write out the comment lines
315			for(int i = 0; i < nProc; i++) {
316			potatoes[i] = 0;
317			}
318	gezelter	1490
319
320	gezelter	2204	os << nTotObjects << "\n";
321			writeCommentLine(os, info_->getSnapshotManager()->getCurrentSnapshot());
322	gezelter	1490
323	gezelter	2204	for(int i = 0; i < info_->getNGlobalMolecules(); i++) {
324	gezelter	1490
325	gezelter	2204	// Get the Node number which has this atom;
326	gezelter	1490
327	gezelter	2204	which_node = info_->getMolToProc(i);
328	gezelter	1490
329	gezelter	2204	if (which_node != masterNode) { //current molecule is in slave node
330			if (potatoes[which_node] + 1 >= MAXTAG) {
331			// The potato was going to exceed the maximum value,
332			// so wrap this processor potato back to 0:
333	gezelter	1490
334	gezelter	2204	potatoes[which_node] = 0;
335			MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0,
336			MPI_COMM_WORLD);
337			}
338	gezelter	1490
339	gezelter	2204	myPotato = potatoes[which_node];
340	gezelter	1490
341	gezelter	2204	//recieve the number of integrableObject in current molecule
342			MPI_Recv(&nCurObj, 1, MPI_INT, which_node, myPotato,
343			MPI_COMM_WORLD, &istatus);
344			myPotato++;
345	gezelter	1490
346	gezelter	2204	for(int l = 0; l < nCurObj; l++) {
347			if (potatoes[which_node] + 2 >= MAXTAG) {
348			// The potato was going to exceed the maximum value,
349			// so wrap this processor potato back to 0:
350	gezelter	1490
351	gezelter	2204	potatoes[which_node] = 0;
352			MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node,
353			0, MPI_COMM_WORLD);
354			}
355	gezelter	1490
356	gezelter	2204	MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR,
357			which_node, myPotato, MPI_COMM_WORLD,
358			&istatus);
359	gezelter	1490
360	gezelter	2204	myPotato++;
361	gezelter	1490
362	gezelter	2204	MPI_Recv(atomData, 13, MPI_DOUBLE, which_node, myPotato,
363			MPI_COMM_WORLD, &istatus);
364			myPotato++;
365	gezelter	1490
366	gezelter	2204	MPI_Get_count(&istatus, MPI_DOUBLE, &msgLen);
367	gezelter	1490
368	gezelter	2204	if (msgLen == 13)
369			isDirectional = 1;
370			else
371			isDirectional = 0;
372	gezelter	1490
373	gezelter	2204	// If we've survived to here, format the line:
374	gezelter	1490
375	gezelter	2204	if (!isDirectional) {
376			sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
377			MPIatomTypeString, atomData[0],
378			atomData[1], atomData[2],
379			atomData[3], atomData[4],
380			atomData[5]);
381	gezelter	1490
382	gezelter	2204	strcat(writeLine,
383			"0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
384			} else {
385			sprintf(writeLine,
386			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
387			MPIatomTypeString,
388			atomData[0],
389			atomData[1],
390			atomData[2],
391			atomData[3],
392			atomData[4],
393			atomData[5],
394			atomData[6],
395			atomData[7],
396			atomData[8],
397			atomData[9],
398			atomData[10],
399			atomData[11],
400			atomData[12]);
401			}
402	gezelter	1490
403	gezelter	2204	os << writeLine;
404	gezelter	1490
405	gezelter	2204	} // end for(int l =0)
406	gezelter	1490
407	gezelter	2204	potatoes[which_node] = myPotato;
408			} else { //master node has current molecule
409	gezelter	1490
410	gezelter	2204	mol = info_->getMoleculeByGlobalIndex(i);
411	gezelter	1490
412	gezelter	2204	if (mol == NULL) {
413			sprintf(painCave.errMsg, "Molecule not found on node %d!", worldRank);
414			painCave.isFatal = 1;
415			simError();
416			}
417	gezelter	1930
418	gezelter	2204	for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL;
419			integrableObject = mol->nextIntegrableObject(ii)) {
420	gezelter	1490
421	gezelter	2204	pos = integrableObject->getPos();
422			vel = integrableObject->getVel();
423	gezelter	1490
424	gezelter	2204	atomData[0] = pos[0];
425			atomData[1] = pos[1];
426			atomData[2] = pos[2];
427	gezelter	1490
428	gezelter	2204	atomData[3] = vel[0];
429			atomData[4] = vel[1];
430			atomData[5] = vel[2];
431	gezelter	1490
432	gezelter	2204	isDirectional = 0;
433	gezelter	1490
434	gezelter	2204	if (integrableObject->isDirectional()) {
435			isDirectional = 1;
436	gezelter	1490
437	gezelter	2204	q = integrableObject->getQ();
438			ji = integrableObject->getJ();
439	gezelter	1490
440	gezelter	2204	for(int j = 0; j < 6; j++) {
441			atomData[j] = atomData[j];
442			}
443	gezelter	1490
444	gezelter	2204	atomData[6] = q[0];
445			atomData[7] = q[1];
446			atomData[8] = q[2];
447			atomData[9] = q[3];
448	gezelter	1490
449	gezelter	2204	atomData[10] = ji[0];
450			atomData[11] = ji[1];
451			atomData[12] = ji[2];
452			}
453	gezelter	1490
454	gezelter	2204	// If we've survived to here, format the line:
455	gezelter	1490
456	gezelter	2204	if (!isDirectional) {
457			sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
458			integrableObject->getType().c_str(), atomData[0],
459			atomData[1], atomData[2],
460			atomData[3], atomData[4],
461			atomData[5]);
462	gezelter	1490
463	gezelter	2204	strcat(writeLine,
464			"0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
465			} else {
466			sprintf(writeLine,
467			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
468			integrableObject->getType().c_str(),
469			atomData[0],
470			atomData[1],
471			atomData[2],
472			atomData[3],
473			atomData[4],
474			atomData[5],
475			atomData[6],
476			atomData[7],
477			atomData[8],
478			atomData[9],
479			atomData[10],
480			atomData[11],
481			atomData[12]);
482			}
483	gezelter	1490
484
485	gezelter	2204	os << writeLine;
486	gezelter	1490
487	gezelter	2204	} //end for(iter = integrableObject.begin())
488			}
489			} //end for(i = 0; i < mpiSim->getNmol())
490	gezelter	1490
491	gezelter	2204	os.flush();
492	tim	1935
493	gezelter	2204	sprintf(checkPointMsg, "Sucessfully took a dump.\n");
494			MPIcheckPoint();
495	gezelter	1490
496	gezelter	2204	delete [] potatoes;
497	gezelter	1930	} else {
498	gezelter	1490
499	gezelter	2204	// worldRank != 0, so I'm a remote node.
500	gezelter	1490
501	gezelter	2204	// Set my magic potato to 0:
502	gezelter	1490
503	gezelter	2204	myPotato = 0;
504	gezelter	1490
505	gezelter	2204	for(int i = 0; i < info_->getNGlobalMolecules(); i++) {
506	gezelter	1490
507	gezelter	2204	// Am I the node which has this integrableObject?
508			int whichNode = info_->getMolToProc(i);
509			if (whichNode == worldRank) {
510			if (myPotato + 1 >= MAXTAG) {
511	gezelter	1490
512	gezelter	2204	// The potato was going to exceed the maximum value,
513			// so wrap this processor potato back to 0 (and block until
514			// node 0 says we can go:
515	gezelter	1490
516	gezelter	2204	MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD,
517			&istatus);
518			}
519	gezelter	1490
520	gezelter	2204	mol = info_->getMoleculeByGlobalIndex(i);
521	gezelter	1930
522	gezelter	1490
523	gezelter	2204	nCurObj = mol->getNIntegrableObjects();
524	gezelter	1490
525	gezelter	2204	MPI_Send(&nCurObj, 1, MPI_INT, 0, myPotato, MPI_COMM_WORLD);
526			myPotato++;
527	gezelter	1490
528	gezelter	2204	for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL;
529			integrableObject = mol->nextIntegrableObject(ii)) {
530	gezelter	1490
531	gezelter	2204	if (myPotato + 2 >= MAXTAG) {
532	gezelter	1490
533	gezelter	2204	// The potato was going to exceed the maximum value,
534			// so wrap this processor potato back to 0 (and block until
535			// node 0 says we can go:
536	gezelter	1490
537	gezelter	2204	MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD,
538			&istatus);
539			}
540	gezelter	1490
541	gezelter	2204	pos = integrableObject->getPos();
542			vel = integrableObject->getVel();
543	gezelter	1490
544	gezelter	2204	atomData[0] = pos[0];
545			atomData[1] = pos[1];
546			atomData[2] = pos[2];
547	gezelter	1490
548	gezelter	2204	atomData[3] = vel[0];
549			atomData[4] = vel[1];
550			atomData[5] = vel[2];
551	gezelter	1490
552	gezelter	2204	isDirectional = 0;
553	gezelter	1490
554	gezelter	2204	if (integrableObject->isDirectional()) {
555			isDirectional = 1;
556	gezelter	1490
557	gezelter	2204	q = integrableObject->getQ();
558			ji = integrableObject->getJ();
559	gezelter	1490
560	gezelter	2204	atomData[6] = q[0];
561			atomData[7] = q[1];
562			atomData[8] = q[2];
563			atomData[9] = q[3];
564	gezelter	1930
565	gezelter	2204	atomData[10] = ji[0];
566			atomData[11] = ji[1];
567			atomData[12] = ji[2];
568			}
569	gezelter	1930
570	gezelter	2204	strncpy(MPIatomTypeString, integrableObject->getType().c_str(), MINIBUFFERSIZE);
571	gezelter	1930
572	gezelter	2204	// null terminate the std::string before sending (just in case):
573			MPIatomTypeString[MINIBUFFERSIZE - 1] = '\0';
574	gezelter	1930
575	gezelter	2204	MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
576			myPotato, MPI_COMM_WORLD);
577	gezelter	1930
578	gezelter	2204	myPotato++;
579	gezelter	1930
580	gezelter	2204	if (isDirectional) {
581			MPI_Send(atomData, 13, MPI_DOUBLE, 0, myPotato,
582			MPI_COMM_WORLD);
583			} else {
584			MPI_Send(atomData, 6, MPI_DOUBLE, 0, myPotato,
585			MPI_COMM_WORLD);
586			}
587	gezelter	1930
588	gezelter	2204	myPotato++;
589			}
590	gezelter	1930
591	gezelter	2204	}
592	gezelter	1930
593	gezelter	2204	}
594			sprintf(checkPointMsg, "Sucessfully took a dump.\n");
595			MPIcheckPoint();
596	gezelter	1930	}
597
598			#endif // is_mpi
599
600	gezelter	2204	}
601	gezelter	1490
602	gezelter	2204	void DumpWriter::writeDump() {
603	tim	2314	writeFrame(*dumpFile_);
604	gezelter	2204	}
605	tim	2060
606	gezelter	2204	void DumpWriter::writeEor() {
607	tim	2314	std::ostream* eorStream;
608	tim	2060
609			#ifdef IS_MPI
610			if (worldRank == 0) {
611			#endif // is_mpi
612
613	tim	2314	eorStream = createOStream(eorFilename_);
614	tim	2060
615			#ifdef IS_MPI
616			}
617			#endif // is_mpi
618
619	tim	2314	writeFrame(*eorStream);
620
621			#ifdef IS_MPI
622			if (worldRank == 0) {
623			#endif // is_mpi
624			delete eorStream;
625
626			#ifdef IS_MPI
627			}
628			#endif // is_mpi
629
630	gezelter	2204	}
631	tim	2060
632
633	gezelter	2204	void DumpWriter::writeDumpAndEor() {
634	tim	2060	std::vector<std::streambuf*> buffers;
635	tim	2314	std::ostream* eorStream;
636	tim	2060	#ifdef IS_MPI
637			if (worldRank == 0) {
638			#endif // is_mpi
639
640	tim	2314	buffers.push_back(dumpFile_->rdbuf());
641	tim	2060
642	tim	2314	eorStream = createOStream(eorFilename_);
643	tim	2060
644	tim	2314	buffers.push_back(eorStream->rdbuf());
645	tim	2060
646			#ifdef IS_MPI
647			}
648			#endif // is_mpi
649
650			TeeBuf tbuf(buffers.begin(), buffers.end());
651			std::ostream os(&tbuf);
652
653			writeFrame(os);
654	tim	2314
655			#ifdef IS_MPI
656			if (worldRank == 0) {
657			#endif // is_mpi
658			delete eorStream;
659
660			#ifdef IS_MPI
661			}
662			#endif // is_mpi
663	tim	2060
664	gezelter	2204	}
665	tim	2060
666	tim	2314	std::ostream* DumpWriter::createOStream(const std::string& filename) {
667	tim	2318
668	tim	2314	std::ostream* newOStream;
669	tim	2318	#ifdef HAVE_LIBZ
670	tim	2314	if (needCompression_) {
671			newOStream = new ogzstream(filename.c_str());
672			} else {
673			newOStream = new std::ofstream(filename.c_str());
674			}
675	tim	2318	#else
676			newOStream = new std::ofstream(filename.c_str());
677			#endif
678	tim	2314	return newOStream;
679			}
680	tim	2060
681	gezelter	1930	}//end namespace oopse