src/io/DumpWriter.cpp

#define _LARGEFILE_SOURCE64
#define _FILE_OFFSET_BITS 64

#include <string.h>
#include <iostream>
#include <fstream>
#include <algorithm>
#include <utility>

#ifdef IS_MPI
#include <mpi.h>
#include "brains/mpiSimulation.hpp"

namespace dWrite{
  void DieDieDie( void );
}

using namespace dWrite;
#endif //is_mpi

#include "io/ReadWrite.hpp"
#include "utils/simError.h"

DumpWriter::DumpWriter( SimInfo* the_entry_plug ){

  entry_plug = the_entry_plug;

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif // is_mpi

    dumpFile.open(entry_plug->sampleName.c_str(), ios::out | ios::trunc );

    if( !dumpFile ){

      sprintf( painCave.errMsg,
               "Could not open \"%s\" for dump output.\n",
               entry_plug->sampleName.c_str());
      painCave.isFatal = 1;
      simError();
    }

#ifdef IS_MPI
  }

  //sort the local atoms by global index
  sortByGlobalIndex();
  
  sprintf( checkPointMsg,
           "Sucessfully opened output file for dumping.\n");
  MPIcheckPoint();
#endif // is_mpi
}

DumpWriter::~DumpWriter( ){

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif // is_mpi

    dumpFile.close();

#ifdef IS_MPI
  }
#endif // is_mpi
}

#ifdef IS_MPI

/**
 * A hook function to load balancing
 */

void DumpWriter::update(){
  sortByGlobalIndex();          
}
  
/**
 * Auxiliary sorting function
 */
 
bool indexSortingCriterion(const pair<int, int>& p1, const pair<int, int>& p2){
  return p1.second < p2.second;
}

/**
 * Sorting the local index by global index
 */
 
void DumpWriter::sortByGlobalIndex(){
  Molecule* mols = entry_plug->molecules;  
  indexArray.clear();
  
  for(int i = 0; i < entry_plug->n_mol;i++) 
    indexArray.push_back(make_pair(i, mols[i].getGlobalIndex()));
  
  sort(indexArray.begin(), indexArray.end(), indexSortingCriterion);    
}

#endif

void DumpWriter::writeDump(double currentTime){

  ofstream finalOut;
  vector<ofstream*> fileStreams;

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif    
    finalOut.open( entry_plug->finalName.c_str(), ios::out | ios::trunc );
    if( !finalOut ){
      sprintf( painCave.errMsg,
               "Could not open \"%s\" for final dump output.\n",
               entry_plug->finalName.c_str() );
      painCave.isFatal = 1;
      simError();
    }
#ifdef IS_MPI
  }
#endif // is_mpi

  fileStreams.push_back(&finalOut); 
  fileStreams.push_back(&dumpFile);

  writeFrame(fileStreams, currentTime);

#ifdef IS_MPI
  finalOut.close();
#endif
        
}

void DumpWriter::writeFinal(double currentTime){

  ofstream finalOut;
  vector<ofstream*> fileStreams;

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif // is_mpi

    finalOut.open( entry_plug->finalName.c_str(), ios::out | ios::trunc );

    if( !finalOut ){
      sprintf( painCave.errMsg,
               "Could not open \"%s\" for final dump output.\n",
               entry_plug->finalName.c_str() );
      painCave.isFatal = 1;
      simError();
    }

#ifdef IS_MPI
  }
#endif // is_mpi
  
  fileStreams.push_back(&finalOut);  
  writeFrame(fileStreams, currentTime);

#ifdef IS_MPI
  finalOut.close();
#endif
  
}

void DumpWriter::writeFrame( vector<ofstream*>& outFile, double currentTime ){

  const int BUFFERSIZE = 2000;
  const int MINIBUFFERSIZE = 100;

  char tempBuffer[BUFFERSIZE];  
  char writeLine[BUFFERSIZE];

  int i;
  unsigned int k;

#ifdef IS_MPI
  
  /*********************************************************************
   * Documentation?  You want DOCUMENTATION?
   * 
   * Why all the potatoes below?  
   *
   * To make a long story short, the original version of DumpWriter
   * worked in the most inefficient way possible.  Node 0 would 
   * poke each of the node for an individual atom's formatted data 
   * as node 0 worked its way down the global index. This was particularly 
   * inefficient since the method blocked all processors at every atom 
   * (and did it twice!).
   *
   * An intermediate version of DumpWriter could be described from Node
   * zero's perspective as follows:
   * 
   *  1) Have 100 of your friends stand in a circle.
   *  2) When you say go, have all of them start tossing potatoes at
   *     you (one at a time).
   *  3) Catch the potatoes.
   *
   * It was an improvement, but MPI has buffers and caches that could 
   * best be described in this analogy as "potato nets", so there's no 
   * need to block the processors atom-by-atom.
   * 
   * This new and improved DumpWriter works in an even more efficient 
   * way:
   * 
   *  1) Have 100 of your friend stand in a circle.
   *  2) When you say go, have them start tossing 5-pound bags of 
   *     potatoes at you.
   *  3) Once you've caught a friend's bag of potatoes,
   *     toss them a spud to let them know they can toss another bag.
   *
   * How's THAT for documentation?
   *
   *********************************************************************/

  int *potatoes;
  int myPotato;

  int nProc;
  int j, which_node, done, which_atom, local_index, currentIndex;
  double atomData[13];
  int isDirectional;
  char* atomTypeString;
  char MPIatomTypeString[MINIBUFFERSIZE];
  int nObjects;
  int msgLen; // the length of message actually recieved at master nodes
#endif //is_mpi

  double q[4], ji[3];
  DirectionalAtom* dAtom;
  double pos[3], vel[3];
  int nTotObjects;
  StuntDouble* sd;
  char* molName;
  vector<StuntDouble*> integrableObjects;
  vector<StuntDouble*>::iterator iter;
  nTotObjects = entry_plug->getTotIntegrableObjects();
#ifndef IS_MPI
  
  for(k = 0; k < outFile.size(); k++){
    *outFile[k] << nTotObjects << "\n";

    *outFile[k] << currentTime << ";\t"
               << entry_plug->Hmat[0][0] << "\t"
                     << entry_plug->Hmat[1][0] << "\t"
                     << entry_plug->Hmat[2][0] << ";\t"
               
               << entry_plug->Hmat[0][1] << "\t"
                     << entry_plug->Hmat[1][1] << "\t"
                     << entry_plug->Hmat[2][1] << ";\t"

                     << entry_plug->Hmat[0][2] << "\t"
                     << entry_plug->Hmat[1][2] << "\t"
                     << entry_plug->Hmat[2][2] << ";";

    //write out additional parameters, such as chi and eta
    *outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
  }
  
  for( i=0; i< entry_plug->n_mol; i++ ){

    integrableObjects = entry_plug->molecules[i].getIntegrableObjects();
    molName = (entry_plug->compStamps[entry_plug->molecules[i].getStampID()])->getID();
    
    for( iter = integrableObjects.begin();iter !=  integrableObjects.end(); ++iter){
      sd = *iter;
      sd->getPos(pos);
      sd->getVel(vel);

      sprintf( tempBuffer,
             "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
             sd->getType(),
             pos[0],
             pos[1],
             pos[2],
             vel[0],
             vel[1],
             vel[2]);
      strcpy( writeLine, tempBuffer );

      if( sd->isDirectional() ){

        sd->getQ( q );
        sd->getJ( ji );

        sprintf( tempBuffer,
               "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
               q[0],
               q[1],
               q[2],
               q[3],
                 ji[0],
                 ji[1],
                 ji[2]);
        strcat( writeLine, tempBuffer );
      }
      else
        strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
    
      for(k = 0; k < outFile.size(); k++)
        *outFile[k] << writeLine;      
    }
    
  }

#else // is_mpi

  /* code to find maximum tag value */
  
  int *tagub, flag, MAXTAG;
  MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);
  if (flag) {
    MAXTAG = *tagub;
  } else {
    MAXTAG = 32767;
  }  

  int haveError;

  MPI_Status istatus;
  int nCurObj;
  int *MolToProcMap = mpiSim->getMolToProcMap();

  // write out header and node 0's coordinates

  if( worldRank == 0 ){

    // Node 0 needs a list of the magic potatoes for each processor;

    nProc = mpiSim->getNProcessors();
    potatoes = new int[nProc];

    //write out the comment lines
    for (i = 0; i < nProc; i++) 
      potatoes[i] = 0;
    
    for(k = 0; k < outFile.size(); k++){
      *outFile[k] << nTotObjects << "\n";
      
      *outFile[k] << currentTime << ";\t"
                  << entry_plug->Hmat[0][0] << "\t"
                  << entry_plug->Hmat[1][0] << "\t"
                  << entry_plug->Hmat[2][0] << ";\t"
        
                  << entry_plug->Hmat[0][1] << "\t"
                  << entry_plug->Hmat[1][1] << "\t"
                  << entry_plug->Hmat[2][1] << ";\t"
        
                  << entry_plug->Hmat[0][2] << "\t"
                  << entry_plug->Hmat[1][2] << "\t"
                  << entry_plug->Hmat[2][2] << ";";
      
      *outFile[k] << entry_plug->the_integrator->getAdditionalParameters() 
                  << endl;
    }
    
    currentIndex = 0;
    
    for (i = 0 ; i < mpiSim->getNMolGlobal(); i++ ) {
      
      // Get the Node number which has this atom;
      
      which_node = MolToProcMap[i];
      
      if (which_node != 0) {
        
        if (potatoes[which_node] + 1 >= MAXTAG) {
          // The potato was going to exceed the maximum value, 
          // so wrap this processor potato back to 0:         

          potatoes[which_node] = 0;          
          MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0, 
                   MPI_COMM_WORLD);
          
        }

        myPotato = potatoes[which_node];        

        //recieve the number of integrableObject in current molecule
        MPI_Recv(&nCurObj, 1, MPI_INT, which_node,
                 myPotato, MPI_COMM_WORLD, &istatus);
        myPotato++;
        
        for(int l = 0; l < nCurObj; l++){

          if (potatoes[which_node] + 2 >= MAXTAG) {
            // The potato was going to exceed the maximum value, 
            // so wrap this processor potato back to 0:         

            potatoes[which_node] = 0;          
            MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0, MPI_COMM_WORLD);
            
          }

          MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, which_node,
          myPotato, MPI_COMM_WORLD, &istatus);

          atomTypeString = MPIatomTypeString;

          myPotato++;

          MPI_Recv(atomData, 13, MPI_DOUBLE, which_node, myPotato, MPI_COMM_WORLD, &istatus);
          myPotato++;

          MPI_Get_count(&istatus, MPI_DOUBLE, &msgLen);

          if(msgLen  == 13) 
            isDirectional = 1;
          else
            isDirectional = 0;
          
          // If we've survived to here, format the line:
            
          if (!isDirectional) {
        
            sprintf( writeLine,
                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                 atomTypeString,
                 atomData[0],
                 atomData[1],
                 atomData[2],
                 atomData[3],
                 atomData[4],
                 atomData[5]);
        
           strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
        
          } 
          else {
        
                sprintf( writeLine,
                         "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
                         atomTypeString,
                         atomData[0],
                         atomData[1],
                         atomData[2],
                         atomData[3],
                         atomData[4],
                         atomData[5],
                         atomData[6],
                         atomData[7],
                         atomData[8],
                         atomData[9],
                         atomData[10],
                         atomData[11],
                         atomData[12]);
            
          }
          
          for(k = 0; k < outFile.size(); k++)
            *outFile[k] << writeLine;            

        }// end for(int l =0)
        potatoes[which_node] = myPotato;
        
      }
      else {
        
        haveError = 0;
        
            local_index = indexArray[currentIndex].first;        
            
            integrableObjects = (entry_plug->molecules[local_index]).getIntegrableObjects(); 

        for(iter= integrableObjects.begin(); iter != integrableObjects.end(); ++iter){    
                sd = *iter;
            atomTypeString = sd->getType();
            
            sd->getPos(pos);
            sd->getVel(vel);          
          
            atomData[0] = pos[0];
            atomData[1] = pos[1];
            atomData[2] = pos[2];

            atomData[3] = vel[0];
            atomData[4] = vel[1];
            atomData[5] = vel[2];
              
            isDirectional = 0;

            if( sd->isDirectional() ){

              isDirectional = 1;
                
              sd->getQ( q );
              sd->getJ( ji );

              for (int j = 0; j < 6 ; j++)
                atomData[j] = atomData[j];            
              
              atomData[6] = q[0];
              atomData[7] = q[1];
              atomData[8] = q[2];
              atomData[9] = q[3];
              
              atomData[10] = ji[0];
              atomData[11] = ji[1];
              atomData[12] = ji[2];
            }
            
            // If we've survived to here, format the line:
            
            if (!isDirectional) {
        
              sprintf( writeLine,
                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                 atomTypeString,
                 atomData[0],
                 atomData[1],
                 atomData[2],
                 atomData[3],
                 atomData[4],
                 atomData[5]);
        
             strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
        
            } 
            else {
        
                sprintf( writeLine,
                         "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
                         atomTypeString,
                         atomData[0],
                         atomData[1],
                         atomData[2],
                         atomData[3],
                         atomData[4],
                         atomData[5],
                         atomData[6],
                         atomData[7],
                         atomData[8],
                         atomData[9],
                         atomData[10],
                         atomData[11],
                         atomData[12]);
              
            }
            
            for(k = 0; k < outFile.size(); k++)
              *outFile[k] << writeLine;
            
            
        }//end for(iter = integrableObject.begin())
        
      currentIndex++;
      }

    }//end for(i = 0; i < mpiSim->getNmol())
    
    for(k = 0; k < outFile.size(); k++)
      outFile[k]->flush();
    
    sprintf( checkPointMsg,
             "Sucessfully took a dump.\n");
    
    MPIcheckPoint();        
    
    delete[] potatoes;
    
  } else {

    // worldRank != 0, so I'm a remote node.  

    // Set my magic potato to 0:

    myPotato = 0;
    currentIndex = 0;
    
    for (i = 0 ; i < mpiSim->getNMolGlobal(); i++ ) {
      
      // Am I the node which has this integrableObject?
      
      if (MolToProcMap[i] == worldRank) {


        if (myPotato + 1 >= MAXTAG) {
          
          // The potato was going to exceed the maximum value, 
          // so wrap this processor potato back to 0 (and block until
          // node 0 says we can go:
          
          MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus);
          
        }

          local_index = indexArray[currentIndex].first;        
          integrableObjects = entry_plug->molecules[local_index].getIntegrableObjects();
          
          nCurObj = integrableObjects.size();
                      
          MPI_Send(&nCurObj, 1, MPI_INT, 0,
                   myPotato, MPI_COMM_WORLD);
          myPotato++;

          for( iter = integrableObjects.begin(); iter  != integrableObjects.end(); iter++){

            if (myPotato + 2 >= MAXTAG) {
          
              // The potato was going to exceed the maximum value, 
              // so wrap this processor potato back to 0 (and block until
              // node 0 says we can go:
          
              MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus);
              
            }
            
            sd = *iter;
            
            atomTypeString = sd->getType();

            sd->getPos(pos);
            sd->getVel(vel);

            atomData[0] = pos[0];
            atomData[1] = pos[1];
            atomData[2] = pos[2];

            atomData[3] = vel[0];
            atomData[4] = vel[1];
            atomData[5] = vel[2];
              
            isDirectional = 0;

            if( sd->isDirectional() ){

                isDirectional = 1;
                
                sd->getQ( q );
                sd->getJ( ji );
                
                
                atomData[6] = q[0];
                atomData[7] = q[1];
                atomData[8] = q[2];
                atomData[9] = q[3];
      
                atomData[10] = ji[0];
                atomData[11] = ji[1];
                atomData[12] = ji[2];
              }

             
            strncpy(MPIatomTypeString, atomTypeString, MINIBUFFERSIZE);

            // null terminate the string before sending (just in case):
            MPIatomTypeString[MINIBUFFERSIZE-1] = '\0';

            MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
                             myPotato, MPI_COMM_WORLD);
            
            myPotato++;
            
            if (isDirectional) {

              MPI_Send(atomData, 13, MPI_DOUBLE, 0,
                       myPotato, MPI_COMM_WORLD);
              
            } else {

              MPI_Send(atomData, 6, MPI_DOUBLE, 0,
                       myPotato, MPI_COMM_WORLD);
            }

            myPotato++;  

          }

          currentIndex++;    
          
        }
      
      }

    sprintf( checkPointMsg,
             "Successfully took a dump.\n");
    MPIcheckPoint();                
    
  }
  
#endif // is_mpi
}

#ifdef IS_MPI

// a couple of functions to let us escape the write loop

void dWrite::DieDieDie( void ){

  MPI_Finalize();
  exit (0);
}

#endif //is_mpi
Revision:	1772
Committed:	Tue Nov 23 22:48:31 2004 UTC (19 years, 7 months ago) by chrisfen
File size:	17366 byte(s)
Log Message:	Improvements to restraints
#	User	Rev	Content
1	gezelter	1490	#define _LARGEFILE_SOURCE64
2			#define _FILE_OFFSET_BITS 64
3
4			#include <string.h>
5			#include <iostream>
6			#include <fstream>
7			#include <algorithm>
8			#include <utility>
9
10			#ifdef IS_MPI
11			#include <mpi.h>
12	tim	1492	#include "brains/mpiSimulation.hpp"
13	gezelter	1490
14			namespace dWrite{
15			void DieDieDie( void );
16			}
17
18			using namespace dWrite;
19			#endif //is_mpi
20
21	tim	1492	#include "io/ReadWrite.hpp"
22			#include "utils/simError.h"
23	gezelter	1490
24			DumpWriter::DumpWriter( SimInfo* the_entry_plug ){
25
26			entry_plug = the_entry_plug;
27
28			#ifdef IS_MPI
29			if(worldRank == 0 ){
30			#endif // is_mpi
31
32			dumpFile.open(entry_plug->sampleName.c_str(), ios::out \| ios::trunc );
33
34			if( !dumpFile ){
35
36			sprintf( painCave.errMsg,
37			"Could not open \"%s\" for dump output.\n",
38			entry_plug->sampleName.c_str());
39			painCave.isFatal = 1;
40			simError();
41			}
42
43			#ifdef IS_MPI
44			}
45
46			//sort the local atoms by global index
47			sortByGlobalIndex();
48
49			sprintf( checkPointMsg,
50			"Sucessfully opened output file for dumping.\n");
51			MPIcheckPoint();
52			#endif // is_mpi
53			}
54
55			DumpWriter::~DumpWriter( ){
56
57			#ifdef IS_MPI
58			if(worldRank == 0 ){
59			#endif // is_mpi
60
61			dumpFile.close();
62
63			#ifdef IS_MPI
64			}
65			#endif // is_mpi
66			}
67
68			#ifdef IS_MPI
69
70			/**
71			* A hook function to load balancing
72			*/
73
74			void DumpWriter::update(){
75			sortByGlobalIndex();
76			}
77
78			/**
79			* Auxiliary sorting function
80			*/
81
82			bool indexSortingCriterion(const pair<int, int>& p1, const pair<int, int>& p2){
83			return p1.second < p2.second;
84			}
85
86			/**
87			* Sorting the local index by global index
88			*/
89
90			void DumpWriter::sortByGlobalIndex(){
91			Molecule* mols = entry_plug->molecules;
92			indexArray.clear();
93
94			for(int i = 0; i < entry_plug->n_mol;i++)
95			indexArray.push_back(make_pair(i, mols[i].getGlobalIndex()));
96
97			sort(indexArray.begin(), indexArray.end(), indexSortingCriterion);
98			}
99
100			#endif
101
102			void DumpWriter::writeDump(double currentTime){
103
104			ofstream finalOut;
105			vector<ofstream*> fileStreams;
106
107			#ifdef IS_MPI
108			if(worldRank == 0 ){
109			#endif
110			finalOut.open( entry_plug->finalName.c_str(), ios::out \| ios::trunc );
111			if( !finalOut ){
112			sprintf( painCave.errMsg,
113			"Could not open \"%s\" for final dump output.\n",
114			entry_plug->finalName.c_str() );
115			painCave.isFatal = 1;
116			simError();
117			}
118			#ifdef IS_MPI
119			}
120			#endif // is_mpi
121
122			fileStreams.push_back(&finalOut);
123			fileStreams.push_back(&dumpFile);
124
125			writeFrame(fileStreams, currentTime);
126
127			#ifdef IS_MPI
128			finalOut.close();
129			#endif
130
131			}
132
133			void DumpWriter::writeFinal(double currentTime){
134
135			ofstream finalOut;
136			vector<ofstream*> fileStreams;
137
138			#ifdef IS_MPI
139			if(worldRank == 0 ){
140			#endif // is_mpi
141
142			finalOut.open( entry_plug->finalName.c_str(), ios::out \| ios::trunc );
143
144			if( !finalOut ){
145			sprintf( painCave.errMsg,
146			"Could not open \"%s\" for final dump output.\n",
147			entry_plug->finalName.c_str() );
148			painCave.isFatal = 1;
149			simError();
150			}
151
152			#ifdef IS_MPI
153			}
154			#endif // is_mpi
155
156			fileStreams.push_back(&finalOut);
157			writeFrame(fileStreams, currentTime);
158
159			#ifdef IS_MPI
160			finalOut.close();
161			#endif
162
163			}
164
165			void DumpWriter::writeFrame( vector<ofstream*>& outFile, double currentTime ){
166
167			const int BUFFERSIZE = 2000;
168			const int MINIBUFFERSIZE = 100;
169
170			char tempBuffer[BUFFERSIZE];
171			char writeLine[BUFFERSIZE];
172
173			int i;
174			unsigned int k;
175
176			#ifdef IS_MPI
177
178			/*********************************************************************
179			* Documentation? You want DOCUMENTATION?
180			*
181			* Why all the potatoes below?
182			*
183			* To make a long story short, the original version of DumpWriter
184			* worked in the most inefficient way possible. Node 0 would
185			* poke each of the node for an individual atom's formatted data
186			* as node 0 worked its way down the global index. This was particularly
187			* inefficient since the method blocked all processors at every atom
188			* (and did it twice!).
189			*
190			* An intermediate version of DumpWriter could be described from Node
191			* zero's perspective as follows:
192			*
193			* 1) Have 100 of your friends stand in a circle.
194			* 2) When you say go, have all of them start tossing potatoes at
195			* you (one at a time).
196			* 3) Catch the potatoes.
197			*
198			* It was an improvement, but MPI has buffers and caches that could
199			* best be described in this analogy as "potato nets", so there's no
200			* need to block the processors atom-by-atom.
201			*
202			* This new and improved DumpWriter works in an even more efficient
203			* way:
204			*
205			* 1) Have 100 of your friend stand in a circle.
206			* 2) When you say go, have them start tossing 5-pound bags of
207			* potatoes at you.
208			* 3) Once you've caught a friend's bag of potatoes,
209			* toss them a spud to let them know they can toss another bag.
210			*
211			* How's THAT for documentation?
212			*
213			*********************************************************************/
214
215			int *potatoes;
216			int myPotato;
217
218			int nProc;
219			int j, which_node, done, which_atom, local_index, currentIndex;
220			double atomData[13];
221			int isDirectional;
222			char* atomTypeString;
223			char MPIatomTypeString[MINIBUFFERSIZE];
224			int nObjects;
225			int msgLen; // the length of message actually recieved at master nodes
226			#endif //is_mpi
227
228			double q[4], ji[3];
229			DirectionalAtom* dAtom;
230			double pos[3], vel[3];
231			int nTotObjects;
232			StuntDouble* sd;
233			char* molName;
234			vector<StuntDouble*> integrableObjects;
235			vector<StuntDouble*>::iterator iter;
236			nTotObjects = entry_plug->getTotIntegrableObjects();
237			#ifndef IS_MPI
238
239			for(k = 0; k < outFile.size(); k++){
240			*outFile[k] << nTotObjects << "\n";
241
242			*outFile[k] << currentTime << ";\t"
243			<< entry_plug->Hmat[0][0] << "\t"
244			<< entry_plug->Hmat[1][0] << "\t"
245			<< entry_plug->Hmat[2][0] << ";\t"
246
247			<< entry_plug->Hmat[0][1] << "\t"
248			<< entry_plug->Hmat[1][1] << "\t"
249			<< entry_plug->Hmat[2][1] << ";\t"
250
251			<< entry_plug->Hmat[0][2] << "\t"
252			<< entry_plug->Hmat[1][2] << "\t"
253			<< entry_plug->Hmat[2][2] << ";";
254
255			//write out additional parameters, such as chi and eta
256			*outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
257			}
258
259			for( i=0; i< entry_plug->n_mol; i++ ){
260
261			integrableObjects = entry_plug->molecules[i].getIntegrableObjects();
262			molName = (entry_plug->compStamps[entry_plug->molecules[i].getStampID()])->getID();
263
264			for( iter = integrableObjects.begin();iter != integrableObjects.end(); ++iter){
265			sd = *iter;
266			sd->getPos(pos);
267			sd->getVel(vel);
268
269			sprintf( tempBuffer,
270			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
271			sd->getType(),
272			pos[0],
273			pos[1],
274			pos[2],
275			vel[0],
276			vel[1],
277			vel[2]);
278			strcpy( writeLine, tempBuffer );
279
280			if( sd->isDirectional() ){
281
282			sd->getQ( q );
283			sd->getJ( ji );
284
285			sprintf( tempBuffer,
286			"%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
287			q[0],
288			q[1],
289			q[2],
290			q[3],
291			ji[0],
292			ji[1],
293			ji[2]);
294			strcat( writeLine, tempBuffer );
295			}
296			else
297			strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
298
299			for(k = 0; k < outFile.size(); k++)
300			*outFile[k] << writeLine;
301			}
302	chrisfen	1772
303			}
304	gezelter	1490
305			#else // is_mpi
306
307			/* code to find maximum tag value */
308
309			int *tagub, flag, MAXTAG;
310			MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);
311			if (flag) {
312			MAXTAG = *tagub;
313			} else {
314			MAXTAG = 32767;
315			}
316
317			int haveError;
318
319			MPI_Status istatus;
320			int nCurObj;
321			int *MolToProcMap = mpiSim->getMolToProcMap();
322
323			// write out header and node 0's coordinates
324
325			if( worldRank == 0 ){
326
327			// Node 0 needs a list of the magic potatoes for each processor;
328
329			nProc = mpiSim->getNProcessors();
330			potatoes = new int[nProc];
331
332			//write out the comment lines
333			for (i = 0; i < nProc; i++)
334			potatoes[i] = 0;
335
336	chrisfen	1772	for(k = 0; k < outFile.size(); k++){
337			*outFile[k] << nTotObjects << "\n";
338
339			*outFile[k] << currentTime << ";\t"
340			<< entry_plug->Hmat[0][0] << "\t"
341			<< entry_plug->Hmat[1][0] << "\t"
342			<< entry_plug->Hmat[2][0] << ";\t"
343
344			<< entry_plug->Hmat[0][1] << "\t"
345			<< entry_plug->Hmat[1][1] << "\t"
346			<< entry_plug->Hmat[2][1] << ";\t"
347
348			<< entry_plug->Hmat[0][2] << "\t"
349			<< entry_plug->Hmat[1][2] << "\t"
350			<< entry_plug->Hmat[2][2] << ";";
351
352			*outFile[k] << entry_plug->the_integrator->getAdditionalParameters()
353			<< endl;
354	gezelter	1490	}
355	chrisfen	1772
356	gezelter	1490	currentIndex = 0;
357	chrisfen	1772
358	gezelter	1490	for (i = 0 ; i < mpiSim->getNMolGlobal(); i++ ) {
359
360			// Get the Node number which has this atom;
361
362			which_node = MolToProcMap[i];
363
364			if (which_node != 0) {
365
366			if (potatoes[which_node] + 1 >= MAXTAG) {
367			// The potato was going to exceed the maximum value,
368			// so wrap this processor potato back to 0:
369
370			potatoes[which_node] = 0;
371	chrisfen	1772	MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0,
372			MPI_COMM_WORLD);
373	gezelter	1490
374			}
375
376			myPotato = potatoes[which_node];
377
378			//recieve the number of integrableObject in current molecule
379			MPI_Recv(&nCurObj, 1, MPI_INT, which_node,
380			myPotato, MPI_COMM_WORLD, &istatus);
381			myPotato++;
382
383			for(int l = 0; l < nCurObj; l++){
384
385			if (potatoes[which_node] + 2 >= MAXTAG) {
386			// The potato was going to exceed the maximum value,
387			// so wrap this processor potato back to 0:
388
389			potatoes[which_node] = 0;
390			MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0, MPI_COMM_WORLD);
391
392			}
393
394			MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, which_node,
395			myPotato, MPI_COMM_WORLD, &istatus);
396
397			atomTypeString = MPIatomTypeString;
398
399			myPotato++;
400
401			MPI_Recv(atomData, 13, MPI_DOUBLE, which_node, myPotato, MPI_COMM_WORLD, &istatus);
402			myPotato++;
403
404			MPI_Get_count(&istatus, MPI_DOUBLE, &msgLen);
405
406			if(msgLen == 13)
407			isDirectional = 1;
408			else
409			isDirectional = 0;
410
411			// If we've survived to here, format the line:
412
413			if (!isDirectional) {
414
415			sprintf( writeLine,
416			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
417			atomTypeString,
418			atomData[0],
419			atomData[1],
420			atomData[2],
421			atomData[3],
422			atomData[4],
423			atomData[5]);
424
425			strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
426
427			}
428			else {
429
430			sprintf( writeLine,
431			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
432			atomTypeString,
433			atomData[0],
434			atomData[1],
435			atomData[2],
436			atomData[3],
437			atomData[4],
438			atomData[5],
439			atomData[6],
440			atomData[7],
441			atomData[8],
442			atomData[9],
443			atomData[10],
444			atomData[11],
445			atomData[12]);
446
447			}
448
449			for(k = 0; k < outFile.size(); k++)
450			*outFile[k] << writeLine;
451
452			}// end for(int l =0)
453			potatoes[which_node] = myPotato;
454	chrisfen	1772
455	gezelter	1490	}
456			else {
457	chrisfen	1772
458	gezelter	1490	haveError = 0;
459
460			local_index = indexArray[currentIndex].first;
461	chrisfen	1772
462			integrableObjects = (entry_plug->molecules[local_index]).getIntegrableObjects();
463	gezelter	1490
464			for(iter= integrableObjects.begin(); iter != integrableObjects.end(); ++iter){
465			sd = *iter;
466			atomTypeString = sd->getType();
467
468			sd->getPos(pos);
469			sd->getVel(vel);
470
471			atomData[0] = pos[0];
472			atomData[1] = pos[1];
473			atomData[2] = pos[2];
474
475			atomData[3] = vel[0];
476			atomData[4] = vel[1];
477			atomData[5] = vel[2];
478
479			isDirectional = 0;
480
481			if( sd->isDirectional() ){
482
483			isDirectional = 1;
484
485			sd->getQ( q );
486			sd->getJ( ji );
487
488			for (int j = 0; j < 6 ; j++)
489			atomData[j] = atomData[j];
490
491			atomData[6] = q[0];
492			atomData[7] = q[1];
493			atomData[8] = q[2];
494			atomData[9] = q[3];
495
496			atomData[10] = ji[0];
497			atomData[11] = ji[1];
498			atomData[12] = ji[2];
499			}
500
501			// If we've survived to here, format the line:
502
503			if (!isDirectional) {
504
505			sprintf( writeLine,
506			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
507			atomTypeString,
508			atomData[0],
509			atomData[1],
510			atomData[2],
511			atomData[3],
512			atomData[4],
513			atomData[5]);
514
515			strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
516
517			}
518			else {
519
520			sprintf( writeLine,
521			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
522			atomTypeString,
523			atomData[0],
524			atomData[1],
525			atomData[2],
526			atomData[3],
527			atomData[4],
528			atomData[5],
529			atomData[6],
530			atomData[7],
531			atomData[8],
532			atomData[9],
533			atomData[10],
534			atomData[11],
535			atomData[12]);
536
537			}
538
539			for(k = 0; k < outFile.size(); k++)
540			*outFile[k] << writeLine;
541
542
543			}//end for(iter = integrableObject.begin())
544
545			currentIndex++;
546			}
547
548			}//end for(i = 0; i < mpiSim->getNmol())
549
550			for(k = 0; k < outFile.size(); k++)
551			outFile[k]->flush();
552
553			sprintf( checkPointMsg,
554			"Sucessfully took a dump.\n");
555
556			MPIcheckPoint();
557
558			delete[] potatoes;
559
560			} else {
561
562			// worldRank != 0, so I'm a remote node.
563
564			// Set my magic potato to 0:
565
566			myPotato = 0;
567			currentIndex = 0;
568
569			for (i = 0 ; i < mpiSim->getNMolGlobal(); i++ ) {
570
571			// Am I the node which has this integrableObject?
572
573			if (MolToProcMap[i] == worldRank) {
574
575
576			if (myPotato + 1 >= MAXTAG) {
577
578			// The potato was going to exceed the maximum value,
579			// so wrap this processor potato back to 0 (and block until
580			// node 0 says we can go:
581
582			MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus);
583
584			}
585
586			local_index = indexArray[currentIndex].first;
587			integrableObjects = entry_plug->molecules[local_index].getIntegrableObjects();
588
589			nCurObj = integrableObjects.size();
590
591			MPI_Send(&nCurObj, 1, MPI_INT, 0,
592	chrisfen	1772	myPotato, MPI_COMM_WORLD);
593	gezelter	1490	myPotato++;
594
595			for( iter = integrableObjects.begin(); iter != integrableObjects.end(); iter++){
596
597			if (myPotato + 2 >= MAXTAG) {
598
599			// The potato was going to exceed the maximum value,
600			// so wrap this processor potato back to 0 (and block until
601			// node 0 says we can go:
602
603			MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus);
604
605			}
606
607			sd = *iter;
608
609			atomTypeString = sd->getType();
610
611			sd->getPos(pos);
612			sd->getVel(vel);
613
614			atomData[0] = pos[0];
615			atomData[1] = pos[1];
616			atomData[2] = pos[2];
617
618			atomData[3] = vel[0];
619			atomData[4] = vel[1];
620			atomData[5] = vel[2];
621
622			isDirectional = 0;
623
624			if( sd->isDirectional() ){
625
626			isDirectional = 1;
627
628			sd->getQ( q );
629			sd->getJ( ji );
630
631
632			atomData[6] = q[0];
633			atomData[7] = q[1];
634			atomData[8] = q[2];
635			atomData[9] = q[3];
636
637			atomData[10] = ji[0];
638			atomData[11] = ji[1];
639			atomData[12] = ji[2];
640			}
641
642
643			strncpy(MPIatomTypeString, atomTypeString, MINIBUFFERSIZE);
644
645			// null terminate the string before sending (just in case):
646			MPIatomTypeString[MINIBUFFERSIZE-1] = '\0';
647
648			MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
649			myPotato, MPI_COMM_WORLD);
650
651			myPotato++;
652
653			if (isDirectional) {
654
655			MPI_Send(atomData, 13, MPI_DOUBLE, 0,
656			myPotato, MPI_COMM_WORLD);
657
658			} else {
659
660			MPI_Send(atomData, 6, MPI_DOUBLE, 0,
661			myPotato, MPI_COMM_WORLD);
662			}
663
664			myPotato++;
665
666			}
667
668			currentIndex++;
669
670			}
671
672			}
673
674			sprintf( checkPointMsg,
675	chrisfen	1772	"Successfully took a dump.\n");
676	gezelter	1490	MPIcheckPoint();
677
678	chrisfen	1772	}
679	gezelter	1490
680			#endif // is_mpi
681			}
682
683			#ifdef IS_MPI
684
685			// a couple of functions to let us escape the write loop
686
687			void dWrite::DieDieDie( void ){
688
689			MPI_Finalize();
690			exit (0);
691			}
692
693			#endif //is_mpi