OOPSE/libmdtools/DumpWriter.cpp

#define _LARGEFILE_SOURCE64
#define _FILE_OFFSET_BITS 64

#include <string.h>
#include <iostream>
#include <fstream>
#include <algorithm>
#include <utility>

#ifdef IS_MPI
#include <mpi.h>
#include "mpiSimulation.hpp"

namespace dWrite{
  void DieDieDie( void );
}

using namespace dWrite;
#endif //is_mpi

#include "ReadWrite.hpp"
#include "simError.h"

DumpWriter::DumpWriter( SimInfo* the_entry_plug ){

  entry_plug = the_entry_plug;

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif // is_mpi

    dumpFile.open(entry_plug->sampleName, ios::out | ios::trunc );

    if( !dumpFile ){

      sprintf( painCave.errMsg,
               "Could not open \"%s\" for dump output.\n",
               entry_plug->sampleName);
      painCave.isFatal = 1;
      simError();
    }

#ifdef IS_MPI
  }

  //sort the local atoms by global index
  sortByGlobalIndex();
  
  sprintf( checkPointMsg,
           "Sucessfully opened output file for dumping.\n");
  MPIcheckPoint();
#endif // is_mpi
}

DumpWriter::~DumpWriter( ){

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif // is_mpi

    dumpFile.close();

#ifdef IS_MPI
  }
#endif // is_mpi
}

#ifdef IS_MPI

/**
 * A hook function to load balancing
 */

void DumpWriter::update(){
  sortByGlobalIndex();          
}
  
/**
 * Auxiliary sorting function
 */
 
bool indexSortingCriterion(const pair<int, int>& p1, const pair<int, int>& p2){
  return p1.second < p2.second;
}

/**
 * Sorting the local index by global index
 */
 
void DumpWriter::sortByGlobalIndex(){
  Atom** atoms = entry_plug->atoms;
  
  indexArray.clear();
  
  for(int i = 0; i < mpiSim->getMyNlocal();i++) 
    indexArray.push_back(make_pair(i, atoms[i]->getGlobalIndex()));
  
  sort(indexArray.begin(), indexArray.end(), indexSortingCriterion);    
}

#endif

void DumpWriter::writeDump(double currentTime){

  ofstream finalOut;
  vector<ofstream*> fileStreams;

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif    
    finalOut.open( entry_plug->finalName, ios::out | ios::trunc );
    if( !finalOut ){
      sprintf( painCave.errMsg,
               "Could not open \"%s\" for final dump output.\n",
               entry_plug->finalName );
      painCave.isFatal = 1;
      simError();
    }
#ifdef IS_MPI
  }
#endif // is_mpi

  fileStreams.push_back(&finalOut); 
  fileStreams.push_back(&dumpFile);

  writeFrame(fileStreams, currentTime);

#ifdef IS_MPI
  finalOut.close();
#endif
        
}

void DumpWriter::writeFinal(double currentTime){

  ofstream finalOut;
  vector<ofstream*> fileStreams;

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif // is_mpi

    finalOut.open( entry_plug->finalName, ios::out | ios::trunc );

    if( !finalOut ){
      sprintf( painCave.errMsg,
               "Could not open \"%s\" for final dump output.\n",
               entry_plug->finalName );
      painCave.isFatal = 1;
      simError();
    }

#ifdef IS_MPI
  }
#endif // is_mpi
  
  fileStreams.push_back(&finalOut);  
  writeFrame(fileStreams, currentTime);

#ifdef IS_MPI
  finalOut.close();
#endif
  
}

void DumpWriter::writeFrame( vector<ofstream*>& outFile, double currentTime ){

  const int BUFFERSIZE = 2000;
  const int MINIBUFFERSIZE = 100;

  char tempBuffer[BUFFERSIZE];  
  char writeLine[BUFFERSIZE];

  int i, k;

#ifdef IS_MPI
  
  /*********************************************************************
   * Documentation?  You want DOCUMENTATION?
   * 
   * Why all the potatoes below?  
   *
   * To make a long story short, the original version of DumpWriter
   * worked in the most inefficient way possible.  Node 0 would 
   * poke each of the node for an individual atom's formatted data 
   * as node 0 worked its way down the global index. This was particularly 
   * inefficient since the method blocked all processors at every atom 
   * (and did it twice!).
   *
   * An intermediate version of DumpWriter could be described from Node
   * zero's perspective as follows:
   * 
   *  1) Have 100 of your friends stand in a circle.
   *  2) When you say go, have all of them start tossing potatoes at
   *     you (one at a time).
   *  3) Catch the potatoes.
   *
   * It was an improvement, but MPI has buffers and caches that could 
   * best be described in this analogy as "potato nets", so there's no 
   * need to block the processors atom-by-atom.
   * 
   * This new and improved DumpWriter works in an even more efficient 
   * way:
   * 
   *  1) Have 100 of your friend stand in a circle.
   *  2) When you say go, have them start tossing 5-pound bags of 
   *     potatoes at you.
   *  3) Once you've caught a friend's bag of potatoes,
   *     toss them a spud to let them know they can toss another bag.
   *
   * How's THAT for documentation?
   *
   *********************************************************************/

  int *potatoes;
  int myPotato;

  int nProc;
  int j, which_node, done, which_atom, local_index, currentIndex;
  double atomData6[6];
  double atomData13[13];
  int isDirectional;
  char* atomTypeString;
  char MPIatomTypeString[MINIBUFFERSIZE];

#else //is_mpi
  int nAtoms = entry_plug->n_atoms;
#endif //is_mpi

  double q[4], ji[3];
  DirectionalAtom* dAtom;
  Atom** atoms = entry_plug->atoms;
  double pos[3], vel[3];

#ifndef IS_MPI
  
  for(k = 0; k < outFile.size(); k++){
    *outFile[k] << nAtoms << "\n";

    *outFile[k] << currentTime << ";\t"
               << entry_plug->Hmat[0][0] << "\t"
                     << entry_plug->Hmat[1][0] << "\t"
                     << entry_plug->Hmat[2][0] << ";\t"
               
               << entry_plug->Hmat[0][1] << "\t"
                     << entry_plug->Hmat[1][1] << "\t"
                     << entry_plug->Hmat[2][1] << ";\t"

                     << entry_plug->Hmat[0][2] << "\t"
                     << entry_plug->Hmat[1][2] << "\t"
                     << entry_plug->Hmat[2][2] << ";";

    //write out additional parameters, such as chi and eta
    *outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
  }
  
  for( i=0; i<nAtoms; i++ ){

    atoms[i]->getPos(pos);
    atoms[i]->getVel(vel);

    sprintf( tempBuffer,
             "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
             atoms[i]->getType(),
             pos[0],
             pos[1],
             pos[2],
             vel[0],
             vel[1],
             vel[2]);
    strcpy( writeLine, tempBuffer );

    if( atoms[i]->isDirectional() ){

      dAtom = (DirectionalAtom *)atoms[i];
      dAtom->getQ( q );
      dAtom->getJ( ji );

      sprintf( tempBuffer,
               "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
               q[0],
               q[1],
               q[2],
               q[3],
               ji[0],
               ji[1],
               ji[2]);
      strcat( writeLine, tempBuffer );
    }
    else
      strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );

    for(k = 0; k < outFile.size(); k++)
      *outFile[k] << writeLine;
  }

#else // is_mpi

  /* code to find maximum tag value */
  
  int *tagub, flag, MAXTAG;
  MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);
  if (flag) {
    MAXTAG = *tagub;
  } else {
    MAXTAG = 32767;
  }  

  int haveError;

  MPI_Status istatus;
  int *AtomToProcMap = mpiSim->getAtomToProcMap();

  // write out header and node 0's coordinates

  if( worldRank == 0 ){

    // Node 0 needs a list of the magic potatoes for each processor;

    nProc = mpiSim->getNumberProcessors();
    potatoes = new int[nProc];

    //write out the comment lines
    for (i = 0; i < nProc; i++) 
      potatoes[i] = 0;
    
      for(k = 0; k < outFile.size(); k++){
        *outFile[k] << mpiSim->getTotAtoms() << "\n";

        *outFile[k] << currentTime << ";\t"
                         << entry_plug->Hmat[0][0] << "\t"
                         << entry_plug->Hmat[1][0] << "\t"
                         << entry_plug->Hmat[2][0] << ";\t"

                         << entry_plug->Hmat[0][1] << "\t"
                         << entry_plug->Hmat[1][1] << "\t"
                         << entry_plug->Hmat[2][1] << ";\t"

                         << entry_plug->Hmat[0][2] << "\t"
                         << entry_plug->Hmat[1][2] << "\t"
                         << entry_plug->Hmat[2][2] << ";";
  
        *outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
    }

    currentIndex = 0;

    for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) {
      
      // Get the Node number which has this atom;
      
      which_node = AtomToProcMap[i];
      
      if (which_node != 0) {

        if (potatoes[which_node] + 3 >= MAXTAG) {
          // The potato was going to exceed the maximum value, 
          // so wrap this processor potato back to 0:         

          potatoes[which_node] = 0;          
          MPI_Send(0, 1, MPI_INT, which_node, 0, MPI_COMM_WORLD);
          
        }

        myPotato = potatoes[which_node];        
        
        MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, which_node,
                 myPotato, MPI_COMM_WORLD, &istatus);
        
        atomTypeString = MPIatomTypeString;
        
        myPotato++;

        MPI_Recv(&isDirectional, 1, MPI_INT, which_node,
                 myPotato, MPI_COMM_WORLD, &istatus);
               
        myPotato++;

        if (isDirectional) {          
          MPI_Recv(atomData13, 13, MPI_DOUBLE, which_node,
                   myPotato, MPI_COMM_WORLD, &istatus);
        } else {
          MPI_Recv(atomData6, 6, MPI_DOUBLE, which_node,
                   myPotato, MPI_COMM_WORLD, &istatus);          
        }
        
        myPotato++;
        potatoes[which_node] = myPotato;

      } else {
        
        haveError = 0;
        which_atom = i;
        
          local_index = indexArray[currentIndex].first;        
          
          if (which_atom == indexArray[currentIndex].second) {
            
            atomTypeString = atoms[local_index]->getType();
            
          atoms[local_index]->getPos(pos);
          atoms[local_index]->getVel(vel);          
          
          atomData6[0] = pos[0];
          atomData6[1] = pos[1];
          atomData6[2] = pos[2];

          atomData6[3] = vel[0];
          atomData6[4] = vel[1];
          atomData6[5] = vel[2];
          
          isDirectional = 0;

          if( atoms[local_index]->isDirectional() ){

            isDirectional = 1;
            
            dAtom = (DirectionalAtom *)atoms[local_index];
            dAtom->getQ( q );
            dAtom->getJ( ji );

            for (int j = 0; j < 6 ; j++)
              atomData13[j] = atomData6[j];            
            
            atomData13[6] = q[0];
            atomData13[7] = q[1];
            atomData13[8] = q[2];
            atomData13[9] = q[3];
            
            atomData13[10] = ji[0];
            atomData13[11] = ji[1];
            atomData13[12] = ji[2];
          }
          
        } else {
          sprintf(painCave.errMsg,
                  "Atom %d not found on processor %d, currentIndex = %d, local_index = %d\n",
                  which_atom, worldRank, currentIndex, local_index );
          haveError= 1;
          simError();
        }
        
        if(haveError) DieDieDie();
        
        currentIndex++;
      }
      // If we've survived to here, format the line:
      
      if (!isDirectional) {
        
        sprintf( writeLine,
                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                 atomTypeString,
                 atomData6[0],
                 atomData6[1],
                 atomData6[2],
                 atomData6[3],
                 atomData6[4],
                 atomData6[5]);
        
        strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
        
      } else {
        
        sprintf( writeLine,
                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
                 atomTypeString,
                 atomData13[0],
                 atomData13[1],
                 atomData13[2],
                 atomData13[3],
                 atomData13[4],
                 atomData13[5],
                 atomData13[6],
                 atomData13[7],
                 atomData13[8],
                 atomData13[9],
                 atomData13[10],
                 atomData13[11],
                 atomData13[12]);
        
      }
      
      for(k = 0; k < outFile.size(); k++)
        *outFile[k] << writeLine;
    }
    
    for(k = 0; k < outFile.size(); k++)
      outFile[k]->flush();
    
    sprintf( checkPointMsg,
             "Sucessfully took a dump.\n");
    
    MPIcheckPoint();        
    
    delete[] potatoes;
    
  } else {

    // worldRank != 0, so I'm a remote node.  

    // Set my magic potato to 0:

    myPotato = 0;
    currentIndex = 0;
    
    for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) {
      
      // Am I the node which has this atom?
      
      if (AtomToProcMap[i] == worldRank) {

        if (myPotato + 3 >= MAXTAG) {
          
          // The potato was going to exceed the maximum value, 
          // so wrap this processor potato back to 0 (and block until
          // node 0 says we can go:
          
          MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus);
          
        }
        which_atom = i; 

        local_index = indexArray[currentIndex].first;        
                
        if (which_atom == indexArray[currentIndex].second) {
         
          atomTypeString = atoms[local_index]->getType();
          
          atoms[local_index]->getPos(pos);
          atoms[local_index]->getVel(vel);
          
          atomData6[0] = pos[0];
          atomData6[1] = pos[1];
          atomData6[2] = pos[2];

          atomData6[3] = vel[0];
          atomData6[4] = vel[1];
          atomData6[5] = vel[2];
          
          isDirectional = 0;

          if( atoms[local_index]->isDirectional() ){

            isDirectional = 1;
            
            dAtom = (DirectionalAtom *)atoms[local_index];
            dAtom->getQ( q );
            dAtom->getJ( ji );
            
            for (int j = 0; j < 6 ; j++)
              atomData13[j] = atomData6[j];
            
            atomData13[6] = q[0];
            atomData13[7] = q[1];
            atomData13[8] = q[2];
            atomData13[9] = q[3];
  
            atomData13[10] = ji[0];
            atomData13[11] = ji[1];
            atomData13[12] = ji[2];
          }

        } else {
          sprintf(painCave.errMsg,
                  "Atom %d not found on processor %d, currentIndex = %d, local_index = %d\n",
                  which_atom, worldRank, currentIndex, local_index );
          haveError= 1;
          simError();
        }
        
        strncpy(MPIatomTypeString, atomTypeString, MINIBUFFERSIZE);

        // null terminate the string before sending (just in case):
        MPIatomTypeString[MINIBUFFERSIZE-1] = '\0';

        MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
                             myPotato, MPI_COMM_WORLD);
        
        myPotato++;

        MPI_Send(&isDirectional, 1, MPI_INT, 0,
                             myPotato, MPI_COMM_WORLD);
        
        myPotato++;
        
        if (isDirectional) {

          MPI_Send(atomData13, 13, MPI_DOUBLE, 0,
                   myPotato, MPI_COMM_WORLD);
          
        } else {

          MPI_Send(atomData6, 6, MPI_DOUBLE, 0,
                   myPotato, MPI_COMM_WORLD);
        }

        myPotato++;  
        currentIndex++;    
      }
    }

    sprintf( checkPointMsg,
             "Sucessfully took a dump.\n");
    MPIcheckPoint();        
    
  } 
  
#endif // is_mpi
}

#ifdef IS_MPI

// a couple of functions to let us escape the write loop

void dWrite::DieDieDie( void ){

  MPI_Finalize();
  exit (0);
}

#endif //is_mpi
Revision:	1097
Committed:	Mon Apr 12 20:32:20 2004 UTC (20 years, 2 months ago) by gezelter
File size:	14861 byte(s)
Log Message:	Changes for RigidBody dynamics (Somewhat extensive)
#	User	Rev	Content
1	tim	1078	#define _LARGEFILE_SOURCE64
2	mmeineke	723	#define _FILE_OFFSET_BITS 64
3
4	gezelter	829	#include <string.h>
5	mmeineke	377	#include <iostream>
6			#include <fstream>
7	tim	929	#include <algorithm>
8			#include <utility>
9	mmeineke	377
10			#ifdef IS_MPI
11			#include <mpi.h>
12			#include "mpiSimulation.hpp"
13	mmeineke	440
14			namespace dWrite{
15	gezelter	907	void DieDieDie( void );
16	mmeineke	440	}
17
18			using namespace dWrite;
19	mmeineke	377	#endif //is_mpi
20
21			#include "ReadWrite.hpp"
22			#include "simError.h"
23
24			DumpWriter::DumpWriter( SimInfo* the_entry_plug ){
25
26			entry_plug = the_entry_plug;
27
28			#ifdef IS_MPI
29			if(worldRank == 0 ){
30			#endif // is_mpi
31	tim	837
32	tim	929	dumpFile.open(entry_plug->sampleName, ios::out \| ios::trunc );
33	tim	837
34	tim	929	if( !dumpFile ){
35	tim	837
36	mmeineke	377	sprintf( painCave.errMsg,
37			"Could not open \"%s\" for dump output.\n",
38	tim	929	entry_plug->sampleName);
39	mmeineke	377	painCave.isFatal = 1;
40			simError();
41			}
42	mmeineke	469
43	mmeineke	377	#ifdef IS_MPI
44			}
45
46	tim	929	//sort the local atoms by global index
47			sortByGlobalIndex();
48
49	mmeineke	377	sprintf( checkPointMsg,
50			"Sucessfully opened output file for dumping.\n");
51			MPIcheckPoint();
52			#endif // is_mpi
53			}
54
55			DumpWriter::~DumpWriter( ){
56
57			#ifdef IS_MPI
58			if(worldRank == 0 ){
59			#endif // is_mpi
60
61	tim	929	dumpFile.close();
62	mmeineke	377
63			#ifdef IS_MPI
64			}
65			#endif // is_mpi
66			}
67
68	tim	929	#ifdef IS_MPI
69	tim	837
70	tim	929	/**
71			* A hook function to load balancing
72			*/
73
74			void DumpWriter::update(){
75			sortByGlobalIndex();
76			}
77
78			/**
79			* Auxiliary sorting function
80			*/
81
82			bool indexSortingCriterion(const pair<int, int>& p1, const pair<int, int>& p2){
83			return p1.second < p2.second;
84			}
85
86			/**
87			* Sorting the local index by global index
88			*/
89
90			void DumpWriter::sortByGlobalIndex(){
91			Atom** atoms = entry_plug->atoms;
92
93			indexArray.clear();
94
95	chuckv	949	for(int i = 0; i < mpiSim->getMyNlocal();i++)
96	tim	929	indexArray.push_back(make_pair(i, atoms[i]->getGlobalIndex()));
97
98			sort(indexArray.begin(), indexArray.end(), indexSortingCriterion);
99			}
100	chuckv	949
101	tim	929	#endif
102
103			void DumpWriter::writeDump(double currentTime){
104
105	tim	936	ofstream finalOut;
106	tim	934	vector<ofstream*> fileStreams;
107
108			#ifdef IS_MPI
109			if(worldRank == 0 ){
110	tim	952	#endif
111	tim	936	finalOut.open( entry_plug->finalName, ios::out \| ios::trunc );
112			if( !finalOut ){
113			sprintf( painCave.errMsg,
114			"Could not open \"%s\" for final dump output.\n",
115			entry_plug->finalName );
116			painCave.isFatal = 1;
117			simError();
118			}
119	tim	952	#ifdef IS_MPI
120	tim	934	}
121			#endif // is_mpi
122
123			fileStreams.push_back(&finalOut);
124			fileStreams.push_back(&dumpFile);
125
126			writeFrame(fileStreams, currentTime);
127	tim	936
128			#ifdef IS_MPI
129			finalOut.close();
130			#endif
131	tim	929
132			}
133
134			void DumpWriter::writeFinal(double currentTime){
135
136	tim	936	ofstream finalOut;
137	tim	934	vector<ofstream*> fileStreams;
138
139	tim	929	#ifdef IS_MPI
140			if(worldRank == 0 ){
141	mmeineke	951	#endif // is_mpi
142	tim	936
143			finalOut.open( entry_plug->finalName, ios::out \| ios::trunc );
144
145			if( !finalOut ){
146			sprintf( painCave.errMsg,
147			"Could not open \"%s\" for final dump output.\n",
148			entry_plug->finalName );
149			painCave.isFatal = 1;
150			simError();
151			}
152
153	mmeineke	951	#ifdef IS_MPI
154	tim	934	}
155	tim	929	#endif // is_mpi
156
157	tim	934	fileStreams.push_back(&finalOut);
158			writeFrame(fileStreams, currentTime);
159	tim	936
160			#ifdef IS_MPI
161			finalOut.close();
162			#endif
163	tim	929
164			}
165
166	tim	934	void DumpWriter::writeFrame( vector<ofstream*>& outFile, double currentTime ){
167	tim	929
168	mmeineke	377	const int BUFFERSIZE = 2000;
169	gezelter	912	const int MINIBUFFERSIZE = 100;
170	gezelter	907
171	tim	936	char tempBuffer[BUFFERSIZE];
172	mmeineke	377	char writeLine[BUFFERSIZE];
173
174	tim	934	int i, k;
175	gezelter	916
176	mmeineke	787	#ifdef IS_MPI
177	gezelter	916
178	gezelter	947	/*********************************************************************
179			* Documentation? You want DOCUMENTATION?
180			*
181			* Why all the potatoes below?
182			*
183			* To make a long story short, the original version of DumpWriter
184			* worked in the most inefficient way possible. Node 0 would
185			* poke each of the node for an individual atom's formatted data
186			* as node 0 worked its way down the global index. This was particularly
187			* inefficient since the method blocked all processors at every atom
188			* (and did it twice!).
189			*
190			* An intermediate version of DumpWriter could be described from Node
191			* zero's perspective as follows:
192			*
193			* 1) Have 100 of your friends stand in a circle.
194			* 2) When you say go, have all of them start tossing potatoes at
195			* you (one at a time).
196			* 3) Catch the potatoes.
197			*
198			* It was an improvement, but MPI has buffers and caches that could
199			* best be described in this analogy as "potato nets", so there's no
200			* need to block the processors atom-by-atom.
201			*
202			* This new and improved DumpWriter works in an even more efficient
203			* way:
204			*
205			* 1) Have 100 of your friend stand in a circle.
206			* 2) When you say go, have them start tossing 5-pound bags of
207			* potatoes at you.
208			* 3) Once you've caught a friend's bag of potatoes,
209			* toss them a spud to let them know they can toss another bag.
210			*
211			* How's THAT for documentation?
212			*
213			*********************************************************************/
214
215	gezelter	916	int *potatoes;
216			int myPotato;
217
218			int nProc;
219	tim	929	int j, which_node, done, which_atom, local_index, currentIndex;
220	gezelter	916	double atomData6[6];
221			double atomData13[13];
222	gezelter	907	int isDirectional;
223			char* atomTypeString;
224	gezelter	910	char MPIatomTypeString[MINIBUFFERSIZE];
225	gezelter	916
226	mmeineke	787	#else //is_mpi
227			int nAtoms = entry_plug->n_atoms;
228			#endif //is_mpi
229
230	gezelter	1097	double q[4], ji[3];
231	mmeineke	377	DirectionalAtom* dAtom;
232			Atom** atoms = entry_plug->atoms;
233	mmeineke	670	double pos[3], vel[3];
234	tim	837
235	mmeineke	377	#ifndef IS_MPI
236	tim	934
237			for(k = 0; k < outFile.size(); k++){
238			*outFile[k] << nAtoms << "\n";
239	tim	837
240	tim	934	*outFile[k] << currentTime << ";\t"
241			<< entry_plug->Hmat[0][0] << "\t"
242			<< entry_plug->Hmat[1][0] << "\t"
243			<< entry_plug->Hmat[2][0] << ";\t"
244
245			<< entry_plug->Hmat[0][1] << "\t"
246			<< entry_plug->Hmat[1][1] << "\t"
247			<< entry_plug->Hmat[2][1] << ";\t"
248	tim	837
249	tim	934	<< entry_plug->Hmat[0][2] << "\t"
250			<< entry_plug->Hmat[1][2] << "\t"
251			<< entry_plug->Hmat[2][2] << ";";
252	mmeineke	572
253	tim	934	//write out additional parameters, such as chi and eta
254			*outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
255			}
256
257	mmeineke	377	for( i=0; i<nAtoms; i++ ){
258	tim	837
259	mmeineke	670	atoms[i]->getPos(pos);
260			atoms[i]->getVel(vel);
261	mmeineke	377
262			sprintf( tempBuffer,
263			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
264			atoms[i]->getType(),
265	mmeineke	670	pos[0],
266			pos[1],
267			pos[2],
268			vel[0],
269			vel[1],
270			vel[2]);
271	mmeineke	377	strcpy( writeLine, tempBuffer );
272
273			if( atoms[i]->isDirectional() ){
274	tim	837
275	mmeineke	377	dAtom = (DirectionalAtom *)atoms[i];
276			dAtom->getQ( q );
277	gezelter	1097	dAtom->getJ( ji );
278	tim	837
279	mmeineke	377	sprintf( tempBuffer,
280			"%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
281			q[0],
282			q[1],
283			q[2],
284			q[3],
285	gezelter	1097	ji[0],
286			ji[1],
287			ji[2]);
288	mmeineke	377	strcat( writeLine, tempBuffer );
289			}
290			else
291			strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
292	tim	837
293	tim	934	for(k = 0; k < outFile.size(); k++)
294			*outFile[k] << writeLine;
295	mmeineke	377	}
296
297			#else // is_mpi
298	gezelter	416
299	chuckv	913	/* code to find maximum tag value */
300	tim	919
301	tim	920	int *tagub, flag, MAXTAG;
302	chuckv	913	MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);
303			if (flag) {
304	tim	920	MAXTAG = *tagub;
305	chuckv	913	} else {
306			MAXTAG = 32767;
307	gezelter	916	}
308	mmeineke	440
309			int haveError;
310
311	mmeineke	447	MPI_Status istatus;
312	gezelter	416	int *AtomToProcMap = mpiSim->getAtomToProcMap();
313	tim	837
314	mmeineke	377	// write out header and node 0's coordinates
315	tim	837
316	mmeineke	377	if( worldRank == 0 ){
317	gezelter	916
318			// Node 0 needs a list of the magic potatoes for each processor;
319
320			nProc = mpiSim->getNumberProcessors();
321			potatoes = new int[nProc];
322
323	tim	934	//write out the comment lines
324	gezelter	916	for (i = 0; i < nProc; i++)
325			potatoes[i] = 0;
326
327	tim	934	for(k = 0; k < outFile.size(); k++){
328			*outFile[k] << mpiSim->getTotAtoms() << "\n";
329	tim	837
330	tim	934	*outFile[k] << currentTime << ";\t"
331			<< entry_plug->Hmat[0][0] << "\t"
332			<< entry_plug->Hmat[1][0] << "\t"
333			<< entry_plug->Hmat[2][0] << ";\t"
334	tim	837
335	tim	934	<< entry_plug->Hmat[0][1] << "\t"
336			<< entry_plug->Hmat[1][1] << "\t"
337			<< entry_plug->Hmat[2][1] << ";\t"
338	tim	837
339	tim	934	<< entry_plug->Hmat[0][2] << "\t"
340			<< entry_plug->Hmat[1][2] << "\t"
341			<< entry_plug->Hmat[2][2] << ";";
342
343			*outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
344			}
345	tim	837
346	tim	929	currentIndex = 0;
347	tim	934
348	gezelter	416	for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) {
349	chuckv	913
350	gezelter	417	// Get the Node number which has this atom;
351	chuckv	913
352	tim	837	which_node = AtomToProcMap[i];
353	chuckv	913
354	gezelter	907	if (which_node != 0) {
355	gezelter	916
356			if (potatoes[which_node] + 3 >= MAXTAG) {
357			// The potato was going to exceed the maximum value,
358			// so wrap this processor potato back to 0:
359
360			potatoes[which_node] = 0;
361			MPI_Send(0, 1, MPI_INT, which_node, 0, MPI_COMM_WORLD);
362
363			}
364
365			myPotato = potatoes[which_node];
366	gezelter	907
367	gezelter	910	MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, which_node,
368	gezelter	916	myPotato, MPI_COMM_WORLD, &istatus);
369	gezelter	907
370	tim	920	atomTypeString = MPIatomTypeString;
371
372	gezelter	916	myPotato++;
373
374	gezelter	907	MPI_Recv(&isDirectional, 1, MPI_INT, which_node,
375	gezelter	916	myPotato, MPI_COMM_WORLD, &istatus);
376
377			myPotato++;
378	gezelter	907
379	gezelter	916	if (isDirectional) {
380			MPI_Recv(atomData13, 13, MPI_DOUBLE, which_node,
381			myPotato, MPI_COMM_WORLD, &istatus);
382			} else {
383			MPI_Recv(atomData6, 6, MPI_DOUBLE, which_node,
384			myPotato, MPI_COMM_WORLD, &istatus);
385	gezelter	907	}
386	gezelter	916
387			myPotato++;
388			potatoes[which_node] = myPotato;
389	gezelter	907
390			} else {
391
392	tim	934	haveError = 0;
393	chuckv	949	which_atom = i;
394	gezelter	916
395	chuckv	949	local_index = indexArray[currentIndex].first;
396	gezelter	916
397	chuckv	949	if (which_atom == indexArray[currentIndex].second) {
398
399			atomTypeString = atoms[local_index]->getType();
400
401			atoms[local_index]->getPos(pos);
402			atoms[local_index]->getVel(vel);
403
404	gezelter	916	atomData6[0] = pos[0];
405			atomData6[1] = pos[1];
406			atomData6[2] = pos[2];
407	tim	837
408	gezelter	916	atomData6[3] = vel[0];
409			atomData6[4] = vel[1];
410			atomData6[5] = vel[2];
411	gezelter	907
412			isDirectional = 0;
413
414	chuckv	436	if( atoms[local_index]->isDirectional() ){
415	tim	837
416	gezelter	907	isDirectional = 1;
417
418	chuckv	436	dAtom = (DirectionalAtom *)atoms[local_index];
419			dAtom->getQ( q );
420	gezelter	1097	dAtom->getJ( ji );
421	gezelter	916
422			for (int j = 0; j < 6 ; j++)
423			atomData13[j] = atomData6[j];
424	gezelter	907
425	gezelter	916	atomData13[6] = q[0];
426			atomData13[7] = q[1];
427			atomData13[8] = q[2];
428			atomData13[9] = q[3];
429
430	gezelter	1097	atomData13[10] = ji[0];
431			atomData13[11] = ji[1];
432			atomData13[12] = ji[2];
433	gezelter	907	}
434	gezelter	916
435	gezelter	907	} else {
436	chuckv	949	sprintf(painCave.errMsg,
437			"Atom %d not found on processor %d, currentIndex = %d, local_index = %d\n",
438			which_atom, worldRank, currentIndex, local_index );
439			haveError= 1;
440			simError();
441			}
442	gezelter	916
443	tim	934	if(haveError) DieDieDie();
444	gezelter	916
445	chuckv	949	currentIndex++;
446	tim	926	}
447			// If we've survived to here, format the line:
448
449			if (!isDirectional) {
450
451	tim	934	sprintf( writeLine,
452	chuckv	949	"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
453			atomTypeString,
454			atomData6[0],
455			atomData6[1],
456			atomData6[2],
457			atomData6[3],
458			atomData6[4],
459			atomData6[5]);
460	tim	926
461	chuckv	949	strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
462
463	tim	926	} else {
464
465	chuckv	949	sprintf( writeLine,
466			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
467			atomTypeString,
468			atomData13[0],
469			atomData13[1],
470			atomData13[2],
471			atomData13[3],
472			atomData13[4],
473			atomData13[5],
474			atomData13[6],
475			atomData13[7],
476			atomData13[8],
477			atomData13[9],
478			atomData13[10],
479			atomData13[11],
480			atomData13[12]);
481	gezelter	907
482			}
483	tim	926
484	tim	934	for(k = 0; k < outFile.size(); k++)
485			*outFile[k] << writeLine;
486	mmeineke	377	}
487	tim	926
488	tim	934	for(k = 0; k < outFile.size(); k++)
489			outFile[k]->flush();
490
491	gezelter	907	sprintf( checkPointMsg,
492			"Sucessfully took a dump.\n");
493	chuckv	949
494	gezelter	907	MPIcheckPoint();
495	chuckv	949
496	tim	919	delete[] potatoes;
497	chuckv	949
498	gezelter	415	} else {
499	tim	837
500	gezelter	907	// worldRank != 0, so I'm a remote node.
501	gezelter	916
502			// Set my magic potato to 0:
503
504			myPotato = 0;
505	tim	929	currentIndex = 0;
506	gezelter	907
507			for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) {
508
509			// Am I the node which has this atom?
510
511			if (AtomToProcMap[i] == worldRank) {
512	tim	837
513	gezelter	916	if (myPotato + 3 >= MAXTAG) {
514	chuckv	949
515	gezelter	916	// The potato was going to exceed the maximum value,
516			// so wrap this processor potato back to 0 (and block until
517			// node 0 says we can go:
518	chuckv	949
519	gezelter	916	MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus);
520
521			}
522	tim	919	which_atom = i;
523	chuckv	949
524			local_index = indexArray[currentIndex].first;
525	tim	920
526	chuckv	949	if (which_atom == indexArray[currentIndex].second) {
527	gezelter	907
528			atomTypeString = atoms[local_index]->getType();
529	chuckv	949
530			atoms[local_index]->getPos(pos);
531			atoms[local_index]->getVel(vel);
532
533	gezelter	916	atomData6[0] = pos[0];
534			atomData6[1] = pos[1];
535			atomData6[2] = pos[2];
536	tim	837
537	gezelter	916	atomData6[3] = vel[0];
538			atomData6[4] = vel[1];
539			atomData6[5] = vel[2];
540	gezelter	907
541			isDirectional = 0;
542	tim	837
543	gezelter	907	if( atoms[local_index]->isDirectional() ){
544	mmeineke	377
545	gezelter	907	isDirectional = 1;
546
547			dAtom = (DirectionalAtom *)atoms[local_index];
548			dAtom->getQ( q );
549	gezelter	1097	dAtom->getJ( ji );
550	gezelter	907
551	gezelter	916	for (int j = 0; j < 6 ; j++)
552			atomData13[j] = atomData6[j];
553
554			atomData13[6] = q[0];
555			atomData13[7] = q[1];
556			atomData13[8] = q[2];
557			atomData13[9] = q[3];
558	tim	934
559	gezelter	1097	atomData13[10] = ji[0];
560			atomData13[11] = ji[1];
561			atomData13[12] = ji[2];
562	gezelter	907	}
563	tim	837
564	gezelter	907	} else {
565	chuckv	949	sprintf(painCave.errMsg,
566			"Atom %d not found on processor %d, currentIndex = %d, local_index = %d\n",
567			which_atom, worldRank, currentIndex, local_index );
568			haveError= 1;
569			simError();
570			}
571
572	gezelter	916	strncpy(MPIatomTypeString, atomTypeString, MINIBUFFERSIZE);
573	tim	837
574	gezelter	916	// null terminate the string before sending (just in case):
575			MPIatomTypeString[MINIBUFFERSIZE-1] = '\0';
576	tim	837
577	gezelter	910	MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
578	tim	934	myPotato, MPI_COMM_WORLD);
579	gezelter	907
580	gezelter	916	myPotato++;
581
582	gezelter	907	MPI_Send(&isDirectional, 1, MPI_INT, 0,
583	tim	934	myPotato, MPI_COMM_WORLD);
584	gezelter	907
585	gezelter	916	myPotato++;
586
587	gezelter	907	if (isDirectional) {
588
589	gezelter	916	MPI_Send(atomData13, 13, MPI_DOUBLE, 0,
590			myPotato, MPI_COMM_WORLD);
591	gezelter	907
592	gezelter	916	} else {
593
594			MPI_Send(atomData6, 6, MPI_DOUBLE, 0,
595			myPotato, MPI_COMM_WORLD);
596	gezelter	907	}
597	gezelter	916
598	tim	929	myPotato++;
599			currentIndex++;
600	mmeineke	377	}
601	gezelter	907	}
602	mmeineke	440
603	gezelter	907	sprintf( checkPointMsg,
604	gezelter	916	"Sucessfully took a dump.\n");
605	gezelter	907	MPIcheckPoint();
606
607	gezelter	916	}
608	gezelter	907
609	mmeineke	377	#endif // is_mpi
610			}
611	mmeineke	440
612			#ifdef IS_MPI
613
614			// a couple of functions to let us escape the write loop
615
616	gezelter	907	void dWrite::DieDieDie( void ){
617	tim	837
618	mmeineke	440	MPI_Finalize();
619			exit (0);
620			}
621
622			#endif //is_mpi