OOPSE/libmdtools/DumpWriter.cpp

#define _FILE_OFFSET_BITS 64

#include <string.h>
#include <iostream>
#include <fstream>
#include <algorithm>
#include <utility>

#ifdef IS_MPI
#include <mpi.h>
#include "mpiSimulation.hpp"

namespace dWrite{
  void DieDieDie( void );
}

using namespace dWrite;
#endif //is_mpi

#include "ReadWrite.hpp"
#include "simError.h"

DumpWriter::DumpWriter( SimInfo* the_entry_plug ){

  entry_plug = the_entry_plug;

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif // is_mpi

    dumpFile.open(entry_plug->sampleName, ios::out | ios::trunc );

    if( !dumpFile ){

      sprintf( painCave.errMsg,
               "Could not open \"%s\" for dump output.\n",
               entry_plug->sampleName);
      painCave.isFatal = 1;
      simError();
    }

#ifdef IS_MPI
  }

  //sort the local atoms by global index
  sortByGlobalIndex();
  
  sprintf( checkPointMsg,
           "Sucessfully opened output file for dumping.\n");
  MPIcheckPoint();
#endif // is_mpi
}

DumpWriter::~DumpWriter( ){

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif // is_mpi

    dumpFile.close();

#ifdef IS_MPI
  }
#endif // is_mpi
}

#ifdef IS_MPI

/**
 * A hook function to load balancing
 */

void DumpWriter::update(){
  sortByGlobalIndex();          
}
  
/**
 * Auxiliary sorting function
 */
 
bool indexSortingCriterion(const pair<int, int>& p1, const pair<int, int>& p2){
  return p1.second < p2.second;
}

/**
 * Sorting the local index by global index
 */
 
void DumpWriter::sortByGlobalIndex(){
  Atom** atoms = entry_plug->atoms;
  
  indexArray.clear();
  
  for(int i = 0; i < mpiSim->getMyNlocal();i++) 
    indexArray.push_back(make_pair(i, atoms[i]->getGlobalIndex()));
  
  sort(indexArray.begin(), indexArray.end(), indexSortingCriterion);    

  //for (int i = 0; i < mpiSim->getMyNlocal(); i++) {
  //  printf("node %d has global %d at local %d\n", worldRank, indexArray[i].second, indexArray[i].first);
  //}
    
}

#endif

void DumpWriter::writeDump(double currentTime){

  ofstream finalOut;
  vector<ofstream*> fileStreams;

#ifdef IS_MPI
  printf("Hello from node %d\n", worldRank);
  sortByGlobalIndex();
  if(worldRank == 0 ){
    
    finalOut.open( entry_plug->finalName, ios::out | ios::trunc );
    if( !finalOut ){
      sprintf( painCave.errMsg,
               "Could not open \"%s\" for final dump output.\n",
               entry_plug->finalName );
      painCave.isFatal = 1;
      simError();
    }
  }
#endif // is_mpi

  fileStreams.push_back(&finalOut); 
  fileStreams.push_back(&dumpFile);

  writeFrame(fileStreams, currentTime);

#ifdef IS_MPI
  finalOut.close();
#endif
        
}

void DumpWriter::writeFinal(double currentTime){

  ofstream finalOut;
  vector<ofstream*> fileStreams;

#ifdef IS_MPI
  if(worldRank == 0 ){

    finalOut.open( entry_plug->finalName, ios::out | ios::trunc );

    if( !finalOut ){
      sprintf( painCave.errMsg,
               "Could not open \"%s\" for final dump output.\n",
               entry_plug->finalName );
      painCave.isFatal = 1;
      simError();
    }

  }
#endif // is_mpi
  
  fileStreams.push_back(&finalOut);  
  writeFrame(fileStreams, currentTime);

#ifdef IS_MPI
  finalOut.close();
#endif
  
}

void DumpWriter::writeFrame( vector<ofstream*>& outFile, double currentTime ){

  const int BUFFERSIZE = 2000;
  const int MINIBUFFERSIZE = 100;

  char tempBuffer[BUFFERSIZE];  
  char writeLine[BUFFERSIZE];

  int i, k;

#ifdef IS_MPI
  
  /*********************************************************************
   * Documentation?  You want DOCUMENTATION?
   * 
   * Why all the potatoes below?  
   *
   * To make a long story short, the original version of DumpWriter
   * worked in the most inefficient way possible.  Node 0 would 
   * poke each of the node for an individual atom's formatted data 
   * as node 0 worked its way down the global index. This was particularly 
   * inefficient since the method blocked all processors at every atom 
   * (and did it twice!).
   *
   * An intermediate version of DumpWriter could be described from Node
   * zero's perspective as follows:
   * 
   *  1) Have 100 of your friends stand in a circle.
   *  2) When you say go, have all of them start tossing potatoes at
   *     you (one at a time).
   *  3) Catch the potatoes.
   *
   * It was an improvement, but MPI has buffers and caches that could 
   * best be described in this analogy as "potato nets", so there's no 
   * need to block the processors atom-by-atom.
   * 
   * This new and improved DumpWriter works in an even more efficient 
   * way:
   * 
   *  1) Have 100 of your friend stand in a circle.
   *  2) When you say go, have them start tossing 5-pound bags of 
   *     potatoes at you.
   *  3) Once you've caught a friend's bag of potatoes,
   *     toss them a spud to let them know they can toss another bag.
   *
   * How's THAT for documentation?
   *
   *********************************************************************/

  int *potatoes;
  int myPotato;

  int nProc;
  int j, which_node, done, which_atom, local_index, currentIndex;
  double atomData6[6];
  double atomData13[13];
  int isDirectional;
  char* atomTypeString;
  char MPIatomTypeString[MINIBUFFERSIZE];

#else //is_mpi
  int nAtoms = entry_plug->n_atoms;
#endif //is_mpi

  double q[4];
  DirectionalAtom* dAtom;
  Atom** atoms = entry_plug->atoms;
  double pos[3], vel[3];

#ifndef IS_MPI
  
  for(k = 0; k < outFile.size(); k++){
    *outFile[k] << nAtoms << "\n";

    *outFile[k] << currentTime << ";\t"
               << entry_plug->Hmat[0][0] << "\t"
                     << entry_plug->Hmat[1][0] << "\t"
                     << entry_plug->Hmat[2][0] << ";\t"
               
               << entry_plug->Hmat[0][1] << "\t"
                     << entry_plug->Hmat[1][1] << "\t"
                     << entry_plug->Hmat[2][1] << ";\t"

                     << entry_plug->Hmat[0][2] << "\t"
                     << entry_plug->Hmat[1][2] << "\t"
                     << entry_plug->Hmat[2][2] << ";";

    //write out additional parameters, such as chi and eta
    *outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
  }
  
  for( i=0; i<nAtoms; i++ ){

    atoms[i]->getPos(pos);
    atoms[i]->getVel(vel);

    sprintf( tempBuffer,
             "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
             atoms[i]->getType(),
             pos[0],
             pos[1],
             pos[2],
             vel[0],
             vel[1],
             vel[2]);
    strcpy( writeLine, tempBuffer );

    if( atoms[i]->isDirectional() ){

      dAtom = (DirectionalAtom *)atoms[i];
      dAtom->getQ( q );

      sprintf( tempBuffer,
               "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
               q[0],
               q[1],
               q[2],
               q[3],
               dAtom->getJx(),
               dAtom->getJy(),
               dAtom->getJz());
      strcat( writeLine, tempBuffer );
    }
    else
      strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );

    for(k = 0; k < outFile.size(); k++)
      *outFile[k] << writeLine;
  }

#else // is_mpi

  /* code to find maximum tag value */
  
  int *tagub, flag, MAXTAG;
  MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);
  if (flag) {
    MAXTAG = *tagub;
  } else {
    MAXTAG = 32767;
  }  

  int haveError;

  MPI_Status istatus;
  int *AtomToProcMap = mpiSim->getAtomToProcMap();

  // write out header and node 0's coordinates

  if( worldRank == 0 ){

    // Node 0 needs a list of the magic potatoes for each processor;

    nProc = mpiSim->getNumberProcessors();
    potatoes = new int[nProc];

    //write out the comment lines
    for (i = 0; i < nProc; i++) 
      potatoes[i] = 0;
    
      for(k = 0; k < outFile.size(); k++){
        *outFile[k] << mpiSim->getTotAtoms() << "\n";

        *outFile[k] << currentTime << ";\t"
                         << entry_plug->Hmat[0][0] << "\t"
                         << entry_plug->Hmat[1][0] << "\t"
                         << entry_plug->Hmat[2][0] << ";\t"

                         << entry_plug->Hmat[0][1] << "\t"
                         << entry_plug->Hmat[1][1] << "\t"
                         << entry_plug->Hmat[2][1] << ";\t"

                         << entry_plug->Hmat[0][2] << "\t"
                         << entry_plug->Hmat[1][2] << "\t"
                         << entry_plug->Hmat[2][2] << ";";
  
        *outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
    }

    currentIndex = 0;

    for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) {
      
      // Get the Node number which has this atom;
      
      which_node = AtomToProcMap[i];
      
      if (which_node != 0) {

        if (potatoes[which_node] + 3 >= MAXTAG) {
          // The potato was going to exceed the maximum value, 
          // so wrap this processor potato back to 0:         

          potatoes[which_node] = 0;          
          MPI_Send(0, 1, MPI_INT, which_node, 0, MPI_COMM_WORLD);
          
        }

        myPotato = potatoes[which_node];        
        
        MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, which_node,
                 myPotato, MPI_COMM_WORLD, &istatus);
        
        atomTypeString = MPIatomTypeString;
        
        myPotato++;

        MPI_Recv(&isDirectional, 1, MPI_INT, which_node,
                 myPotato, MPI_COMM_WORLD, &istatus);
               
        myPotato++;

        if (isDirectional) {          
          MPI_Recv(atomData13, 13, MPI_DOUBLE, which_node,
                   myPotato, MPI_COMM_WORLD, &istatus);
        } else {
          MPI_Recv(atomData6, 6, MPI_DOUBLE, which_node,
                   myPotato, MPI_COMM_WORLD, &istatus);          
        }
        
        myPotato++;
        potatoes[which_node] = myPotato;

      } else {
        
        haveError = 0;
        which_atom = i;
        
        //local_index = -1;

        //for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
        //  if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
        //}
        
        //if (local_index != -1) {
          
          local_index = indexArray[currentIndex].first;        
          
          if (which_atom == indexArray[currentIndex].second) {
            
            atomTypeString = atoms[local_index]->getType();
            
          atoms[local_index]->getPos(pos);
          atoms[local_index]->getVel(vel);          
          
          atomData6[0] = pos[0];
          atomData6[1] = pos[1];
          atomData6[2] = pos[2];

          atomData6[3] = vel[0];
          atomData6[4] = vel[1];
          atomData6[5] = vel[2];
          
          isDirectional = 0;

          if( atoms[local_index]->isDirectional() ){

            isDirectional = 1;
            
            dAtom = (DirectionalAtom *)atoms[local_index];
            dAtom->getQ( q );

            for (int j = 0; j < 6 ; j++)
              atomData13[j] = atomData6[j];            
            
            atomData13[6] = q[0];
            atomData13[7] = q[1];
            atomData13[8] = q[2];
            atomData13[9] = q[3];
            
            atomData13[10] = dAtom->getJx();
            atomData13[11] = dAtom->getJy();
            atomData13[12] = dAtom->getJz();
          }
          
        } else {
          sprintf(painCave.errMsg,
                  "Atom %d not found on processor %d, currentIndex = %d, local_index = %d\n",
                  which_atom, worldRank, currentIndex, local_index );
          haveError= 1;
          simError();
        }
        
        if(haveError) DieDieDie();
        
        currentIndex++;
      }
      // If we've survived to here, format the line:
      
      if (!isDirectional) {
        
        sprintf( writeLine,
                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                 atomTypeString,
                 atomData6[0],
                 atomData6[1],
                 atomData6[2],
                 atomData6[3],
                 atomData6[4],
                 atomData6[5]);
        
        strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
        
      } else {
        
        sprintf( writeLine,
                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
                 atomTypeString,
                 atomData13[0],
                 atomData13[1],
                 atomData13[2],
                 atomData13[3],
                 atomData13[4],
                 atomData13[5],
                 atomData13[6],
                 atomData13[7],
                 atomData13[8],
                 atomData13[9],
                 atomData13[10],
                 atomData13[11],
                 atomData13[12]);
        
      }
      
      for(k = 0; k < outFile.size(); k++)
        *outFile[k] << writeLine;
    }
    
    for(k = 0; k < outFile.size(); k++)
      outFile[k]->flush();
    
    sprintf( checkPointMsg,
             "Sucessfully took a dump.\n");
    
    MPIcheckPoint();        
    
    delete[] potatoes;
    
  } else {

    // worldRank != 0, so I'm a remote node.  

    // Set my magic potato to 0:

    myPotato = 0;
    currentIndex = 0;
    
    for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) {
      
      // Am I the node which has this atom?
      
      if (AtomToProcMap[i] == worldRank) {

        if (myPotato + 3 >= MAXTAG) {
          
          // The potato was going to exceed the maximum value, 
          // so wrap this processor potato back to 0 (and block until
          // node 0 says we can go:
          
          MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus);
          
        }
        which_atom = i; 

        //local_index = -1;

        //for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
        // if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
        //}
        
        //if (local_index != -1) {

        local_index = indexArray[currentIndex].first;        
                
        if (which_atom == indexArray[currentIndex].second) {
         
          atomTypeString = atoms[local_index]->getType();
          
          atoms[local_index]->getPos(pos);
          atoms[local_index]->getVel(vel);
          
          atomData6[0] = pos[0];
          atomData6[1] = pos[1];
          atomData6[2] = pos[2];

          atomData6[3] = vel[0];
          atomData6[4] = vel[1];
          atomData6[5] = vel[2];
          
          isDirectional = 0;

          if( atoms[local_index]->isDirectional() ){

            isDirectional = 1;
            
            dAtom = (DirectionalAtom *)atoms[local_index];
            dAtom->getQ( q );
            
            for (int j = 0; j < 6 ; j++)
              atomData13[j] = atomData6[j];
            
            atomData13[6] = q[0];
            atomData13[7] = q[1];
            atomData13[8] = q[2];
            atomData13[9] = q[3];
  
            atomData13[10] = dAtom->getJx();
            atomData13[11] = dAtom->getJy();
            atomData13[12] = dAtom->getJz();
          }

        } else {
          sprintf(painCave.errMsg,
                  "Atom %d not found on processor %d, currentIndex = %d, local_index = %d\n",
                  which_atom, worldRank, currentIndex, local_index );
          haveError= 1;
          simError();
        }
        
        strncpy(MPIatomTypeString, atomTypeString, MINIBUFFERSIZE);

        // null terminate the string before sending (just in case):
        MPIatomTypeString[MINIBUFFERSIZE-1] = '\0';

        MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
                             myPotato, MPI_COMM_WORLD);
        
        myPotato++;

        MPI_Send(&isDirectional, 1, MPI_INT, 0,
                             myPotato, MPI_COMM_WORLD);
        
        myPotato++;
        
        if (isDirectional) {

          MPI_Send(atomData13, 13, MPI_DOUBLE, 0,
                   myPotato, MPI_COMM_WORLD);
          
        } else {

          MPI_Send(atomData6, 6, MPI_DOUBLE, 0,
                   myPotato, MPI_COMM_WORLD);
        }

        myPotato++;  
        currentIndex++;    
      }
    }

    sprintf( checkPointMsg,
             "Sucessfully took a dump.\n");
    MPIcheckPoint();        
    
  } 
  
#endif // is_mpi
}

#ifdef IS_MPI

// a couple of functions to let us escape the write loop

void dWrite::DieDieDie( void ){

  MPI_Finalize();
  exit (0);
}

#endif //is_mpi
Revision:	949
Committed:	Thu Jan 15 21:57:10 2004 UTC (20 years, 5 months ago) by chuckv
File size:	15449 byte(s)
Log Message:	Fixes for Dumps
#	User	Rev	Content
1	mmeineke	723	#define _FILE_OFFSET_BITS 64
2
3	gezelter	829	#include <string.h>
4	mmeineke	377	#include <iostream>
5			#include <fstream>
6	tim	929	#include <algorithm>
7			#include <utility>
8	mmeineke	377
9			#ifdef IS_MPI
10			#include <mpi.h>
11			#include "mpiSimulation.hpp"
12	mmeineke	440
13			namespace dWrite{
14	gezelter	907	void DieDieDie( void );
15	mmeineke	440	}
16
17			using namespace dWrite;
18	mmeineke	377	#endif //is_mpi
19
20			#include "ReadWrite.hpp"
21			#include "simError.h"
22
23			DumpWriter::DumpWriter( SimInfo* the_entry_plug ){
24
25			entry_plug = the_entry_plug;
26
27			#ifdef IS_MPI
28			if(worldRank == 0 ){
29			#endif // is_mpi
30	tim	837
31	tim	929	dumpFile.open(entry_plug->sampleName, ios::out \| ios::trunc );
32	tim	837
33	tim	929	if( !dumpFile ){
34	tim	837
35	mmeineke	377	sprintf( painCave.errMsg,
36			"Could not open \"%s\" for dump output.\n",
37	tim	929	entry_plug->sampleName);
38	mmeineke	377	painCave.isFatal = 1;
39			simError();
40			}
41	mmeineke	469
42	mmeineke	377	#ifdef IS_MPI
43			}
44
45	tim	929	//sort the local atoms by global index
46			sortByGlobalIndex();
47
48	mmeineke	377	sprintf( checkPointMsg,
49			"Sucessfully opened output file for dumping.\n");
50			MPIcheckPoint();
51			#endif // is_mpi
52			}
53
54			DumpWriter::~DumpWriter( ){
55
56			#ifdef IS_MPI
57			if(worldRank == 0 ){
58			#endif // is_mpi
59
60	tim	929	dumpFile.close();
61	mmeineke	377
62			#ifdef IS_MPI
63			}
64			#endif // is_mpi
65			}
66
67	tim	929	#ifdef IS_MPI
68	tim	837
69	tim	929	/**
70			* A hook function to load balancing
71			*/
72
73			void DumpWriter::update(){
74			sortByGlobalIndex();
75			}
76
77			/**
78			* Auxiliary sorting function
79			*/
80
81			bool indexSortingCriterion(const pair<int, int>& p1, const pair<int, int>& p2){
82			return p1.second < p2.second;
83			}
84
85			/**
86			* Sorting the local index by global index
87			*/
88
89			void DumpWriter::sortByGlobalIndex(){
90			Atom** atoms = entry_plug->atoms;
91
92			indexArray.clear();
93
94	chuckv	949	for(int i = 0; i < mpiSim->getMyNlocal();i++)
95	tim	929	indexArray.push_back(make_pair(i, atoms[i]->getGlobalIndex()));
96
97			sort(indexArray.begin(), indexArray.end(), indexSortingCriterion);
98	chuckv	949
99			//for (int i = 0; i < mpiSim->getMyNlocal(); i++) {
100			// printf("node %d has global %d at local %d\n", worldRank, indexArray[i].second, indexArray[i].first);
101			//}
102
103	tim	929	}
104	chuckv	949
105	tim	929	#endif
106
107			void DumpWriter::writeDump(double currentTime){
108
109	tim	936	ofstream finalOut;
110	tim	934	vector<ofstream*> fileStreams;
111
112			#ifdef IS_MPI
113	chuckv	949	printf("Hello from node %d\n", worldRank);
114			sortByGlobalIndex();
115	tim	934	if(worldRank == 0 ){
116	tim	936
117			finalOut.open( entry_plug->finalName, ios::out \| ios::trunc );
118			if( !finalOut ){
119			sprintf( painCave.errMsg,
120			"Could not open \"%s\" for final dump output.\n",
121			entry_plug->finalName );
122			painCave.isFatal = 1;
123			simError();
124			}
125	tim	934	}
126			#endif // is_mpi
127
128			fileStreams.push_back(&finalOut);
129			fileStreams.push_back(&dumpFile);
130
131			writeFrame(fileStreams, currentTime);
132	tim	936
133			#ifdef IS_MPI
134			finalOut.close();
135			#endif
136	tim	929
137			}
138
139			void DumpWriter::writeFinal(double currentTime){
140
141	tim	936	ofstream finalOut;
142	tim	934	vector<ofstream*> fileStreams;
143
144	tim	929	#ifdef IS_MPI
145			if(worldRank == 0 ){
146	tim	936
147			finalOut.open( entry_plug->finalName, ios::out \| ios::trunc );
148
149			if( !finalOut ){
150			sprintf( painCave.errMsg,
151			"Could not open \"%s\" for final dump output.\n",
152			entry_plug->finalName );
153			painCave.isFatal = 1;
154			simError();
155			}
156
157	tim	934	}
158	tim	929	#endif // is_mpi
159
160	tim	934	fileStreams.push_back(&finalOut);
161			writeFrame(fileStreams, currentTime);
162	tim	936
163			#ifdef IS_MPI
164			finalOut.close();
165			#endif
166	tim	929
167			}
168
169	tim	934	void DumpWriter::writeFrame( vector<ofstream*>& outFile, double currentTime ){
170	tim	929
171	mmeineke	377	const int BUFFERSIZE = 2000;
172	gezelter	912	const int MINIBUFFERSIZE = 100;
173	gezelter	907
174	tim	936	char tempBuffer[BUFFERSIZE];
175	mmeineke	377	char writeLine[BUFFERSIZE];
176
177	tim	934	int i, k;
178	gezelter	916
179	mmeineke	787	#ifdef IS_MPI
180	gezelter	916
181	gezelter	947	/*********************************************************************
182			* Documentation? You want DOCUMENTATION?
183			*
184			* Why all the potatoes below?
185			*
186			* To make a long story short, the original version of DumpWriter
187			* worked in the most inefficient way possible. Node 0 would
188			* poke each of the node for an individual atom's formatted data
189			* as node 0 worked its way down the global index. This was particularly
190			* inefficient since the method blocked all processors at every atom
191			* (and did it twice!).
192			*
193			* An intermediate version of DumpWriter could be described from Node
194			* zero's perspective as follows:
195			*
196			* 1) Have 100 of your friends stand in a circle.
197			* 2) When you say go, have all of them start tossing potatoes at
198			* you (one at a time).
199			* 3) Catch the potatoes.
200			*
201			* It was an improvement, but MPI has buffers and caches that could
202			* best be described in this analogy as "potato nets", so there's no
203			* need to block the processors atom-by-atom.
204			*
205			* This new and improved DumpWriter works in an even more efficient
206			* way:
207			*
208			* 1) Have 100 of your friend stand in a circle.
209			* 2) When you say go, have them start tossing 5-pound bags of
210			* potatoes at you.
211			* 3) Once you've caught a friend's bag of potatoes,
212			* toss them a spud to let them know they can toss another bag.
213			*
214			* How's THAT for documentation?
215			*
216			*********************************************************************/
217
218	gezelter	916	int *potatoes;
219			int myPotato;
220
221			int nProc;
222	tim	929	int j, which_node, done, which_atom, local_index, currentIndex;
223	gezelter	916	double atomData6[6];
224			double atomData13[13];
225	gezelter	907	int isDirectional;
226			char* atomTypeString;
227	gezelter	910	char MPIatomTypeString[MINIBUFFERSIZE];
228	gezelter	916
229	mmeineke	787	#else //is_mpi
230			int nAtoms = entry_plug->n_atoms;
231			#endif //is_mpi
232
233	mmeineke	377	double q[4];
234			DirectionalAtom* dAtom;
235			Atom** atoms = entry_plug->atoms;
236	mmeineke	670	double pos[3], vel[3];
237	tim	837
238	mmeineke	377	#ifndef IS_MPI
239	tim	934
240			for(k = 0; k < outFile.size(); k++){
241			*outFile[k] << nAtoms << "\n";
242	tim	837
243	tim	934	*outFile[k] << currentTime << ";\t"
244			<< entry_plug->Hmat[0][0] << "\t"
245			<< entry_plug->Hmat[1][0] << "\t"
246			<< entry_plug->Hmat[2][0] << ";\t"
247
248			<< entry_plug->Hmat[0][1] << "\t"
249			<< entry_plug->Hmat[1][1] << "\t"
250			<< entry_plug->Hmat[2][1] << ";\t"
251	tim	837
252	tim	934	<< entry_plug->Hmat[0][2] << "\t"
253			<< entry_plug->Hmat[1][2] << "\t"
254			<< entry_plug->Hmat[2][2] << ";";
255	mmeineke	572
256	tim	934	//write out additional parameters, such as chi and eta
257			*outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
258			}
259
260	mmeineke	377	for( i=0; i<nAtoms; i++ ){
261	tim	837
262	mmeineke	670	atoms[i]->getPos(pos);
263			atoms[i]->getVel(vel);
264	mmeineke	377
265			sprintf( tempBuffer,
266			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
267			atoms[i]->getType(),
268	mmeineke	670	pos[0],
269			pos[1],
270			pos[2],
271			vel[0],
272			vel[1],
273			vel[2]);
274	mmeineke	377	strcpy( writeLine, tempBuffer );
275
276			if( atoms[i]->isDirectional() ){
277	tim	837
278	mmeineke	377	dAtom = (DirectionalAtom *)atoms[i];
279			dAtom->getQ( q );
280	tim	837
281	mmeineke	377	sprintf( tempBuffer,
282			"%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
283			q[0],
284			q[1],
285			q[2],
286			q[3],
287			dAtom->getJx(),
288			dAtom->getJy(),
289			dAtom->getJz());
290			strcat( writeLine, tempBuffer );
291			}
292			else
293			strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
294	tim	837
295	tim	934	for(k = 0; k < outFile.size(); k++)
296			*outFile[k] << writeLine;
297	mmeineke	377	}
298
299			#else // is_mpi
300	gezelter	416
301	chuckv	913	/* code to find maximum tag value */
302	tim	919
303	tim	920	int *tagub, flag, MAXTAG;
304	chuckv	913	MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);
305			if (flag) {
306	tim	920	MAXTAG = *tagub;
307	chuckv	913	} else {
308			MAXTAG = 32767;
309	gezelter	916	}
310	mmeineke	440
311			int haveError;
312
313	mmeineke	447	MPI_Status istatus;
314	gezelter	416	int *AtomToProcMap = mpiSim->getAtomToProcMap();
315	tim	837
316	mmeineke	377	// write out header and node 0's coordinates
317	tim	837
318	mmeineke	377	if( worldRank == 0 ){
319	gezelter	916
320			// Node 0 needs a list of the magic potatoes for each processor;
321
322			nProc = mpiSim->getNumberProcessors();
323			potatoes = new int[nProc];
324
325	tim	934	//write out the comment lines
326	gezelter	916	for (i = 0; i < nProc; i++)
327			potatoes[i] = 0;
328
329	tim	934	for(k = 0; k < outFile.size(); k++){
330			*outFile[k] << mpiSim->getTotAtoms() << "\n";
331	tim	837
332	tim	934	*outFile[k] << currentTime << ";\t"
333			<< entry_plug->Hmat[0][0] << "\t"
334			<< entry_plug->Hmat[1][0] << "\t"
335			<< entry_plug->Hmat[2][0] << ";\t"
336	tim	837
337	tim	934	<< entry_plug->Hmat[0][1] << "\t"
338			<< entry_plug->Hmat[1][1] << "\t"
339			<< entry_plug->Hmat[2][1] << ";\t"
340	tim	837
341	tim	934	<< entry_plug->Hmat[0][2] << "\t"
342			<< entry_plug->Hmat[1][2] << "\t"
343			<< entry_plug->Hmat[2][2] << ";";
344
345			*outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
346			}
347	tim	837
348	tim	929	currentIndex = 0;
349	tim	934
350	gezelter	416	for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) {
351	chuckv	913
352	gezelter	417	// Get the Node number which has this atom;
353	chuckv	913
354	tim	837	which_node = AtomToProcMap[i];
355	chuckv	913
356	gezelter	907	if (which_node != 0) {
357	gezelter	916
358			if (potatoes[which_node] + 3 >= MAXTAG) {
359			// The potato was going to exceed the maximum value,
360			// so wrap this processor potato back to 0:
361
362			potatoes[which_node] = 0;
363			MPI_Send(0, 1, MPI_INT, which_node, 0, MPI_COMM_WORLD);
364
365			}
366
367			myPotato = potatoes[which_node];
368	gezelter	907
369	gezelter	910	MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, which_node,
370	gezelter	916	myPotato, MPI_COMM_WORLD, &istatus);
371	gezelter	907
372	tim	920	atomTypeString = MPIatomTypeString;
373
374	gezelter	916	myPotato++;
375
376	gezelter	907	MPI_Recv(&isDirectional, 1, MPI_INT, which_node,
377	gezelter	916	myPotato, MPI_COMM_WORLD, &istatus);
378
379			myPotato++;
380	gezelter	907
381	gezelter	916	if (isDirectional) {
382			MPI_Recv(atomData13, 13, MPI_DOUBLE, which_node,
383			myPotato, MPI_COMM_WORLD, &istatus);
384			} else {
385			MPI_Recv(atomData6, 6, MPI_DOUBLE, which_node,
386			myPotato, MPI_COMM_WORLD, &istatus);
387	gezelter	907	}
388	gezelter	916
389			myPotato++;
390			potatoes[which_node] = myPotato;
391	gezelter	907
392			} else {
393
394	tim	934	haveError = 0;
395	chuckv	949	which_atom = i;
396	gezelter	916
397	chuckv	949	//local_index = -1;
398
399			//for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
400			// if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
401			//}
402
403			//if (local_index != -1) {
404
405			local_index = indexArray[currentIndex].first;
406	gezelter	916
407	chuckv	949	if (which_atom == indexArray[currentIndex].second) {
408
409			atomTypeString = atoms[local_index]->getType();
410
411			atoms[local_index]->getPos(pos);
412			atoms[local_index]->getVel(vel);
413
414	gezelter	916	atomData6[0] = pos[0];
415			atomData6[1] = pos[1];
416			atomData6[2] = pos[2];
417	tim	837
418	gezelter	916	atomData6[3] = vel[0];
419			atomData6[4] = vel[1];
420			atomData6[5] = vel[2];
421	gezelter	907
422			isDirectional = 0;
423
424	chuckv	436	if( atoms[local_index]->isDirectional() ){
425	tim	837
426	gezelter	907	isDirectional = 1;
427
428	chuckv	436	dAtom = (DirectionalAtom *)atoms[local_index];
429			dAtom->getQ( q );
430	gezelter	916
431			for (int j = 0; j < 6 ; j++)
432			atomData13[j] = atomData6[j];
433	gezelter	907
434	gezelter	916	atomData13[6] = q[0];
435			atomData13[7] = q[1];
436			atomData13[8] = q[2];
437			atomData13[9] = q[3];
438
439			atomData13[10] = dAtom->getJx();
440			atomData13[11] = dAtom->getJy();
441			atomData13[12] = dAtom->getJz();
442	gezelter	907	}
443	gezelter	916
444	gezelter	907	} else {
445	chuckv	949	sprintf(painCave.errMsg,
446			"Atom %d not found on processor %d, currentIndex = %d, local_index = %d\n",
447			which_atom, worldRank, currentIndex, local_index );
448			haveError= 1;
449			simError();
450			}
451	gezelter	916
452	tim	934	if(haveError) DieDieDie();
453	gezelter	916
454	chuckv	949	currentIndex++;
455	tim	926	}
456			// If we've survived to here, format the line:
457
458			if (!isDirectional) {
459
460	tim	934	sprintf( writeLine,
461	chuckv	949	"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
462			atomTypeString,
463			atomData6[0],
464			atomData6[1],
465			atomData6[2],
466			atomData6[3],
467			atomData6[4],
468			atomData6[5]);
469	tim	926
470	chuckv	949	strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
471
472	tim	926	} else {
473
474	chuckv	949	sprintf( writeLine,
475			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
476			atomTypeString,
477			atomData13[0],
478			atomData13[1],
479			atomData13[2],
480			atomData13[3],
481			atomData13[4],
482			atomData13[5],
483			atomData13[6],
484			atomData13[7],
485			atomData13[8],
486			atomData13[9],
487			atomData13[10],
488			atomData13[11],
489			atomData13[12]);
490	gezelter	907
491			}
492	tim	926
493	tim	934	for(k = 0; k < outFile.size(); k++)
494			*outFile[k] << writeLine;
495	mmeineke	377	}
496	tim	926
497	tim	934	for(k = 0; k < outFile.size(); k++)
498			outFile[k]->flush();
499
500	gezelter	907	sprintf( checkPointMsg,
501			"Sucessfully took a dump.\n");
502	chuckv	949
503	gezelter	907	MPIcheckPoint();
504	chuckv	949
505	tim	919	delete[] potatoes;
506	chuckv	949
507	gezelter	415	} else {
508	tim	837
509	gezelter	907	// worldRank != 0, so I'm a remote node.
510	gezelter	916
511			// Set my magic potato to 0:
512
513			myPotato = 0;
514	tim	929	currentIndex = 0;
515	gezelter	907
516			for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) {
517
518			// Am I the node which has this atom?
519
520			if (AtomToProcMap[i] == worldRank) {
521	tim	837
522	gezelter	916	if (myPotato + 3 >= MAXTAG) {
523	chuckv	949
524	gezelter	916	// The potato was going to exceed the maximum value,
525			// so wrap this processor potato back to 0 (and block until
526			// node 0 says we can go:
527	chuckv	949
528	gezelter	916	MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus);
529
530			}
531	tim	919	which_atom = i;
532	chuckv	949
533			//local_index = -1;
534
535			//for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
536			// if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
537			//}
538
539			//if (local_index != -1) {
540
541			local_index = indexArray[currentIndex].first;
542	tim	920
543	chuckv	949	if (which_atom == indexArray[currentIndex].second) {
544	gezelter	907
545			atomTypeString = atoms[local_index]->getType();
546	chuckv	949
547			atoms[local_index]->getPos(pos);
548			atoms[local_index]->getVel(vel);
549
550	gezelter	916	atomData6[0] = pos[0];
551			atomData6[1] = pos[1];
552			atomData6[2] = pos[2];
553	tim	837
554	gezelter	916	atomData6[3] = vel[0];
555			atomData6[4] = vel[1];
556			atomData6[5] = vel[2];
557	gezelter	907
558			isDirectional = 0;
559	tim	837
560	gezelter	907	if( atoms[local_index]->isDirectional() ){
561	mmeineke	377
562	gezelter	907	isDirectional = 1;
563
564			dAtom = (DirectionalAtom *)atoms[local_index];
565			dAtom->getQ( q );
566
567	gezelter	916	for (int j = 0; j < 6 ; j++)
568			atomData13[j] = atomData6[j];
569
570			atomData13[6] = q[0];
571			atomData13[7] = q[1];
572			atomData13[8] = q[2];
573			atomData13[9] = q[3];
574	tim	934
575	gezelter	916	atomData13[10] = dAtom->getJx();
576			atomData13[11] = dAtom->getJy();
577			atomData13[12] = dAtom->getJz();
578	gezelter	907	}
579	tim	837
580	gezelter	907	} else {
581	chuckv	949	sprintf(painCave.errMsg,
582			"Atom %d not found on processor %d, currentIndex = %d, local_index = %d\n",
583			which_atom, worldRank, currentIndex, local_index );
584			haveError= 1;
585			simError();
586			}
587
588	gezelter	916	strncpy(MPIatomTypeString, atomTypeString, MINIBUFFERSIZE);
589	tim	837
590	gezelter	916	// null terminate the string before sending (just in case):
591			MPIatomTypeString[MINIBUFFERSIZE-1] = '\0';
592	tim	837
593	gezelter	910	MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
594	tim	934	myPotato, MPI_COMM_WORLD);
595	gezelter	907
596	gezelter	916	myPotato++;
597
598	gezelter	907	MPI_Send(&isDirectional, 1, MPI_INT, 0,
599	tim	934	myPotato, MPI_COMM_WORLD);
600	gezelter	907
601	gezelter	916	myPotato++;
602
603	gezelter	907	if (isDirectional) {
604
605	gezelter	916	MPI_Send(atomData13, 13, MPI_DOUBLE, 0,
606			myPotato, MPI_COMM_WORLD);
607	gezelter	907
608	gezelter	916	} else {
609
610			MPI_Send(atomData6, 6, MPI_DOUBLE, 0,
611			myPotato, MPI_COMM_WORLD);
612	gezelter	907	}
613	gezelter	916
614	tim	929	myPotato++;
615			currentIndex++;
616	mmeineke	377	}
617	gezelter	907	}
618	mmeineke	440
619	gezelter	907	sprintf( checkPointMsg,
620	gezelter	916	"Sucessfully took a dump.\n");
621	gezelter	907	MPIcheckPoint();
622
623	gezelter	916	}
624	gezelter	907
625	mmeineke	377	#endif // is_mpi
626			}
627	mmeineke	440
628			#ifdef IS_MPI
629
630			// a couple of functions to let us escape the write loop
631
632	gezelter	907	void dWrite::DieDieDie( void ){
633	tim	837
634	mmeineke	440	MPI_Finalize();
635			exit (0);
636			}
637
638			#endif //is_mpi