--- trunk/mdtools/md_code/DumpWriter.cpp	2002/11/26 21:04:43	189
+++ trunk/mdtools/md_code/DumpWriter.cpp	2002/12/16 21:42:14	214
@@ -1,6 +1,7 @@
 #include <cstring>
 #include <iostream>
 #include <fstream>
+#include <mpi.h>
 
 #include "ReadWrite.hpp"
 #include "simError.h"
@@ -50,66 +51,224 @@ void DumpWriter::writeDump( double currentTime ){
 }
 
 void DumpWriter::writeDump( double currentTime ){
+  
+  const int BUFFERSIZE = 2000;
+  char tempBuffer[500];
+  char writeLine[BUFFERSIZE];
 
-#ifdef IS_MPI
-  if(worldRank == 0 ){
-#endif // is_mpi
+  int i;
+  double q[4];
+  DirectionalAtom* dAtom;
+  int nAtoms = entry_plug->n_atoms;
+  Atom** atoms = entry_plug->atoms;
+    
 
-
-    int i;
-    double q[4];
-    DirectionalAtom* dAtom;
-    int nAtoms = entry_plug->n_atoms;
-    Atom** atoms = entry_plug->atoms;
+#ifndef IS_MPI
     
+  outFile << nAtoms << "\n";
     
-    outFile << nAtoms << "\n";
+  outFile << currentTime << "\t" 
+	  << entry_plug->box_x << "\t"
+	  << entry_plug->box_y << "\t"
+	  << entry_plug->box_z << "\n";
     
+  for( i=0; i<nAtoms; i++ ){
+      
+    sprintf( tempBuffer,
+	     "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+	     atoms[i]->getType(),
+	     atoms[i]->getX(),
+	     atoms[i]->getY(),
+	     atoms[i]->getZ(),
+	     atoms[i]->get_vx(),
+	     atoms[i]->get_vy(),
+	     atoms[i]->get_vz());
+    strcpy( writeLine, tempBuffer );
+
+    if( atoms[i]->isDirectional() ){
+	
+      dAtom = (DirectionalAtom *)atoms[i];
+      dAtom->getQ( q );
+	
+      sprintf( tempBuffer,
+	       "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+	       q[0],
+	       q[1],
+	       q[2],
+	       q[3],
+	       dAtom->getJx(),
+	       dAtom->getJy(),
+	       dAtom->getJz());
+      strcat( writeLine, tempBuffer );
+    }
+    else
+      strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
+      
+    outFile << writeLine;
+  }
+  outFile.flush();
+
+#else // is_mpi
+
+  int masterIndex;
+  int nodeAtomsStart;
+  int nodeAtomsEnd;
+  int mpiErr;
+  int sendError;
+  int procIndex;
+    
+  MPI_Status istatus[MPI_STATUS_SIZE];
+
+    
+  // write out header and node 0's coordinates
+
+  if( worldRank == 0 ){
+    outFile << entry_plug->mpiSim->getTotAtoms() << "\n";
+      
     outFile << currentTime << "\t" 
 	    << entry_plug->box_x << "\t"
 	    << entry_plug->box_y << "\t"
 	    << entry_plug->box_z << "\n";
-    
+
+    masterIndex = 0;
     for( i=0; i<nAtoms; i++ ){
       
-      outFile 
-	<< atoms[i]->getType() << "\t"
-	<< atoms[i]->getX() << "\t"
-	<< atoms[i]->getY() << "\t"
-	<< atoms[i]->getZ() << "\t"
-	<< atoms[i]->get_vx() << "\t"
-	<< atoms[i]->get_vy() << "\t"
-	<< atoms[i]->get_vz() << "\t";
-      
-      if( atoms[i]->isDirectional() ){
+      sprintf( tempBuffer,
+	       "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+	       atoms[i]->getType(),
+	       atoms[i]->getX(),
+	       atoms[i]->getY(),
+	       atoms[i]->getZ(),
+	       atoms[i]->get_vx(),
+	       atoms[i]->get_vy(),
+	       atoms[i]->get_vz());
+      strcpy( writeLine, tempBuffer );
 	
+      if( atoms[i]->isDirectional() ){
+	  
 	dAtom = (DirectionalAtom *)atoms[i];
 	dAtom->getQ( q );
+	  
+	sprintf( tempBuffer,
+		 "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+		 q[0],
+		 q[1],
+		 q[2],
+		 q[3],
+		 dAtom->getJx(),
+		 dAtom->getJy(),
+		 dAtom->getJz());
+	strcat( writeLine, tempBuffer );
+      }
+      else
+	strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
 	
-	outFile 
-	  << q[0] << "\t"
-	  << q[1] << "\t"
-	  << q[2] << "\t"
-	  << q[3] << "\t"
-	  << dAtom->getJx() << "\t"
-	  << dAtom->getJy() << "\t"
-	  << dAtom->getJz() << "\n";
+      outfile << writeLine;
+      masterIndex++;
+    }
+    outFile.flush();
+  }
+    
+  for (procIndex = 1; procIndex < entry_plug->mpiSim->getNumberProcessors();
+       procIndex++){
+
+    if( worldRank == 0 ){
+	
+      mpiErr = MPI_Recv(&nodeAtomsStart,1,MPI_INT,procIndex,
+			MPI_ANY_TAG,MPI_COMM_WORLD,istatus);
+
+      mpiErr = MPI_Recv(&nodeAtomsEnd,1,MPI_INT,procIndex,
+			MPI_ANY_TAG,MPI_COMM_WORLD, istatus);
+	
+      // Make sure where node 0 is writing to, matches where the
+      // receiving node expects it to be.
+	
+      if (masterIndex != nodeAtomsStart){
+	sendError = 1;
+	mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,MPI_ANY_TAG,
+			  MPI_COMM_WORLD);
+	sprintf(painCave.errMsg,
+		"DumpWriter error: atoms start index (%d) for "
+		"node %d not equal to master index (%d)",
+		nodeAtomsStart,procIndex,masterIndex );
+	painCave.isFatal = 1;
+	simError();
       }
-      else{
-	outFile 
-	  << 0.0 << "\t"
-	  << 0.0 << "\t"
-	  << 0.0 << "\t"
-	  << 0.0 << "\t"
-	  << 0.0 << "\t"
-	  << 0.0 << "\t"
-	  << 0.0 << "\n";
+	
+      sendError = 0;
+      mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,MPI_ANY_TAG,
+			MPI_COMM_WORLD);
+
+      // recieve the nodes writeLines
+
+      for ( i = nodeAtomStart; i <= nodeAtomEnd, i++){
+	  
+	mpiErr = MPI_Recv(&read_buffer,BUFFERSIZE,MPI_CHAR,procIndex,
+			  MPI_ANY_TAG,MPI_COMM_WORLD,istatus );
+
+	outFile << writeLine;
+	masterIndex++;
       }
     }
-    outFile.flush();
 
-#ifdef IS_MPI
+    else if( worldRank == procIndex ){
+
+      nodeAtomStart = entry_plug->mpiSim->getMyAtomStart();
+      nodeAtomEnd = entry_plug->mpiSim->getMyAtomEnd();
+	
+      mpiErr = MPI_Send(&nodeAtomsStart,1,MPI_INT,0,MPI_ANY_TAG,
+			MPI_COMM_WORLD);
+      mpiErr = MPI_Send(&nodeAtomsEnd,1,MPI_INT,0,MPI_ANY_TAG,
+			MPI_COMM_WORLD);
+	
+      mpiErr = MPI_Recv(&sendError,1,MPI_INT,0,MPI_ANY_TAG,
+			MPI_COMM_WORLD, istatus);
+      if (sendError) mpiCheckpoint();
+
+      // send current node's configuration line by line.
+
+      for( i=0; i<nAtoms; i++ ){
+	  
+	sprintf( tempBuffer,
+		 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+		 atoms[i]->getType(),
+		 atoms[i]->getX(),
+		 atoms[i]->getY(),
+		 atoms[i]->getZ(),
+		 atoms[i]->get_vx(),
+		 atoms[i]->get_vy(),
+		 atoms[i]->get_vz());
+	strcpy( writeLine, tempBuffer );
+	  
+	if( atoms[i]->isDirectional() ){
+	    
+	  dAtom = (DirectionalAtom *)atoms[i];
+	  dAtom->getQ( q );
+	    
+	  sprintf( tempBuffer,
+		   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+		   q[0],
+		   q[1],
+		   q[2],
+		   q[3],
+		   dAtom->getJx(),
+		   dAtom->getJy(),
+		   dAtom->getJz());
+	  strcat( writeLine, tempBuffer );
+	}
+	else
+	  strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
+	  
+	mpiErr = MPI_Send(writeLine,BUFFERSIZE,MPI_CHAR,0,MPI_ANY_TAG,
+			  MPI_COMM_WORLD);
+      }
+    }
+      
+    sprintf(checkPointMsg,"Node %d sent dump configuration.",
+	    procIndex);
+    mpiCheckPoint();
   }
+    
 #endif // is_mpi
 }
 
@@ -117,14 +276,28 @@ void DumpWriter::writeFinal(){
 
 void DumpWriter::writeFinal(){
 
+
+  const int BUFFERSIZE = 2000;
+  char tempBuffer[500];
+  char writeLine[BUFFERSIZE];
+  
+  char finalName[500];
+
+  int i;
+  double q[4];
+  DirectionalAtom* dAtom;
+  int nAtoms = entry_plug->n_atoms;
+  Atom** atoms = entry_plug->atoms;
+  
+  ofstream finalOut;
+  
 #ifdef IS_MPI
   if(worldRank == 0 ){
 #endif // is_mpi
-
-    char finalName[500];
+    
     strcpy( finalName, entry_plug->finalName );
-
-    ofstream finalOut( finalName );
+    
+    finalOut.open( finalName, ios::out | ios::trunc );
     if( !finalOut ){
       sprintf( painCave.errMsg,
 	       "Could not open \"%s\" for final dump output.\n",
@@ -135,60 +308,226 @@ void DumpWriter::writeFinal(){
     
     // finalOut.setf( ios::scientific );
     
+#ifdef IS_MPI
+  }
+  
+  sprintf(checkPointMsg,"Opened file for final configuration\n",procIndex);
+  mpiCheckPoint();  
+  
+#endif //is_mpi
+
     
-    int i;
-    double q[4];
-    DirectionalAtom* dAtom;
-    int nAtoms = entry_plug->n_atoms;
-    Atom** atoms = entry_plug->atoms;
+
+#ifndef IS_MPI
     
+  finalOut << nAtoms << "\n";
     
-    finalOut << nAtoms << "\n";
+  finalOut << currentTime << "\t" 
+	  << entry_plug->box_x << "\t"
+	  << entry_plug->box_y << "\t"
+	  << entry_plug->box_z << "\n";
     
-    finalOut << 0.0 << "\t" 
-	     << entry_plug->box_x << "\t"
-	     << entry_plug->box_y << "\t"
-	     << entry_plug->box_z << "\n";
+  for( i=0; i<nAtoms; i++ ){
+      
+    sprintf( tempBuffer,
+	     "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+	     atoms[i]->getType(),
+	     atoms[i]->getX(),
+	     atoms[i]->getY(),
+	     atoms[i]->getZ(),
+	     atoms[i]->get_vx(),
+	     atoms[i]->get_vy(),
+	     atoms[i]->get_vz());
+    strcpy( writeLine, tempBuffer );
+
+    if( atoms[i]->isDirectional() ){
+	
+      dAtom = (DirectionalAtom *)atoms[i];
+      dAtom->getQ( q );
+	
+      sprintf( tempBuffer,
+	       "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+	       q[0],
+	       q[1],
+	       q[2],
+	       q[3],
+	       dAtom->getJx(),
+	       dAtom->getJy(),
+	       dAtom->getJz());
+      strcat( writeLine, tempBuffer );
+    }
+    else
+      strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
+      
+    finalOut << writeLine;
+  }
+  finalOut.flush();
+
+#else // is_mpi
+
+  int masterIndex;
+  int nodeAtomsStart;
+  int nodeAtomsEnd;
+  int mpiErr;
+  int sendError;
+  int procIndex;
     
+  MPI_Status istatus[MPI_STATUS_SIZE];
+
+    
+  // write out header and node 0's coordinates
+
+  if( worldRank == 0 ){
+    finalOut << entry_plug->mpiSim->getTotAtoms() << "\n";
+      
+    finalOut << currentTime << "\t" 
+	    << entry_plug->box_x << "\t"
+	    << entry_plug->box_y << "\t"
+	    << entry_plug->box_z << "\n";
+
+    masterIndex = 0;
     for( i=0; i<nAtoms; i++ ){
       
-      finalOut 
-	<< atoms[i]->getType() << "\t"
-	<< atoms[i]->getX() << "\t"
-	<< atoms[i]->getY() << "\t"
-	<< atoms[i]->getZ() << "\t"
-	<< atoms[i]->get_vx() << "\t"
-	<< atoms[i]->get_vy() << "\t"
-	<< atoms[i]->get_vz() << "\t";
-      
-      if( atoms[i]->isDirectional() ){
+      sprintf( tempBuffer,
+	       "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+	       atoms[i]->getType(),
+	       atoms[i]->getX(),
+	       atoms[i]->getY(),
+	       atoms[i]->getZ(),
+	       atoms[i]->get_vx(),
+	       atoms[i]->get_vy(),
+	       atoms[i]->get_vz());
+      strcpy( writeLine, tempBuffer );
 	
+      if( atoms[i]->isDirectional() ){
+	  
 	dAtom = (DirectionalAtom *)atoms[i];
 	dAtom->getQ( q );
+	  
+	sprintf( tempBuffer,
+		 "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+		 q[0],
+		 q[1],
+		 q[2],
+		 q[3],
+		 dAtom->getJx(),
+		 dAtom->getJy(),
+		 dAtom->getJz());
+	strcat( writeLine, tempBuffer );
+      }
+      else
+	strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
 	
-	finalOut 
-	  << q[0] << "\t"
-	  << q[1] << "\t"
-	  << q[2] << "\t"
-	  << q[3] << "\t"
-	  << dAtom->getJx() << "\t"
-	  << dAtom->getJy() << "\t"
-	  << dAtom->getJz() << "\n";
+      outfile << writeLine;
+      masterIndex++;
+    }
+    finalOut.flush();
+  }
+    
+  for (procIndex = 1; procIndex < entry_plug->mpiSim->getNumberProcessors();
+       procIndex++){
+
+    if( worldRank == 0 ){
+	
+      mpiErr = MPI_Recv(&nodeAtomsStart,1,MPI_INT,procIndex,
+			MPI_ANY_TAG,MPI_COMM_WORLD,istatus);
+
+      mpiErr = MPI_Recv(&nodeAtomsEnd,1,MPI_INT,procIndex,
+			MPI_ANY_TAG,MPI_COMM_WORLD, istatus);
+	
+      // Make sure where node 0 is writing to, matches where the
+      // receiving node expects it to be.
+	
+      if (masterIndex != nodeAtomsStart){
+	sendError = 1;
+	mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,MPI_ANY_TAG,
+			  MPI_COMM_WORLD);
+	sprintf(painCave.errMsg,
+		"DumpWriter error: atoms start index (%d) for "
+		"node %d not equal to master index (%d)",
+		nodeAtomsStart,procIndex,masterIndex );
+	painCave.isFatal = 1;
+	simError();
       }
-      else{
-	finalOut 
-	  << 0.0 << "\t"
-	  << 0.0 << "\t"
-	  << 0.0 << "\t"
-	  << 0.0 << "\t"
-	  << 0.0 << "\t"
-	  << 0.0 << "\t"
-	  << 0.0 << "\n";
+	
+      sendError = 0;
+      mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,MPI_ANY_TAG,
+			MPI_COMM_WORLD);
+
+      // recieve the nodes writeLines
+
+      for ( i = nodeAtomStart; i <= nodeAtomEnd, i++){
+	  
+	mpiErr = MPI_Recv(&read_buffer,BUFFERSIZE,MPI_CHAR,procIndex,
+			  MPI_ANY_TAG,MPI_COMM_WORLD,istatus );
+
+	finalOut << writeLine;
+	masterIndex++;
       }
+
+      finalOut.flush();
     }
-    finalOut.close();
-    
-#ifdef IS_MPI
+
+    else if( worldRank == procIndex ){
+
+      nodeAtomStart = entry_plug->mpiSim->getMyAtomStart();
+      nodeAtomEnd = entry_plug->mpiSim->getMyAtomEnd();
+	
+      mpiErr = MPI_Send(&nodeAtomsStart,1,MPI_INT,0,MPI_ANY_TAG,
+			MPI_COMM_WORLD);
+      mpiErr = MPI_Send(&nodeAtomsEnd,1,MPI_INT,0,MPI_ANY_TAG,
+			MPI_COMM_WORLD);
+	
+      mpiErr = MPI_Recv(&sendError,1,MPI_INT,0,MPI_ANY_TAG,
+			MPI_COMM_WORLD, istatus);
+      if (sendError) mpiCheckpoint();
+
+      // send current node's configuration line by line.
+
+      for( i=0; i<nAtoms; i++ ){
+	  
+	sprintf( tempBuffer,
+		 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+		 atoms[i]->getType(),
+		 atoms[i]->getX(),
+		 atoms[i]->getY(),
+		 atoms[i]->getZ(),
+		 atoms[i]->get_vx(),
+		 atoms[i]->get_vy(),
+		 atoms[i]->get_vz());
+	strcpy( writeLine, tempBuffer );
+	  
+	if( atoms[i]->isDirectional() ){
+	    
+	  dAtom = (DirectionalAtom *)atoms[i];
+	  dAtom->getQ( q );
+	    
+	  sprintf( tempBuffer,
+		   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+		   q[0],
+		   q[1],
+		   q[2],
+		   q[3],
+		   dAtom->getJx(),
+		   dAtom->getJy(),
+		   dAtom->getJz());
+	  strcat( writeLine, tempBuffer );
+	}
+	else
+	  strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
+	  
+	mpiErr = MPI_Send(writeLine,BUFFERSIZE,MPI_CHAR,0,MPI_ANY_TAG,
+			  MPI_COMM_WORLD);
+      }
+    }
+      
+    sprintf(checkPointMsg,"Node %d sent dump configuration.",
+	    procIndex);
+    mpiCheckPoint();
   }
+
+  if( worldRank == 0 ) finalOut.close();
+
+    
 #endif // is_mpi
 }