--- branches/mmeineke/OOPSE/libmdtools/DumpWriter.cpp	2003/03/21 17:42:12	377
+++ trunk/OOPSE/libmdtools/DumpWriter.cpp	2003/03/28 19:30:59	434
@@ -4,17 +4,14 @@
 
 #ifdef IS_MPI
 #include <mpi.h>
+#include <mpi++.h>
 #include "mpiSimulation.hpp"
-#define TAKE_THIS_TAG 0
+#define TAKE_THIS_TAG 1
 #endif //is_mpi
 
 #include "ReadWrite.hpp"
 #include "simError.h"
 
-
-
-
-
 DumpWriter::DumpWriter( SimInfo* the_entry_plug ){
 
   entry_plug = the_entry_plug;
@@ -68,7 +65,7 @@ void DumpWriter::writeDump( double currentTime ){
   char tempBuffer[BUFFERSIZE];
   char writeLine[BUFFERSIZE];
 
-  int i;
+  int i, j, which_node, done, game_over, which_atom, local_index;
   double q[4];
   DirectionalAtom* dAtom;
   int nAtoms = entry_plug->n_atoms;
@@ -123,192 +120,155 @@ void DumpWriter::writeDump( double currentTime ){
 
 #else // is_mpi
 
-  int masterIndex;
-  int nodeAtomsStart;
-  int nodeAtomsEnd;
-  int mpiErr;
-  int sendError;
-  int procIndex;
-    
-  MPI_Status istatus[MPI_STATUS_SIZE];
-
-    
+  MPI::Status istatus;
+  int *AtomToProcMap = mpiSim->getAtomToProcMap();
+  
   // write out header and node 0's coordinates
-
+  
   if( worldRank == 0 ){
     outFile << mpiSim->getTotAtoms() << "\n";
-      
+    
     outFile << currentTime << "\t" 
 	    << entry_plug->box_x << "\t"
 	    << entry_plug->box_y << "\t"
 	    << entry_plug->box_z << "\n";
-
-    masterIndex = 0;
-    for( i=0; i<nAtoms; i++ ){
+    outFile.flush();
+    for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) {
+      // Get the Node number which has this atom;
       
-      sprintf( tempBuffer,
-	       "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
-	       atoms[i]->getType(),
-	       atoms[i]->getX(),
-	       atoms[i]->getY(),
-	       atoms[i]->getZ(),
-	       atoms[i]->get_vx(),
-	       atoms[i]->get_vy(),
-	       atoms[i]->get_vz());
-      strcpy( writeLine, tempBuffer );
+      which_node = AtomToProcMap[i];     
+      
+      if (which_node == mpiSim->getMyNode()) {
+                sprintf( tempBuffer,
+                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+                 atoms[i]->getType(),
+                 atoms[i]->getX(),
+                 atoms[i]->getY(),
+                 atoms[i]->getZ(),
+                 atoms[i]->get_vx(),
+                 atoms[i]->get_vy(),
+                 atoms[i]->get_vz());
+        strcpy( writeLine, tempBuffer );
 	
-      if( atoms[i]->isDirectional() ){
+        if( atoms[i]->isDirectional() ){
 	  
-	dAtom = (DirectionalAtom *)atoms[i];
-	dAtom->getQ( q );
+          dAtom = (DirectionalAtom *)atoms[i];
+          dAtom->getQ( q );
 	  
-	sprintf( tempBuffer,
-		 "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
-		 q[0],
-		 q[1],
-		 q[2],
-		 q[3],
-		 dAtom->getJx(),
-		 dAtom->getJy(),
-		 dAtom->getJz());
-	strcat( writeLine, tempBuffer );
+          sprintf( tempBuffer,
+                   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+                   q[0],
+                   q[1],
+                   q[2],
+                   q[3],
+                   dAtom->getJx(),
+                   dAtom->getJy(),
+                   dAtom->getJz());
+          strcat( writeLine, tempBuffer );
+        }
+        else
+          strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );       
+        
+      } else {
+        
+	std::cerr << "node 0: sending node " << which_node << " request for atom " << i << "\n";
+        MPI::COMM_WORLD.Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG);
+	std::cerr << "node 0: sent!\n";
+        MPI::COMM_WORLD.Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node,
+                                TAKE_THIS_TAG, istatus);
+	std::cerr << "node 0: got this line: " << writeLine;
       }
-      else
-	strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
-	
+      
       outFile << writeLine;
-      masterIndex++;
+      outFile.flush();
     }
-    outFile.flush();
-  }
-
-  sprintf( checkPointMsg,
-	   "Sucessfully wrote node 0's dump configuration.\n");
-  MPIcheckPoint();
     
-  for (procIndex = 1; procIndex < mpiSim->getNumberProcessors();
-       procIndex++){
-
-    if( worldRank == 0 ){
-      
-      mpiErr = MPI_Recv(&nodeAtomsStart,1,MPI_INT,procIndex,
-			TAKE_THIS_TAG,MPI_COMM_WORLD,istatus);
-      
-      mpiErr = MPI_Recv(&nodeAtomsEnd,1,MPI_INT,procIndex,
-			TAKE_THIS_TAG,MPI_COMM_WORLD, istatus);
-      
-      // Make sure where node 0 is writing to, matches where the
-      // receiving node expects it to be.
-      
-      if (masterIndex != nodeAtomsStart){
-	sendError = 1;
-	mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,TAKE_THIS_TAG,
-			  MPI_COMM_WORLD);
-	sprintf(painCave.errMsg,
-		"DumpWriter error: atoms start index (%d) for "
-		"node %d not equal to master index (%d)",
-		nodeAtomsStart,procIndex,masterIndex );
-	painCave.isFatal = 1;
-	simError();
-      }
-      
-      sendError = 0;
-      mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,TAKE_THIS_TAG,
-			MPI_COMM_WORLD);
-      
-      // recieve the nodes writeLines
-      
-      for ( i = nodeAtomsStart; i <= nodeAtomsEnd; i++){
-	
-	mpiErr = MPI_Recv(writeLine,BUFFERSIZE,MPI_CHAR,procIndex,
-			  TAKE_THIS_TAG,MPI_COMM_WORLD,istatus );
-	
-	outFile << writeLine;
-	masterIndex++;
-      }
+    // kill everyone off:
+    game_over = -1;
+    for (j = 0; j < mpiSim->getNumberProcessors(); j++) {      
+      MPI::COMM_WORLD.Send(&game_over, 1, MPI_INT, j, TAKE_THIS_TAG);
     }
 
-    else if( worldRank == procIndex ){
-
-      nodeAtomsStart = mpiSim->getMyAtomStart();
-      nodeAtomsEnd = mpiSim->getMyAtomEnd();
-
-      mpiErr = MPI_Send(&nodeAtomsStart,1,MPI_INT,0,TAKE_THIS_TAG,
-			MPI_COMM_WORLD);
-      mpiErr = MPI_Send(&nodeAtomsEnd,1,MPI_INT,0,TAKE_THIS_TAG,
-			MPI_COMM_WORLD);
-	
-      sendError = -1;
-      mpiErr = MPI_Recv(&sendError,1,MPI_INT,0,TAKE_THIS_TAG,
-			MPI_COMM_WORLD, istatus);
-
-      if (sendError) MPIcheckPoint();
-
-      // send current node's configuration line by line.
-
-      for( i=0; i<nAtoms; i++ ){
-
-	sprintf( tempBuffer,
-		 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
-		 atoms[i]->getType(),
-		 atoms[i]->getX(),
-		 atoms[i]->getY(),
-		 atoms[i]->getZ(),
-		 atoms[i]->get_vx(),
-		 atoms[i]->get_vy(),
-		 atoms[i]->get_vz()); // check here.
-	strcpy( writeLine, tempBuffer );
+  } else {
+    
+    done = 0;
+    while (!done) {
+      std::cerr << "node: " << mpiSim->getMyNode() << " Waiting for receive \n";
+      MPI::COMM_WORLD.Recv(&which_atom, 1, MPI_INT, 0, 
+                           TAKE_THIS_TAG, istatus);
+      std::cerr << "node: " << mpiSim->getMyNode() << " got request for atom " << which_atom << "\n";
+      if (which_atom == -1) {
+        done=1;
+        continue;
+      } else {
+        local_index=-1;        
+        for (j=0; j < mpiSim->getMyNlocal(); j++) {
+          if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
+        }
+        if (local_index != -1) {
+          //format the line
+          sprintf( tempBuffer,
+                   "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+                   atoms[local_index]->getType(),
+                   atoms[local_index]->getX(),
+                   atoms[local_index]->getY(),
+                   atoms[local_index]->getZ(),
+                   atoms[local_index]->get_vx(),
+                   atoms[local_index]->get_vy(),
+                   atoms[local_index]->get_vz()); // check here.
+          strcpy( writeLine, tempBuffer );
 	  
-	if( atoms[i]->isDirectional() ){
-	    
-	  dAtom = (DirectionalAtom *)atoms[i];
-	  dAtom->getQ( q );
-	    
-	  sprintf( tempBuffer,
-		   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
-		   q[0],
-		   q[1],
-		   q[2],
-		   q[3],
-		   dAtom->getJx(),
-		   dAtom->getJy(),
-		   dAtom->getJz());
-	  strcat( writeLine, tempBuffer );
-	}
-	else
-	  strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
-	  
-	mpiErr = MPI_Send(writeLine,BUFFERSIZE,MPI_CHAR,0,TAKE_THIS_TAG,
-			  MPI_COMM_WORLD);
+          if( atoms[local_index]->isDirectional() ){
+            
+            dAtom = (DirectionalAtom *)atoms[local_index];
+            dAtom->getQ( q );
+            
+            sprintf( tempBuffer,
+                     "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+                     q[0],
+                     q[1],
+                     q[2],
+                     q[3],
+                     dAtom->getJx(),
+                     dAtom->getJy(),
+                     dAtom->getJz());
+            strcat( writeLine, tempBuffer );
+          }
+          else
+            strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
+	  std::cerr << "node: " << mpiSim->getMyNode() << " sending this line" << writeLine;
+          MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, 
+                               TAKE_THIS_TAG);
+        } else {
+          strcpy( writeLine, "ATOM NOT FOUND ON THIS PROCESSOR");
+          MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, 
+                               TAKE_THIS_TAG);
+        }
       }
     }
-      
-    sprintf(checkPointMsg,"Node %d sent dump configuration.",
-	    procIndex);
-    MPIcheckPoint();
-  }
-    
+  }  
+  outFile.flush();
+  sprintf( checkPointMsg,
+	   "Sucessfully took a dump.\n");
+  MPIcheckPoint();
 #endif // is_mpi
 }
 
-
-
 void DumpWriter::writeFinal(){
 
+  char finalName[500];
+  ofstream finalOut;
 
   const int BUFFERSIZE = 2000;
-  char tempBuffer[500];
-  char writeLine[BUFFERSIZE];
-  
-  char finalName[500];
+  char tempBuffer[BUFFERSIZE];
+  char writeLine[BUFFERSIZE];  
 
-  int i;
   double q[4];
   DirectionalAtom* dAtom;
   int nAtoms = entry_plug->n_atoms;
   Atom** atoms = entry_plug->atoms;
+  int i, j, which_node, done, game_over, which_atom, local_index;
   
-  ofstream finalOut;
   
 #ifdef IS_MPI
   if(worldRank == 0 ){
@@ -335,8 +295,7 @@ void DumpWriter::writeFinal(){
   
 #endif //is_mpi
 
-    
-
+  
 #ifndef IS_MPI
     
   finalOut << nAtoms << "\n";
@@ -344,7 +303,7 @@ void DumpWriter::writeFinal(){
   finalOut << entry_plug->box_x << "\t"
 	   << entry_plug->box_y << "\t"
 	   << entry_plug->box_z << "\n";
-    
+  
   for( i=0; i<nAtoms; i++ ){
       
     sprintf( tempBuffer,
@@ -380,172 +339,139 @@ void DumpWriter::writeFinal(){
     finalOut << writeLine;
   }
   finalOut.flush();
+  finalOut.close();
 
 #else // is_mpi
+  
+  MPI::Status istatus;
+  int *AtomToProcMap = mpiSim->getAtomToProcMap();
 
-  int masterIndex;
-  int nodeAtomsStart;
-  int nodeAtomsEnd;
-  int mpiErr;
-  int sendError;
-  int procIndex;
-    
-  MPI_Status istatus[MPI_STATUS_SIZE];
-
-    
   // write out header and node 0's coordinates
-
+  
   if( worldRank == 0 ){
     finalOut << mpiSim->getTotAtoms() << "\n";
-      
+    
     finalOut << entry_plug->box_x << "\t"
-	     << entry_plug->box_y << "\t"
-	     << entry_plug->box_z << "\n";
+	    << entry_plug->box_y << "\t"
+	    << entry_plug->box_z << "\n";
     
-    masterIndex = 0;
-    
-    for( i=0; i<nAtoms; i++ ){
+    for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) {
+      // Get the Node number which has this molecule:
       
-      sprintf( tempBuffer,
-	       "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
-	       atoms[i]->getType(),
-	       atoms[i]->getX(),
-	       atoms[i]->getY(),
-	       atoms[i]->getZ(),
-	       atoms[i]->get_vx(),
-	       atoms[i]->get_vy(),
-	       atoms[i]->get_vz());
-      strcpy( writeLine, tempBuffer );
+      which_node = AtomToProcMap[i];     
+      
+      if (which_node == mpiSim->getMyNode()) {
+        
+        sprintf( tempBuffer,
+                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+                 atoms[i]->getType(),
+                 atoms[i]->getX(),
+                 atoms[i]->getY(),
+                 atoms[i]->getZ(),
+                 atoms[i]->get_vx(),
+                 atoms[i]->get_vy(),
+                 atoms[i]->get_vz());
+        strcpy( writeLine, tempBuffer );
 	
-      if( atoms[i]->isDirectional() ){
+        if( atoms[i]->isDirectional() ){
 	  
-	dAtom = (DirectionalAtom *)atoms[i];
-	dAtom->getQ( q );
+          dAtom = (DirectionalAtom *)atoms[i];
+          dAtom->getQ( q );
 	  
-	sprintf( tempBuffer,
-		 "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
-		 q[0],
-		 q[1],
-		 q[2],
-		 q[3],
-		 dAtom->getJx(),
-		 dAtom->getJy(),
-		 dAtom->getJz());
-	strcat( writeLine, tempBuffer );
+          sprintf( tempBuffer,
+                   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+                   q[0],
+                   q[1],
+                   q[2],
+                   q[3],
+                   dAtom->getJx(),
+                   dAtom->getJy(),
+                   dAtom->getJz());
+          strcat( writeLine, tempBuffer );
+        }
+        else
+          strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );       
+        
+      } else {
+        
+        MPI::COMM_WORLD.Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG);
+        MPI::COMM_WORLD.Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node,
+                                TAKE_THIS_TAG, istatus);
       }
-      else
-	strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
-	
+      
       finalOut << writeLine;
-      masterIndex++;
     }
-    finalOut.flush();
-  }
     
-  for (procIndex = 1; procIndex < mpiSim->getNumberProcessors();
-       procIndex++){
-
-    if( worldRank == 0 ){
-	
-      mpiErr = MPI_Recv(&nodeAtomsStart,1,MPI_INT,procIndex,
-			TAKE_THIS_TAG,MPI_COMM_WORLD,istatus);
-
-      mpiErr = MPI_Recv(&nodeAtomsEnd,1,MPI_INT,procIndex,
-			TAKE_THIS_TAG,MPI_COMM_WORLD, istatus);
-	
-      // Make sure where node 0 is writing to, matches where the
-      // receiving node expects it to be.
-	
-      if (masterIndex != nodeAtomsStart){
-	sendError = 1;
-	mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,TAKE_THIS_TAG,
-			  MPI_COMM_WORLD);
-	sprintf(painCave.errMsg,
-		"DumpWriter error: atoms start index (%d) for "
-		"node %d not equal to master index (%d)",
-		nodeAtomsStart,procIndex,masterIndex );
-	painCave.isFatal = 1;
-	simError();
-      }
-	
-      sendError = 0;
-      mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,TAKE_THIS_TAG,
-			MPI_COMM_WORLD);
-
-      // recieve the nodes writeLines
-
-      for ( i = nodeAtomsStart; i <= nodeAtomsEnd; i++){
-	  
-	mpiErr = MPI_Recv(writeLine,BUFFERSIZE,MPI_CHAR,procIndex,
-			  TAKE_THIS_TAG,MPI_COMM_WORLD,istatus );
-
-	finalOut << writeLine;
-	masterIndex++;
-      }
-
-      finalOut.flush();
+    // kill everyone off:
+    game_over = -1;
+    for (j = 0; j < mpiSim->getNumberProcessors(); j++) {
+      MPI::COMM_WORLD.Send(&game_over, 1, MPI_INT, j, TAKE_THIS_TAG);
     }
 
-    else if( worldRank == procIndex ){
+  } else {
+    
+    done = 0;
+    while (!done) {
+      MPI::COMM_WORLD.Recv(&which_atom, 1, MPI_INT, 0, 
+                           TAKE_THIS_TAG, istatus);
 
-      nodeAtomsStart = mpiSim->getMyAtomStart();
-      nodeAtomsEnd = mpiSim->getMyAtomEnd();
-	
-      mpiErr = MPI_Send(&nodeAtomsStart,1,MPI_INT,0,TAKE_THIS_TAG,
-			MPI_COMM_WORLD);
-      mpiErr = MPI_Send(&nodeAtomsEnd,1,MPI_INT,0,TAKE_THIS_TAG,
-			MPI_COMM_WORLD);
-	
-      mpiErr = MPI_Recv(&sendError,1,MPI_INT,0,TAKE_THIS_TAG,
-			MPI_COMM_WORLD, istatus);
-      if (sendError) MPIcheckPoint();
+      if (which_atom == -1) {
+        done=1;
+        continue;
+      } else {
 
-      // send current node's configuration line by line.
+        local_index=-1;        
+        for (j=0; j < mpiSim->getMyNlocal(); j++) {
+          if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
+        }
+        if (local_index != -1) {
 
-      for( i=0; i<nAtoms; i++ ){
+          //format the line
+          sprintf( tempBuffer,
+                   "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+                   atoms[local_index]->getType(),
+                   atoms[local_index]->getX(),
+                   atoms[local_index]->getY(),
+                   atoms[local_index]->getZ(),
+                   atoms[local_index]->get_vx(),
+                   atoms[local_index]->get_vy(),
+                   atoms[local_index]->get_vz()); // check here.
+          strcpy( writeLine, tempBuffer );
 	  
-	sprintf( tempBuffer,
-		 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
-		 atoms[i]->getType(),
-		 atoms[i]->getX(),
-		 atoms[i]->getY(),
-		 atoms[i]->getZ(),
-		 atoms[i]->get_vx(),
-		 atoms[i]->get_vy(),
-		 atoms[i]->get_vz());
-	strcpy( writeLine, tempBuffer );
-	  
-	if( atoms[i]->isDirectional() ){
+          if( atoms[local_index]->isDirectional() ){
+            
+            dAtom = (DirectionalAtom *)atoms[local_index];
+            dAtom->getQ( q );
 	    
-	  dAtom = (DirectionalAtom *)atoms[i];
-	  dAtom->getQ( q );
-	    
-	  sprintf( tempBuffer,
-		   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
-		   q[0],
-		   q[1],
-		   q[2],
-		   q[3],
-		   dAtom->getJx(),
-		   dAtom->getJy(),
-		   dAtom->getJz());
-	  strcat( writeLine, tempBuffer );
-	}
-	else
-	  strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
-	  
-	mpiErr = MPI_Send(writeLine,BUFFERSIZE,MPI_CHAR,0,TAKE_THIS_TAG,
-			  MPI_COMM_WORLD);
+            sprintf( tempBuffer,
+                     "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+                     q[0],
+                     q[1],
+                     q[2],
+                     q[3],
+                     dAtom->getJx(),
+                     dAtom->getJy(),
+                     dAtom->getJz());
+            strcat( writeLine, tempBuffer );
+          }
+          else
+            strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
+          
+          MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, 
+                               TAKE_THIS_TAG);
+        } else {
+          strcpy( writeLine, "ATOM NOT FOUND ON THIS PROCESSOR");
+          MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, 
+                               TAKE_THIS_TAG);
+        }
       }
     }
-      
-    sprintf(checkPointMsg,"Node %d sent dump configuration.",
-	    procIndex);
-    MPIcheckPoint();
   }
+  finalOut.flush();
+  sprintf( checkPointMsg,
+	   "Sucessfully took a dump.\n");
+  MPIcheckPoint();
 
-  if( worldRank == 0 ) finalOut.close();
-
-    
+  if( worldRank == 0 ) finalOut.close();    
 #endif // is_mpi
 }