--- trunk/OOPSE/libmdtools/DumpWriter.cpp	2003/03/28 19:30:59	434
+++ trunk/OOPSE/libmdtools/DumpWriter.cpp	2003/04/03 20:21:54	447
@@ -4,9 +4,16 @@
 
 #ifdef IS_MPI
 #include <mpi.h>
-#include <mpi++.h>
 #include "mpiSimulation.hpp"
-#define TAKE_THIS_TAG 1
+#define TAKE_THIS_TAG_CHAR 1
+#define TAKE_THIS_TAG_INT 2
+
+namespace dWrite{
+  void nodeZeroError( void );
+  void anonymousNodeDie( void );
+}
+
+using namespace dWrite;
 #endif //is_mpi
 
 #include "ReadWrite.hpp"
@@ -120,7 +127,13 @@ void DumpWriter::writeDump( double currentTime ){
 
 #else // is_mpi
 
-  MPI::Status istatus;
+  // first thing first, suspend fatalities.
+  painCave.isEventLoop = 1;
+
+  int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone
+  int haveError;
+
+  MPI_Status istatus;
   int *AtomToProcMap = mpiSim->getAtomToProcMap();
   
   // write out header and node 0's coordinates
@@ -138,71 +151,12 @@ void DumpWriter::writeDump( double currentTime ){
       
       which_node = AtomToProcMap[i];     
       
-      if (which_node == mpiSim->getMyNode()) {
-                sprintf( tempBuffer,
-                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
-                 atoms[i]->getType(),
-                 atoms[i]->getX(),
-                 atoms[i]->getY(),
-                 atoms[i]->getZ(),
-                 atoms[i]->get_vx(),
-                 atoms[i]->get_vy(),
-                 atoms[i]->get_vz());
-        strcpy( writeLine, tempBuffer );
+      if (which_node == 0 ) {
 	
-        if( atoms[i]->isDirectional() ){
-	  
-          dAtom = (DirectionalAtom *)atoms[i];
-          dAtom->getQ( q );
-	  
-          sprintf( tempBuffer,
-                   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
-                   q[0],
-                   q[1],
-                   q[2],
-                   q[3],
-                   dAtom->getJx(),
-                   dAtom->getJy(),
-                   dAtom->getJz());
-          strcat( writeLine, tempBuffer );
-        }
-        else
-          strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );       
-        
-      } else {
-        
-	std::cerr << "node 0: sending node " << which_node << " request for atom " << i << "\n";
-        MPI::COMM_WORLD.Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG);
-	std::cerr << "node 0: sent!\n";
-        MPI::COMM_WORLD.Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node,
-                                TAKE_THIS_TAG, istatus);
-	std::cerr << "node 0: got this line: " << writeLine;
-      }
-      
-      outFile << writeLine;
-      outFile.flush();
-    }
-    
-    // kill everyone off:
-    game_over = -1;
-    for (j = 0; j < mpiSim->getNumberProcessors(); j++) {      
-      MPI::COMM_WORLD.Send(&game_over, 1, MPI_INT, j, TAKE_THIS_TAG);
-    }
-
-  } else {
-    
-    done = 0;
-    while (!done) {
-      std::cerr << "node: " << mpiSim->getMyNode() << " Waiting for receive \n";
-      MPI::COMM_WORLD.Recv(&which_atom, 1, MPI_INT, 0, 
-                           TAKE_THIS_TAG, istatus);
-      std::cerr << "node: " << mpiSim->getMyNode() << " got request for atom " << which_atom << "\n";
-      if (which_atom == -1) {
-        done=1;
-        continue;
-      } else {
-        local_index=-1;        
-        for (j=0; j < mpiSim->getMyNlocal(); j++) {
+	haveError = 0;
+	which_atom = i;
+	local_index=-1;        
+        for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
           if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
         }
         if (local_index != -1) {
@@ -233,24 +187,125 @@ void DumpWriter::writeDump( double currentTime ){
                      dAtom->getJy(),
                      dAtom->getJz());
             strcat( writeLine, tempBuffer );
-          }
-          else
-            strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
-	  std::cerr << "node: " << mpiSim->getMyNode() << " sending this line" << writeLine;
-          MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, 
-                               TAKE_THIS_TAG);
-        } else {
-          strcpy( writeLine, "ATOM NOT FOUND ON THIS PROCESSOR");
-          MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, 
-                               TAKE_THIS_TAG);
-        }
+	    
+	  }
+	  else
+	    strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );       
+	}
+	else {
+	  sprintf(painCave.errMsg,
+		  "Atom %d not found on processor %d\n",
+		  i, worldRank );
+	  haveError= 1;
+	  simError();
+	} 
+	
+	if(haveError) nodeZeroError();
+
       }
+      else {
+	myStatus = 1;
+	MPI_Send(&myStatus, 1, MPI_INT, which_node, 
+		 TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
+        MPI_Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG_INT, 
+		 MPI_COMM_WORLD);
+        MPI_Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node,
+		 TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD, &istatus);
+        MPI_Recv(&myStatus, 1, MPI_INT, which_node, 
+		 TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
+	
+	if(!myStatus) nodeZeroError();
+
+      }
+      
+      outFile << writeLine;
+      outFile.flush();
     }
+    
+    // kill everyone off:
+    myStatus = -1;
+    for (j = 0; j < mpiSim->getNumberProcessors(); j++) {      
+      MPI_Send(&myStatus, 1, MPI_INT, j, 
+	       TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
+    }
+
+  } else {
+    
+    done = 0;
+    while (!done) {
+      
+      MPI_Recv(&myStatus, 1, MPI_INT, 0, 
+	       TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
+
+      if(!myStatus) anonymousNodeDie();
+      
+      if(myStatus < 0) break;
+
+      MPI_Recv(&which_atom, 1, MPI_INT, 0, 
+	       TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
+      
+      myStatus = 1;
+      local_index=-1;        
+      for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
+	if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
+      }
+      if (local_index != -1) {
+	//format the line
+	sprintf( tempBuffer,
+		 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+		 atoms[local_index]->getType(),
+		 atoms[local_index]->getX(),
+		 atoms[local_index]->getY(),
+		 atoms[local_index]->getZ(),
+		 atoms[local_index]->get_vx(),
+		 atoms[local_index]->get_vy(),
+		 atoms[local_index]->get_vz()); // check here.
+	strcpy( writeLine, tempBuffer );
+	
+	if( atoms[local_index]->isDirectional() ){
+	  
+	  dAtom = (DirectionalAtom *)atoms[local_index];
+	  dAtom->getQ( q );
+	  
+	  sprintf( tempBuffer,
+		   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+		   q[0],
+		   q[1],
+		   q[2],
+		   q[3],
+		   dAtom->getJx(),
+		   dAtom->getJy(),
+		   dAtom->getJz());
+	  strcat( writeLine, tempBuffer );
+	}
+	else{
+	  strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
+	}
+      } 
+      else {
+	sprintf(painCave.errMsg,
+		"Atom %d not found on processor %d\n",
+		which_atom, worldRank );
+	myStatus = 0;
+	simError();
+
+	strcpy( writeLine, "Hello, I'm an error.\n");
+      }
+
+      MPI_Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, 
+	       TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD);
+      MPI_Send( &myStatus, 1, MPI_INT, 0, 
+		TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
+    }
   }  
   outFile.flush();
   sprintf( checkPointMsg,
 	   "Sucessfully took a dump.\n");
   MPIcheckPoint();
+
+ // last  thing last, enable  fatalities.
+  painCave.isEventLoop = 0;
+
 #endif // is_mpi
 }
 
@@ -343,11 +398,18 @@ void DumpWriter::writeFinal(){
 
 #else // is_mpi
   
-  MPI::Status istatus;
+  // first thing first, suspend fatalities.
+  painCave.isEventLoop = 1;
+
+  int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone
+  int haveError;
+
+  MPI_Status istatus;
   int *AtomToProcMap = mpiSim->getAtomToProcMap();
 
   // write out header and node 0's coordinates
   
+  haveError = 0;
   if( worldRank == 0 ){
     finalOut << mpiSim->getTotAtoms() << "\n";
     
@@ -361,117 +423,183 @@ void DumpWriter::writeFinal(){
       which_node = AtomToProcMap[i];     
       
       if (which_node == mpiSim->getMyNode()) {
-        
-        sprintf( tempBuffer,
-                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
-                 atoms[i]->getType(),
-                 atoms[i]->getX(),
-                 atoms[i]->getY(),
-                 atoms[i]->getZ(),
-                 atoms[i]->get_vx(),
-                 atoms[i]->get_vy(),
-                 atoms[i]->get_vz());
-        strcpy( writeLine, tempBuffer );
-	
-        if( atoms[i]->isDirectional() ){
-	  
-          dAtom = (DirectionalAtom *)atoms[i];
-          dAtom->getQ( q );
-	  
-          sprintf( tempBuffer,
-                   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
-                   q[0],
-                   q[1],
-                   q[2],
-                   q[3],
-                   dAtom->getJx(),
-                   dAtom->getJy(),
-                   dAtom->getJz());
-          strcat( writeLine, tempBuffer );
+
+	which_atom = i;
+	local_index=-1;        
+        for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
+          if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
         }
-        else
-          strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );       
+        if (local_index != -1) {        
+	  sprintf( tempBuffer,
+		   "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+		   atoms[local_index]->getType(),
+		   atoms[local_index]->getX(),
+		   atoms[local_index]->getY(),
+		   atoms[local_index]->getZ(),
+		   atoms[local_index]->get_vx(),
+		   atoms[local_index]->get_vy(),
+		   atoms[local_index]->get_vz());
+	  strcpy( writeLine, tempBuffer );
+	  
+	  if( atoms[local_index]->isDirectional() ){
+	    
+	    dAtom = (DirectionalAtom *)atoms[local_index];
+	    dAtom->getQ( q );
+	    
+	    sprintf( tempBuffer,
+		     "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+		     q[0],
+		     q[1],
+		     q[2],
+		     q[3],
+		     dAtom->getJx(),
+		     dAtom->getJy(),
+		     dAtom->getJz());
+	    strcat( writeLine, tempBuffer );
+	  }
+	  else
+	    strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );       
+   	}
+	else {
+	  sprintf(painCave.errMsg,
+		  "Atom %d not found on processor %d\n",
+		  i, worldRank );
+	  haveError= 1;
+	  simError();
+	} 
+
+	if(haveError) nodeZeroError();
+     
+      } 
+      else {
         
-      } else {
-        
-        MPI::COMM_WORLD.Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG);
-        MPI::COMM_WORLD.Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node,
-                                TAKE_THIS_TAG, istatus);
+	myStatus = 1;
+	MPI_Send(&myStatus, 1, MPI_INT, which_node, 
+		 TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
+        MPI_Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG_INT,
+		 MPI_COMM_WORLD);
+        MPI_Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node,
+		 TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD, &istatus);
+        MPI_Recv(&myStatus, 1, MPI_INT, which_node, 
+		 TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
+	
+	if(!myStatus) nodeZeroError();
       }
       
       finalOut << writeLine;
     }
     
     // kill everyone off:
-    game_over = -1;
-    for (j = 0; j < mpiSim->getNumberProcessors(); j++) {
-      MPI::COMM_WORLD.Send(&game_over, 1, MPI_INT, j, TAKE_THIS_TAG);
+    myStatus = -1;
+    for (j = 0; j < mpiSim->getNumberProcessors(); j++) {      
+      MPI_Send(&myStatus, 1, MPI_INT, j, 
+	       TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
     }
 
   } else {
     
     done = 0;
     while (!done) {
-      MPI::COMM_WORLD.Recv(&which_atom, 1, MPI_INT, 0, 
-                           TAKE_THIS_TAG, istatus);
 
-      if (which_atom == -1) {
-        done=1;
-        continue;
-      } else {
+      MPI_Recv(&myStatus, 1, MPI_INT, 0, 
+	       TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
+      
+      if(!myStatus) anonymousNodeDie();
+      
+      if(myStatus < 0) break;
+      
+      MPI_Recv(&which_atom, 1, MPI_INT, 0, 
+	       TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
+      
+      myStatus = 1;
+      local_index=-1;        
+      for (j=0; j < mpiSim->getMyNlocal(); j++) {
+	if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
+      }
+      if (local_index != -1) {
 
-        local_index=-1;        
-        for (j=0; j < mpiSim->getMyNlocal(); j++) {
-          if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
-        }
-        if (local_index != -1) {
-
-          //format the line
-          sprintf( tempBuffer,
-                   "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
-                   atoms[local_index]->getType(),
-                   atoms[local_index]->getX(),
-                   atoms[local_index]->getY(),
-                   atoms[local_index]->getZ(),
-                   atoms[local_index]->get_vx(),
-                   atoms[local_index]->get_vy(),
-                   atoms[local_index]->get_vz()); // check here.
-          strcpy( writeLine, tempBuffer );
+	//format the line
+	sprintf( tempBuffer,
+		 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+		 atoms[local_index]->getType(),
+		 atoms[local_index]->getX(),
+		 atoms[local_index]->getY(),
+		 atoms[local_index]->getZ(),
+		 atoms[local_index]->get_vx(),
+		 atoms[local_index]->get_vy(),
+		 atoms[local_index]->get_vz()); // check here.
+	strcpy( writeLine, tempBuffer );
+	
+	if( atoms[local_index]->isDirectional() ){
 	  
-          if( atoms[local_index]->isDirectional() ){
-            
-            dAtom = (DirectionalAtom *)atoms[local_index];
-            dAtom->getQ( q );
-	    
-            sprintf( tempBuffer,
-                     "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
-                     q[0],
-                     q[1],
-                     q[2],
-                     q[3],
-                     dAtom->getJx(),
-                     dAtom->getJy(),
-                     dAtom->getJz());
-            strcat( writeLine, tempBuffer );
-          }
-          else
-            strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
-          
-          MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, 
-                               TAKE_THIS_TAG);
-        } else {
-          strcpy( writeLine, "ATOM NOT FOUND ON THIS PROCESSOR");
-          MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, 
-                               TAKE_THIS_TAG);
-        }
+	  dAtom = (DirectionalAtom *)atoms[local_index];
+	  dAtom->getQ( q );
+	  
+	  sprintf( tempBuffer,
+		   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+		   q[0],
+		   q[1],
+		   q[2],
+		   q[3],
+		   dAtom->getJx(),
+		   dAtom->getJy(),
+		   dAtom->getJz());
+	  strcat( writeLine, tempBuffer );
+	}
+	else{
+	  strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
+	}
+      } 
+      else {
+	sprintf(painCave.errMsg,
+		"Atom %d not found on processor %d\n",
+		which_atom, worldRank );
+	myStatus = 0;
+	simError();
+	
+	strcpy( writeLine, "Hello, I'm an error.\n");
       }
+
+      MPI_Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, 
+	       TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD);
+      MPI_Send( &myStatus, 1, MPI_INT, 0, 
+		TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
     }
   }
   finalOut.flush();
   sprintf( checkPointMsg,
 	   "Sucessfully took a dump.\n");
   MPIcheckPoint();
-
+  
   if( worldRank == 0 ) finalOut.close();    
 #endif // is_mpi
 }
+
+
+
+#ifdef IS_MPI
+
+// a couple of functions to let us escape the write loop
+
+void dWrite::nodeZeroError( void ){
+  int j, myStatus;
+  
+  myStatus = 0;
+  for (j = 0; j < mpiSim->getNumberProcessors(); j++) {      
+    MPI_Send( &myStatus, 1, MPI_INT, j, 
+	      TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
+  }  
+  
+
+  MPI_Finalize();
+  exit (0);
+  
+}
+
+void dWrite::anonymousNodeDie( void ){
+
+  MPI_Finalize();
+  exit (0);
+}
+
+#endif //is_mpi