--- trunk/OOPSE/libmdtools/DumpWriter.cpp	2003/03/27 01:49:45	417
+++ trunk/OOPSE/libmdtools/DumpWriter.cpp	2003/04/03 21:12:51	450
@@ -4,9 +4,16 @@
 
 #ifdef IS_MPI
 #include <mpi.h>
-#include <mpi++.h>
 #include "mpiSimulation.hpp"
-#define TAKE_THIS_TAG 0
+#define TAKE_THIS_TAG_CHAR 1
+#define TAKE_THIS_TAG_INT 2
+
+namespace dWrite{
+  void nodeZeroError( void );
+  void anonymousNodeDie( void );
+}
+
+using namespace dWrite;
 #endif //is_mpi
 
 #include "ReadWrite.hpp"
@@ -65,7 +72,7 @@ void DumpWriter::writeDump( double currentTime ){
   char tempBuffer[BUFFERSIZE];
   char writeLine[BUFFERSIZE];
 
-  int i, j, which_node, done, game_over, which_atom;
+  int i, j, which_node, done, which_atom, local_index;
   double q[4];
   DirectionalAtom* dAtom;
   int nAtoms = entry_plug->n_atoms;
@@ -120,7 +127,13 @@ void DumpWriter::writeDump( double currentTime ){
 
 #else // is_mpi
 
-  MPI::Status istatus;
+  // first thing first, suspend fatalities.
+  painCave.isEventLoop = 1;
+
+  int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone
+  int haveError;
+
+  MPI_Status istatus;
   int *AtomToProcMap = mpiSim->getAtomToProcMap();
   
   // write out header and node 0's coordinates
@@ -132,89 +145,128 @@ void DumpWriter::writeDump( double currentTime ){
 	    << entry_plug->box_x << "\t"
 	    << entry_plug->box_y << "\t"
 	    << entry_plug->box_z << "\n";
-    
+    outFile.flush();
     for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) {
       // Get the Node number which has this atom;
       
       which_node = AtomToProcMap[i];     
       
-      if (which_node == mpiSim->getMyNode()) {
-        
-        sprintf( tempBuffer,
-                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
-                 atoms[i]->getType(),
-                 atoms[i]->getX(),
-                 atoms[i]->getY(),
-                 atoms[i]->getZ(),
-                 atoms[i]->get_vx(),
-                 atoms[i]->get_vy(),
-                 atoms[i]->get_vz());
-        strcpy( writeLine, tempBuffer );
+      if (which_node == 0 ) {
 	
-        if( atoms[i]->isDirectional() ){
-	  
-          dAtom = (DirectionalAtom *)atoms[i];
-          dAtom->getQ( q );
-	  
-          sprintf( tempBuffer,
-                   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
-                   q[0],
-                   q[1],
-                   q[2],
-                   q[3],
-                   dAtom->getJx(),
-                   dAtom->getJy(),
-                   dAtom->getJz());
-          strcat( writeLine, tempBuffer );
+	haveError = 0;
+	which_atom = i;
+	local_index=-1;        
+        for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
+          if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
         }
-        else
-          strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );       
-        
-      } else {
-        
-        MPI::COMM_WORLD.Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG);
-        MPI::COMM_WORLD.Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node,
-                                TAKE_THIS_TAG, istatus);
+        if (local_index != -1) {
+          //format the line
+          sprintf( tempBuffer,
+                   "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+                   atoms[local_index]->getType(),
+                   atoms[local_index]->getX(),
+                   atoms[local_index]->getY(),
+                   atoms[local_index]->getZ(),
+                   atoms[local_index]->get_vx(),
+                   atoms[local_index]->get_vy(),
+                   atoms[local_index]->get_vz()); // check here.
+          strcpy( writeLine, tempBuffer );
+	  
+          if( atoms[local_index]->isDirectional() ){
+            
+            dAtom = (DirectionalAtom *)atoms[local_index];
+            dAtom->getQ( q );
+            
+            sprintf( tempBuffer,
+                     "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+                     q[0],
+                     q[1],
+                     q[2],
+                     q[3],
+                     dAtom->getJx(),
+                     dAtom->getJy(),
+                     dAtom->getJz());
+            strcat( writeLine, tempBuffer );
+	    
+	  }
+	  else
+	    strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );       
+	}
+	else {
+	  sprintf(painCave.errMsg,
+		  "Atom %d not found on processor %d\n",
+		  i, worldRank );
+	  haveError= 1;
+	  simError();
+	} 
+	
+	if(haveError) nodeZeroError();
+
       }
+      else {
+	myStatus = 1;
+	MPI_Send(&myStatus, 1, MPI_INT, which_node, 
+		 TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
+        MPI_Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG_INT, 
+		 MPI_COMM_WORLD);
+        MPI_Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node,
+		 TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD, &istatus);
+        MPI_Recv(&myStatus, 1, MPI_INT, which_node, 
+		 TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
+	
+	if(!myStatus) nodeZeroError();
+
+      }
       
       outFile << writeLine;
+      outFile.flush();
     }
     
     // kill everyone off:
-    game_over = -1;
+    myStatus = -1;
     for (j = 0; j < mpiSim->getNumberProcessors(); j++) {      
-      MPI::COMM_WORLD.Send(&game_over, 1, MPI_INT, j, TAKE_THIS_TAG);
+      MPI_Send(&myStatus, 1, MPI_INT, j, 
+	       TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
     }
 
   } else {
     
     done = 0;
     while (!done) {
-      MPI::COMM_WORLD.Recv(&which_atom, 1, MPI_INT, 0, 
-                              TAKE_THIS_TAG, istatus);
+      
+      MPI_Recv(&myStatus, 1, MPI_INT, 0, 
+	       TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
 
-      if (which_atom == -1) {
-        done=1;
-        continue;
-      } else {
+      if(!myStatus) anonymousNodeDie();
+      
+      if(myStatus < 0) break;
 
-        //format the line
+      MPI_Recv(&which_atom, 1, MPI_INT, 0, 
+	       TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
+      
+      myStatus = 1;
+      local_index=-1;        
+      for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
+	if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
+      }
+      if (local_index != -1) {
+	//format the line
 	sprintf( tempBuffer,
 		 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
-		 atoms[which_atom]->getType(),
-		 atoms[which_atom]->getX(),
-		 atoms[which_atom]->getY(),
-		 atoms[which_atom]->getZ(),
-		 atoms[which_atom]->get_vx(),
-		 atoms[which_atom]->get_vy(),
-		 atoms[which_atom]->get_vz()); // check here.
+		 atoms[local_index]->getType(),
+		 atoms[local_index]->getX(),
+		 atoms[local_index]->getY(),
+		 atoms[local_index]->getZ(),
+		 atoms[local_index]->get_vx(),
+		 atoms[local_index]->get_vy(),
+		 atoms[local_index]->get_vz()); // check here.
 	strcpy( writeLine, tempBuffer );
+	
+	if( atoms[local_index]->isDirectional() ){
 	  
-	if( atoms[which_atom]->isDirectional() ){
-	    
-	  dAtom = (DirectionalAtom *)atoms[which_atom];
+	  dAtom = (DirectionalAtom *)atoms[local_index];
 	  dAtom->getQ( q );
-	    
+	  
 	  sprintf( tempBuffer,
 		   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
 		   q[0],
@@ -226,18 +278,34 @@ void DumpWriter::writeDump( double currentTime ){
 		   dAtom->getJz());
 	  strcat( writeLine, tempBuffer );
 	}
-	else
+	else{
 	  strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
-        
-        MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, 
-                             TAKE_THIS_TAG);
+	}
+      } 
+      else {
+	sprintf(painCave.errMsg,
+		"Atom %d not found on processor %d\n",
+		which_atom, worldRank );
+	myStatus = 0;
+	simError();
+
+	strcpy( writeLine, "Hello, I'm an error.\n");
       }
+
+      MPI_Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, 
+	       TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD);
+      MPI_Send( &myStatus, 1, MPI_INT, 0, 
+		TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
     }
   }  
   outFile.flush();
   sprintf( checkPointMsg,
 	   "Sucessfully took a dump.\n");
   MPIcheckPoint();
+
+ // last  thing last, enable  fatalities.
+  painCave.isEventLoop = 0;
+
 #endif // is_mpi
 }
 
@@ -254,7 +322,7 @@ void DumpWriter::writeFinal(){
   DirectionalAtom* dAtom;
   int nAtoms = entry_plug->n_atoms;
   Atom** atoms = entry_plug->atoms;
-  int i, j, which_node, done, game_over, which_atom;
+  int i, j, which_node, done, game_over, which_atom, local_index;
   
   
 #ifdef IS_MPI
@@ -330,11 +398,18 @@ void DumpWriter::writeFinal(){
 
 #else // is_mpi
   
-  MPI::Status istatus;
+  // first thing first, suspend fatalities.
+  painCave.isEventLoop = 1;
+
+  int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone
+  int haveError;
+
+  MPI_Status istatus;
   int *AtomToProcMap = mpiSim->getAtomToProcMap();
 
   // write out header and node 0's coordinates
   
+  haveError = 0;
   if( worldRank == 0 ){
     finalOut << mpiSim->getTotAtoms() << "\n";
     
@@ -348,82 +423,118 @@ void DumpWriter::writeFinal(){
       which_node = AtomToProcMap[i];     
       
       if (which_node == mpiSim->getMyNode()) {
-        
-        sprintf( tempBuffer,
-                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
-                 atoms[i]->getType(),
-                 atoms[i]->getX(),
-                 atoms[i]->getY(),
-                 atoms[i]->getZ(),
-                 atoms[i]->get_vx(),
-                 atoms[i]->get_vy(),
-                 atoms[i]->get_vz());
-        strcpy( writeLine, tempBuffer );
-	
-        if( atoms[i]->isDirectional() ){
-	  
-          dAtom = (DirectionalAtom *)atoms[i];
-          dAtom->getQ( q );
-	  
-          sprintf( tempBuffer,
-                   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
-                   q[0],
-                   q[1],
-                   q[2],
-                   q[3],
-                   dAtom->getJx(),
-                   dAtom->getJy(),
-                   dAtom->getJz());
-          strcat( writeLine, tempBuffer );
+
+	which_atom = i;
+	local_index=-1;        
+        for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
+          if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
         }
-        else
-          strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );       
+        if (local_index != -1) {        
+	  sprintf( tempBuffer,
+		   "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+		   atoms[local_index]->getType(),
+		   atoms[local_index]->getX(),
+		   atoms[local_index]->getY(),
+		   atoms[local_index]->getZ(),
+		   atoms[local_index]->get_vx(),
+		   atoms[local_index]->get_vy(),
+		   atoms[local_index]->get_vz());
+	  strcpy( writeLine, tempBuffer );
+	  
+	  if( atoms[local_index]->isDirectional() ){
+	    
+	    dAtom = (DirectionalAtom *)atoms[local_index];
+	    dAtom->getQ( q );
+	    
+	    sprintf( tempBuffer,
+		     "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+		     q[0],
+		     q[1],
+		     q[2],
+		     q[3],
+		     dAtom->getJx(),
+		     dAtom->getJy(),
+		     dAtom->getJz());
+	    strcat( writeLine, tempBuffer );
+	  }
+	  else
+	    strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );       
+   	}
+	else {
+	  sprintf(painCave.errMsg,
+		  "Atom %d not found on processor %d\n",
+		  i, worldRank );
+	  haveError= 1;
+	  simError();
+	} 
+
+	if(haveError) nodeZeroError();
+     
+      } 
+      else {
         
-      } else {
-        
-        MPI::COMM_WORLD.Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG);
-        MPI::COMM_WORLD.Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node,
-                                TAKE_THIS_TAG, istatus);
+	myStatus = 1;
+	MPI_Send(&myStatus, 1, MPI_INT, which_node, 
+		 TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
+        MPI_Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG_INT,
+		 MPI_COMM_WORLD);
+        MPI_Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node,
+		 TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD, &istatus);
+        MPI_Recv(&myStatus, 1, MPI_INT, which_node, 
+		 TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
+	
+	if(!myStatus) nodeZeroError();
       }
       
       finalOut << writeLine;
     }
     
     // kill everyone off:
-    game_over = -1;
-    for (j = 0; j < mpiSim->getNumberProcessors(); j++) {
-      MPI::COMM_WORLD.Send(&game_over, 1, MPI_INT, j, TAKE_THIS_TAG);
+    myStatus = -1;
+    for (j = 0; j < mpiSim->getNumberProcessors(); j++) {      
+      MPI_Send(&myStatus, 1, MPI_INT, j, 
+	       TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
     }
 
   } else {
     
     done = 0;
     while (!done) {
-      MPI::COMM_WORLD.Recv(&which_atom, 1, MPI_INT, 0, 
-                           TAKE_THIS_TAG, istatus);
 
-      if (which_atom == -1) {
-        done=1;
-        continue;
-      } else {
+      MPI_Recv(&myStatus, 1, MPI_INT, 0, 
+	       TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
+      
+      if(!myStatus) anonymousNodeDie();
+      
+      if(myStatus < 0) break;
+      
+      MPI_Recv(&which_atom, 1, MPI_INT, 0, 
+	       TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
+      
+      myStatus = 1;
+      local_index=-1;        
+      for (j=0; j < mpiSim->getMyNlocal(); j++) {
+	if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
+      }
+      if (local_index != -1) {
 
-        //format the line
+	//format the line
 	sprintf( tempBuffer,
 		 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
-		 atoms[which_atom]->getType(),
-		 atoms[which_atom]->getX(),
-		 atoms[which_atom]->getY(),
-		 atoms[which_atom]->getZ(),
-		 atoms[which_atom]->get_vx(),
-		 atoms[which_atom]->get_vy(),
-		 atoms[which_atom]->get_vz()); // check here.
+		 atoms[local_index]->getType(),
+		 atoms[local_index]->getX(),
+		 atoms[local_index]->getY(),
+		 atoms[local_index]->getZ(),
+		 atoms[local_index]->get_vx(),
+		 atoms[local_index]->get_vy(),
+		 atoms[local_index]->get_vz()); // check here.
 	strcpy( writeLine, tempBuffer );
+	
+	if( atoms[local_index]->isDirectional() ){
 	  
-	if( atoms[which_atom]->isDirectional() ){
-	    
-	  dAtom = (DirectionalAtom *)atoms[which_atom];
+	  dAtom = (DirectionalAtom *)atoms[local_index];
 	  dAtom->getQ( q );
-	    
+	  
 	  sprintf( tempBuffer,
 		   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
 		   q[0],
@@ -435,19 +546,60 @@ void DumpWriter::writeFinal(){
 		   dAtom->getJz());
 	  strcat( writeLine, tempBuffer );
 	}
-	else
+	else{
 	  strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
-        
-        MPI::COMM_WORLD.Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, 
-                             TAKE_THIS_TAG);
+	}
+      } 
+      else {
+	sprintf(painCave.errMsg,
+		"Atom %d not found on processor %d\n",
+		which_atom, worldRank );
+	myStatus = 0;
+	simError();
+	
+	strcpy( writeLine, "Hello, I'm an error.\n");
       }
+
+      MPI_Send(writeLine, BUFFERSIZE, MPI_CHAR, 0, 
+	       TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD);
+      MPI_Send( &myStatus, 1, MPI_INT, 0, 
+		TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
     }
-  }  
+  }
   finalOut.flush();
   sprintf( checkPointMsg,
 	   "Sucessfully took a dump.\n");
   MPIcheckPoint();
-
+  
   if( worldRank == 0 ) finalOut.close();    
 #endif // is_mpi
 }
+
+
+
+#ifdef IS_MPI
+
+// a couple of functions to let us escape the write loop
+
+void dWrite::nodeZeroError( void ){
+  int j, myStatus;
+  
+  myStatus = 0;
+  for (j = 0; j < mpiSim->getNumberProcessors(); j++) {      
+    MPI_Send( &myStatus, 1, MPI_INT, j, 
+	      TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
+  }  
+  
+
+  MPI_Finalize();
+  exit (0);
+  
+}
+
+void dWrite::anonymousNodeDie( void ){
+
+  MPI_Finalize();
+  exit (0);
+}
+
+#endif //is_mpi