OOPSE/libmdtools/DumpWriter.cpp

#define _FILE_OFFSET_BITS 64

#include <string.h>
#include <iostream>
#include <fstream>

#ifdef IS_MPI
#include <mpi.h>
#include "mpiSimulation.hpp"
#define TAKE_THIS_TAG_CHAR 15
#define TAKE_THIS_TAG_INT 20 

namespace dWrite{
  void nodeZeroError( void );
  void anonymousNodeDie( void );
}

using namespace dWrite;
#endif //is_mpi

#include "ReadWrite.hpp"
#include "simError.h"

DumpWriter::DumpWriter( SimInfo* the_entry_plug ){

  entry_plug = the_entry_plug;

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif // is_mpi

    strcpy( outName, entry_plug->sampleName );

    outFile.open(outName, ios::out | ios::trunc );

    if( !outFile ){

      sprintf( painCave.errMsg,
               "Could not open \"%s\" for dump output.\n",
               outName);
      painCave.isFatal = 1;
      simError();
    }

    //outFile.setf( ios::scientific );

#ifdef IS_MPI
  }

  sprintf( checkPointMsg,
           "Sucessfully opened output file for dumping.\n");
  MPIcheckPoint();
#endif // is_mpi
}

DumpWriter::~DumpWriter( ){

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif // is_mpi

    outFile.close();

#ifdef IS_MPI
  }
#endif // is_mpi
}

void DumpWriter::writeDump( double currentTime ){

  const int BUFFERSIZE = 2000;
  char tempBuffer[BUFFERSIZE];
  char writeLine[BUFFERSIZE];

  int i;
#ifdef IS_MPI
  int j, which_node, done, which_atom, local_index;
#else //is_mpi
  int nAtoms = entry_plug->n_atoms;
#endif //is_mpi

  double q[4];
  DirectionalAtom* dAtom;
  Atom** atoms = entry_plug->atoms;
  double pos[3], vel[3];


  // write current frame to the eor file

  this->writeFinal( currentTime );

#ifndef IS_MPI

  outFile << nAtoms << "\n";

  outFile << currentTime << ";\t"
          << entry_plug->Hmat[0][0] << "\t"
          << entry_plug->Hmat[1][0] << "\t"
          << entry_plug->Hmat[2][0] << ";\t"

          << entry_plug->Hmat[0][1] << "\t"
          << entry_plug->Hmat[1][1] << "\t"
          << entry_plug->Hmat[2][1] << ";\t"

          << entry_plug->Hmat[0][2] << "\t"
          << entry_plug->Hmat[1][2] << "\t"
          << entry_plug->Hmat[2][2] << ";";
  //write out additional parameters, such as chi and eta
  outFile << entry_plug->the_integrator->getAdditionalParameters();
  outFile << endl;

  for( i=0; i<nAtoms; i++ ){

    atoms[i]->getPos(pos);
    atoms[i]->getVel(vel);

    sprintf( tempBuffer,
             "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
             atoms[i]->getType(),
             pos[0],
             pos[1],
             pos[2],
             vel[0],
             vel[1],
             vel[2]);
    strcpy( writeLine, tempBuffer );

    if( atoms[i]->isDirectional() ){

      dAtom = (DirectionalAtom *)atoms[i];
      dAtom->getQ( q );

      sprintf( tempBuffer,
               "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
               q[0],
               q[1],
               q[2],
               q[3],
               dAtom->getJx(),
               dAtom->getJy(),
               dAtom->getJz());
      strcat( writeLine, tempBuffer );
    }
    else
      strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );

    outFile << writeLine;
  }
  outFile.flush();

#else // is_mpi

  // first thing first, suspend fatalities.
  painCave.isEventLoop = 1;

  int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone
  int haveError;

  MPI_Status istatus;
  int *AtomToProcMap = mpiSim->getAtomToProcMap();

  // write out header and node 0's coordinates

  if( worldRank == 0 ){
    outFile << mpiSim->getTotAtoms() << "\n";

    outFile << currentTime << ";\t"
            << entry_plug->Hmat[0][0] << "\t"
            << entry_plug->Hmat[1][0] << "\t"
            << entry_plug->Hmat[2][0] << ";\t"

            << entry_plug->Hmat[0][1] << "\t"
            << entry_plug->Hmat[1][1] << "\t"
            << entry_plug->Hmat[2][1] << ";\t"

            << entry_plug->Hmat[0][2] << "\t"
            << entry_plug->Hmat[1][2] << "\t"
            << entry_plug->Hmat[2][2] << ";";

    outFile << entry_plug->the_integrator->getAdditionalParameters();
    outFile << endl;
    outFile.flush();
    for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) {
      // Get the Node number which has this atom;

      which_node = AtomToProcMap[i];

      if (which_node == 0 ) {

        haveError = 0;
        which_atom = i;
        local_index=-1;
        for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
          if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
        }
        if (local_index != -1) {
          //format the line

          atoms[local_index]->getPos(pos);
          atoms[local_index]->getVel(vel);

          sprintf( tempBuffer,
                   "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                   atoms[local_index]->getType(),
                   pos[0],
                   pos[1],
                   pos[2],
                   vel[0],
                   vel[1],
                   vel[2]); // check here.
          strcpy( writeLine, tempBuffer );

          if( atoms[local_index]->isDirectional() ){

            dAtom = (DirectionalAtom *)atoms[local_index];
            dAtom->getQ( q );

            sprintf( tempBuffer,
                     "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
                     q[0],
                     q[1],
                     q[2],
                     q[3],
                     dAtom->getJx(),
                     dAtom->getJy(),
                     dAtom->getJz());
            strcat( writeLine, tempBuffer );

          }
          else
            strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
        }
        else {
          sprintf(painCave.errMsg,
                  "Atom %d not found on processor %d\n",
                  i, worldRank );
          haveError= 1;
          simError();
        }

        if(haveError) nodeZeroError();

      }
      else {
        myStatus = 1;
        MPI_Send(&myStatus, 1, MPI_INT, which_node,
                 TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
        MPI_Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG_INT,
                 MPI_COMM_WORLD);
        MPI_Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node,
                 TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD, &istatus);
        MPI_Recv(&myStatus, 1, MPI_INT, which_node,
                 TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);

        if(!myStatus) nodeZeroError();

      }

      outFile << writeLine;
      outFile.flush();
    }

    // kill everyone off:
    myStatus = -1;
    for (j = 1; j < mpiSim->getNumberProcessors(); j++) {
      MPI_Send(&myStatus, 1, MPI_INT, j,
               TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
    }

  } else {

    done = 0;
    while (!done) {

      MPI_Recv(&myStatus, 1, MPI_INT, 0,
               TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);

      if(!myStatus) anonymousNodeDie();

      if(myStatus < 0) break;

      MPI_Recv(&which_atom, 1, MPI_INT, 0,
               TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);

      myStatus = 1;
      local_index=-1;
      for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
        if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
      }
      if (local_index != -1) {
        //format the line

        atoms[local_index]->getPos(pos);
        atoms[local_index]->getVel(vel);

        sprintf( tempBuffer,
                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                 atoms[local_index]->getType(),
                 pos[0],
                 pos[1],
                 pos[2],
                 vel[0],
                 vel[1],
                 vel[2]); // check here.
        strcpy( writeLine, tempBuffer );

        if( atoms[local_index]->isDirectional() ){

          dAtom = (DirectionalAtom *)atoms[local_index];
          dAtom->getQ( q );

          sprintf( tempBuffer,
                   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
                   q[0],
                   q[1],
                   q[2],
                   q[3],
                   dAtom->getJx(),
                   dAtom->getJy(),
                   dAtom->getJz());
          strcat( writeLine, tempBuffer );
        }
        else{
          strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
        }
      }
      else {
        sprintf(painCave.errMsg,
                "Atom %d not found on processor %d\n",
                which_atom, worldRank );
        myStatus = 0;
        simError();

        strcpy( writeLine, "Hello, I'm an error.\n");
      }

      MPI_Send(writeLine, BUFFERSIZE, MPI_CHAR, 0,
               TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD);
      MPI_Send( &myStatus, 1, MPI_INT, 0,
                TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
    }
  }
  outFile.flush();
  sprintf( checkPointMsg,
           "Sucessfully took a dump.\n");
  MPIcheckPoint();

 // last  thing last, enable  fatalities.
  painCave.isEventLoop = 0;

#endif // is_mpi
}

void DumpWriter::writeFinal(double finalTime){

  char finalName[500];
  ofstream finalOut;

  const int BUFFERSIZE = 2000;
  char tempBuffer[BUFFERSIZE];
  char writeLine[BUFFERSIZE];

  double q[4];
  DirectionalAtom* dAtom;
  Atom** atoms = entry_plug->atoms;
  int i;
#ifdef IS_MPI
  int j, which_node, done, which_atom, local_index;
#else //is_mpi
  int nAtoms = entry_plug->n_atoms;
#endif //is_mpi

  double pos[3], vel[3];

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif // is_mpi

    strcpy( finalName, entry_plug->finalName );

    finalOut.open( finalName, ios::out | ios::trunc );
    if( !finalOut ){
      sprintf( painCave.errMsg,
               "Could not open \"%s\" for final dump output.\n",
               finalName );
      painCave.isFatal = 1;
      simError();
    }

    // finalOut.setf( ios::scientific );

#ifdef IS_MPI
  }

  sprintf(checkPointMsg,"Opened file for final configuration\n");
  MPIcheckPoint();

#endif //is_mpi


#ifndef IS_MPI

  finalOut << nAtoms << "\n";

  finalOut << finalTime << ";\t"
           << entry_plug->Hmat[0][0] << "\t"
           << entry_plug->Hmat[1][0] << "\t"
           << entry_plug->Hmat[2][0] << ";\t"

           << entry_plug->Hmat[0][1] << "\t"
           << entry_plug->Hmat[1][1] << "\t"
           << entry_plug->Hmat[2][1] << ";\t"

           << entry_plug->Hmat[0][2] << "\t"
           << entry_plug->Hmat[1][2] << "\t"
           << entry_plug->Hmat[2][2] << ";";

  //write out additional parameters, such as chi and eta
  finalOut << entry_plug->the_integrator->getAdditionalParameters();
  finalOut << endl;

  for( i=0; i<nAtoms; i++ ){

    atoms[i]->getPos(pos);
    atoms[i]->getVel(vel);

    sprintf( tempBuffer,
             "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
             atoms[i]->getType(),
             pos[0],
             pos[1],
             pos[2],
             vel[0],
             vel[1],
             vel[2]);
    strcpy( writeLine, tempBuffer );

    if( atoms[i]->isDirectional() ){

      dAtom = (DirectionalAtom *)atoms[i];
      dAtom->getQ( q );

      sprintf( tempBuffer,
               "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
               q[0],
               q[1],
               q[2],
               q[3],
               dAtom->getJx(),
               dAtom->getJy(),
               dAtom->getJz());
      strcat( writeLine, tempBuffer );
    }
    else
      strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );

    finalOut << writeLine;
  }
  finalOut.flush();
  finalOut.close();

#else // is_mpi

  // first thing first, suspend fatalities.
  painCave.isEventLoop = 1;

  int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone
  int haveError;

  MPI_Status istatus;
  int *AtomToProcMap = mpiSim->getAtomToProcMap();

  // write out header and node 0's coordinates

  haveError = 0;
  if( worldRank == 0 ){
    finalOut << mpiSim->getTotAtoms() << "\n";

    finalOut << finalTime << ";\t"
             << entry_plug->Hmat[0][0] << "\t"
             << entry_plug->Hmat[1][0] << "\t"
             << entry_plug->Hmat[2][0] << ";\t"

             << entry_plug->Hmat[0][1] << "\t"
             << entry_plug->Hmat[1][1] << "\t"
             << entry_plug->Hmat[2][1] << ";\t"

             << entry_plug->Hmat[0][2] << "\t"
             << entry_plug->Hmat[1][2] << "\t"
             << entry_plug->Hmat[2][2] << ";";

    //write out additional parameters, such as chi and eta
    finalOut << entry_plug->the_integrator->getAdditionalParameters();
    finalOut << endl;

    for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) {
      // Get the Node number which has this molecule:

      which_node = AtomToProcMap[i];

      if (which_node == mpiSim->getMyNode()) {

        which_atom = i;
        local_index=-1;
        for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
          if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
        }
        if (local_index != -1) {

          atoms[local_index]->getPos(pos);
          atoms[local_index]->getVel(vel);

          sprintf( tempBuffer,
                   "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                   atoms[local_index]->getType(),
                   pos[0],
                   pos[1],
                   pos[2],
                   vel[0],
                   vel[1],
                   vel[2]);
          strcpy( writeLine, tempBuffer );

          if( atoms[local_index]->isDirectional() ){

            dAtom = (DirectionalAtom *)atoms[local_index];
            dAtom->getQ( q );

            sprintf( tempBuffer,
                     "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
                     q[0],
                     q[1],
                     q[2],
                     q[3],
                     dAtom->getJx(),
                     dAtom->getJy(),
                     dAtom->getJz());
            strcat( writeLine, tempBuffer );
          }
          else
            strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
        }
        else {
          sprintf(painCave.errMsg,
                  "Atom %d not found on processor %d\n",
                  i, worldRank );
          haveError= 1;
          simError();
        }

        if(haveError) nodeZeroError();

      }
      else {

        myStatus = 1;
        MPI_Send(&myStatus, 1, MPI_INT, which_node,
                 TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
        MPI_Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG_INT,
                 MPI_COMM_WORLD);
        MPI_Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node,
                 TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD, &istatus);
        MPI_Recv(&myStatus, 1, MPI_INT, which_node,
                 TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);

        if(!myStatus) nodeZeroError();
      }

      finalOut << writeLine;
    }

    // kill everyone off:
    myStatus = -1;
    for (j = 1; j < mpiSim->getNumberProcessors(); j++) {
      MPI_Send(&myStatus, 1, MPI_INT, j,
               TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
    }

  } else {

    done = 0;
    while (!done) {

      MPI_Recv(&myStatus, 1, MPI_INT, 0,
               TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);

      if(!myStatus) anonymousNodeDie();

      if(myStatus < 0) break;

      MPI_Recv(&which_atom, 1, MPI_INT, 0,
               TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);

      myStatus = 1;
      local_index=-1;
      for (j=0; j < mpiSim->getMyNlocal(); j++) {
        if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
      }
      if (local_index != -1) {

        atoms[local_index]->getPos(pos);
        atoms[local_index]->getVel(vel);

        //format the line
        sprintf( tempBuffer,
                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                 atoms[local_index]->getType(),
                 pos[0],
                 pos[1],
                 pos[2],
                 vel[0],
                 vel[1],
                 vel[2]); // check here.
        strcpy( writeLine, tempBuffer );

        if( atoms[local_index]->isDirectional() ){

          dAtom = (DirectionalAtom *)atoms[local_index];
          dAtom->getQ( q );

          sprintf( tempBuffer,
                   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
                   q[0],
                   q[1],
                   q[2],
                   q[3],
                   dAtom->getJx(),
                   dAtom->getJy(),
                   dAtom->getJz());
          strcat( writeLine, tempBuffer );
        }
        else{
          strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
        }
      }
      else {
        sprintf(painCave.errMsg,
                "Atom %d not found on processor %d\n",
                which_atom, worldRank );
        myStatus = 0;
        simError();

        strcpy( writeLine, "Hello, I'm an error.\n");
      }

      MPI_Send(writeLine, BUFFERSIZE, MPI_CHAR, 0,
               TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD);
      MPI_Send( &myStatus, 1, MPI_INT, 0,
                TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
    }
  }
  finalOut.flush();
  sprintf( checkPointMsg,
           "Sucessfully took a dump.\n");
  MPIcheckPoint();

  if( worldRank == 0 ) finalOut.close();
#endif // is_mpi
}


#ifdef IS_MPI

// a couple of functions to let us escape the write loop

void dWrite::nodeZeroError( void ){
  int j, myStatus;

  myStatus = 0;
  for (j = 0; j < mpiSim->getNumberProcessors(); j++) {
    MPI_Send( &myStatus, 1, MPI_INT, j,
              TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
  }


  MPI_Finalize();
  exit (0);

}

void dWrite::anonymousNodeDie( void ){

  MPI_Finalize();
  exit (0);
}

#endif //is_mpi
Revision:	905
Committed:	Wed Jan 7 19:26:12 2004 UTC (20 years, 6 months ago) by tim
File size:	15318 byte(s)
Log Message:	Fixed a bug of sending message from master node to itself in DumpWriter.cpp and InitializeFromFile.cpp
#	User	Rev	Content
1	mmeineke	723	#define _FILE_OFFSET_BITS 64
2
3	gezelter	829	#include <string.h>
4	mmeineke	377	#include <iostream>
5			#include <fstream>
6
7			#ifdef IS_MPI
8			#include <mpi.h>
9			#include "mpiSimulation.hpp"
10	tim	905	#define TAKE_THIS_TAG_CHAR 15
11			#define TAKE_THIS_TAG_INT 20
12	mmeineke	440
13			namespace dWrite{
14			void nodeZeroError( void );
15			void anonymousNodeDie( void );
16			}
17
18			using namespace dWrite;
19	mmeineke	377	#endif //is_mpi
20
21			#include "ReadWrite.hpp"
22			#include "simError.h"
23
24			DumpWriter::DumpWriter( SimInfo* the_entry_plug ){
25
26			entry_plug = the_entry_plug;
27
28			#ifdef IS_MPI
29			if(worldRank == 0 ){
30			#endif // is_mpi
31	tim	837
32	mmeineke	377	strcpy( outName, entry_plug->sampleName );
33	tim	837
34	mmeineke	377	outFile.open(outName, ios::out \| ios::trunc );
35	tim	837
36	mmeineke	377	if( !outFile ){
37	tim	837
38	mmeineke	377	sprintf( painCave.errMsg,
39			"Could not open \"%s\" for dump output.\n",
40			outName);
41			painCave.isFatal = 1;
42			simError();
43			}
44	mmeineke	469
45	mmeineke	377	//outFile.setf( ios::scientific );
46
47			#ifdef IS_MPI
48			}
49
50			sprintf( checkPointMsg,
51			"Sucessfully opened output file for dumping.\n");
52			MPIcheckPoint();
53			#endif // is_mpi
54			}
55
56			DumpWriter::~DumpWriter( ){
57
58			#ifdef IS_MPI
59			if(worldRank == 0 ){
60			#endif // is_mpi
61
62			outFile.close();
63
64			#ifdef IS_MPI
65			}
66			#endif // is_mpi
67			}
68
69			void DumpWriter::writeDump( double currentTime ){
70	tim	837
71	mmeineke	377	const int BUFFERSIZE = 2000;
72			char tempBuffer[BUFFERSIZE];
73			char writeLine[BUFFERSIZE];
74
75	mmeineke	787	int i;
76			#ifdef IS_MPI
77			int j, which_node, done, which_atom, local_index;
78			#else //is_mpi
79			int nAtoms = entry_plug->n_atoms;
80			#endif //is_mpi
81
82	mmeineke	377	double q[4];
83			DirectionalAtom* dAtom;
84			Atom** atoms = entry_plug->atoms;
85	mmeineke	670	double pos[3], vel[3];
86	tim	837
87
88	mmeineke	804	// write current frame to the eor file
89	mmeineke	377
90	mmeineke	804	this->writeFinal( currentTime );
91
92	mmeineke	377	#ifndef IS_MPI
93	tim	837
94	mmeineke	377	outFile << nAtoms << "\n";
95	tim	837
96			outFile << currentTime << ";\t"
97	mmeineke	590	<< entry_plug->Hmat[0][0] << "\t"
98			<< entry_plug->Hmat[1][0] << "\t"
99			<< entry_plug->Hmat[2][0] << ";\t"
100	mmeineke	572
101	mmeineke	590	<< entry_plug->Hmat[0][1] << "\t"
102			<< entry_plug->Hmat[1][1] << "\t"
103			<< entry_plug->Hmat[2][1] << ";\t"
104	mmeineke	572
105	mmeineke	590	<< entry_plug->Hmat[0][2] << "\t"
106			<< entry_plug->Hmat[1][2] << "\t"
107	tim	837	<< entry_plug->Hmat[2][2] << ";";
108			//write out additional parameters, such as chi and eta
109			outFile << entry_plug->the_integrator->getAdditionalParameters();
110			outFile << endl;
111
112	mmeineke	377	for( i=0; i<nAtoms; i++ ){
113	tim	837
114	mmeineke	670	atoms[i]->getPos(pos);
115			atoms[i]->getVel(vel);
116	mmeineke	377
117			sprintf( tempBuffer,
118			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
119			atoms[i]->getType(),
120	mmeineke	670	pos[0],
121			pos[1],
122			pos[2],
123			vel[0],
124			vel[1],
125			vel[2]);
126	mmeineke	377	strcpy( writeLine, tempBuffer );
127
128			if( atoms[i]->isDirectional() ){
129	tim	837
130	mmeineke	377	dAtom = (DirectionalAtom *)atoms[i];
131			dAtom->getQ( q );
132	tim	837
133	mmeineke	377	sprintf( tempBuffer,
134			"%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
135			q[0],
136			q[1],
137			q[2],
138			q[3],
139			dAtom->getJx(),
140			dAtom->getJy(),
141			dAtom->getJz());
142			strcat( writeLine, tempBuffer );
143			}
144			else
145			strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
146	tim	837
147	mmeineke	377	outFile << writeLine;
148			}
149			outFile.flush();
150
151			#else // is_mpi
152	gezelter	416
153	mmeineke	440	// first thing first, suspend fatalities.
154			painCave.isEventLoop = 1;
155
156			int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone
157			int haveError;
158
159	mmeineke	447	MPI_Status istatus;
160	gezelter	416	int *AtomToProcMap = mpiSim->getAtomToProcMap();
161	tim	837
162	mmeineke	377	// write out header and node 0's coordinates
163	tim	837
164	mmeineke	377	if( worldRank == 0 ){
165			outFile << mpiSim->getTotAtoms() << "\n";
166	tim	837
167			outFile << currentTime << ";\t"
168	gezelter	591	<< entry_plug->Hmat[0][0] << "\t"
169			<< entry_plug->Hmat[1][0] << "\t"
170			<< entry_plug->Hmat[2][0] << ";\t"
171	tim	837
172	gezelter	591	<< entry_plug->Hmat[0][1] << "\t"
173			<< entry_plug->Hmat[1][1] << "\t"
174			<< entry_plug->Hmat[2][1] << ";\t"
175	tim	837
176	gezelter	591	<< entry_plug->Hmat[0][2] << "\t"
177			<< entry_plug->Hmat[1][2] << "\t"
178	tim	837	<< entry_plug->Hmat[2][2] << ";";
179
180			outFile << entry_plug->the_integrator->getAdditionalParameters();
181			outFile << endl;
182	chuckv	434	outFile.flush();
183	gezelter	416	for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) {
184	gezelter	417	// Get the Node number which has this atom;
185	tim	837
186			which_node = AtomToProcMap[i];
187
188	chuckv	436	if (which_node == 0 ) {
189	tim	837
190	mmeineke	440	haveError = 0;
191	chuckv	436	which_atom = i;
192	tim	837	local_index=-1;
193	chuckv	436	for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
194			if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
195			}
196			if (local_index != -1) {
197			//format the line
198	tim	837
199	mmeineke	670	atoms[local_index]->getPos(pos);
200			atoms[local_index]->getVel(vel);
201
202	chuckv	436	sprintf( tempBuffer,
203			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
204			atoms[local_index]->getType(),
205	mmeineke	670	pos[0],
206			pos[1],
207			pos[2],
208			vel[0],
209			vel[1],
210			vel[2]); // check here.
211	chuckv	436	strcpy( writeLine, tempBuffer );
212	tim	837
213	chuckv	436	if( atoms[local_index]->isDirectional() ){
214	tim	837
215	chuckv	436	dAtom = (DirectionalAtom *)atoms[local_index];
216			dAtom->getQ( q );
217	tim	837
218	chuckv	436	sprintf( tempBuffer,
219			"%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
220			q[0],
221			q[1],
222			q[2],
223			q[3],
224			dAtom->getJx(),
225			dAtom->getJy(),
226			dAtom->getJz());
227			strcat( writeLine, tempBuffer );
228	tim	837
229	chuckv	436	}
230			else
231	tim	837	strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
232	chuckv	436	}
233			else {
234	mmeineke	440	sprintf(painCave.errMsg,
235			"Atom %d not found on processor %d\n",
236			i, worldRank );
237			haveError= 1;
238			simError();
239	tim	837	}
240
241	mmeineke	440	if(haveError) nodeZeroError();
242
243			}
244	chuckv	436	else {
245	mmeineke	440	myStatus = 1;
246	tim	837	MPI_Send(&myStatus, 1, MPI_INT, which_node,
247	mmeineke	447	TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
248	tim	837	MPI_Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG_INT,
249	mmeineke	447	MPI_COMM_WORLD);
250			MPI_Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node,
251			TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD, &istatus);
252	tim	837	MPI_Recv(&myStatus, 1, MPI_INT, which_node,
253	mmeineke	447	TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
254	tim	837
255	mmeineke	440	if(!myStatus) nodeZeroError();
256
257	mmeineke	377	}
258	tim	837
259	mmeineke	377	outFile << writeLine;
260	chuckv	434	outFile.flush();
261	mmeineke	377	}
262	tim	837
263	gezelter	415	// kill everyone off:
264	mmeineke	440	myStatus = -1;
265	tim	905	for (j = 1; j < mpiSim->getNumberProcessors(); j++) {
266	tim	837	MPI_Send(&myStatus, 1, MPI_INT, j,
267	mmeineke	447	TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
268	mmeineke	377	}
269
270	gezelter	415	} else {
271	tim	837
272	gezelter	415	done = 0;
273			while (!done) {
274	tim	837
275			MPI_Recv(&myStatus, 1, MPI_INT, 0,
276	mmeineke	447	TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
277	mmeineke	440
278			if(!myStatus) anonymousNodeDie();
279	tim	837
280	mmeineke	440	if(myStatus < 0) break;
281
282	tim	837	MPI_Recv(&which_atom, 1, MPI_INT, 0,
283	mmeineke	447	TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
284	tim	837
285	mmeineke	440	myStatus = 1;
286	tim	837	local_index=-1;
287	mmeineke	440	for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
288			if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
289			}
290			if (local_index != -1) {
291			//format the line
292	mmeineke	670
293			atoms[local_index]->getPos(pos);
294			atoms[local_index]->getVel(vel);
295
296	mmeineke	440	sprintf( tempBuffer,
297			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
298			atoms[local_index]->getType(),
299	mmeineke	670	pos[0],
300			pos[1],
301			pos[2],
302			vel[0],
303			vel[1],
304			vel[2]); // check here.
305	mmeineke	440	strcpy( writeLine, tempBuffer );
306	tim	837
307	mmeineke	440	if( atoms[local_index]->isDirectional() ){
308	tim	837
309	mmeineke	440	dAtom = (DirectionalAtom *)atoms[local_index];
310			dAtom->getQ( q );
311	tim	837
312	mmeineke	440	sprintf( tempBuffer,
313			"%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
314			q[0],
315			q[1],
316			q[2],
317			q[3],
318			dAtom->getJx(),
319			dAtom->getJy(),
320			dAtom->getJz());
321			strcat( writeLine, tempBuffer );
322			}
323			else{
324			strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
325			}
326	tim	837	}
327	mmeineke	440	else {
328			sprintf(painCave.errMsg,
329			"Atom %d not found on processor %d\n",
330			which_atom, worldRank );
331			myStatus = 0;
332			simError();
333
334			strcpy( writeLine, "Hello, I'm an error.\n");
335	mmeineke	377	}
336	mmeineke	440
337	tim	837	MPI_Send(writeLine, BUFFERSIZE, MPI_CHAR, 0,
338	mmeineke	447	TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD);
339	tim	837	MPI_Send( &myStatus, 1, MPI_INT, 0,
340	mmeineke	447	TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
341	mmeineke	377	}
342	tim	837	}
343	gezelter	415	outFile.flush();
344			sprintf( checkPointMsg,
345			"Sucessfully took a dump.\n");
346			MPIcheckPoint();
347	mmeineke	440
348			// last thing last, enable fatalities.
349			painCave.isEventLoop = 0;
350
351	mmeineke	377	#endif // is_mpi
352			}
353
354	mmeineke	572	void DumpWriter::writeFinal(double finalTime){
355	gezelter	416
356	mmeineke	377	char finalName[500];
357			ofstream finalOut;
358	gezelter	416
359			const int BUFFERSIZE = 2000;
360			char tempBuffer[BUFFERSIZE];
361	tim	837	char writeLine[BUFFERSIZE];
362	gezelter	416
363			double q[4];
364			DirectionalAtom* dAtom;
365	mmeineke	787	Atom** atoms = entry_plug->atoms;
366			int i;
367			#ifdef IS_MPI
368			int j, which_node, done, which_atom, local_index;
369			#else //is_mpi
370	gezelter	416	int nAtoms = entry_plug->n_atoms;
371	mmeineke	787	#endif //is_mpi
372	tim	837
373	mmeineke	670	double pos[3], vel[3];
374	tim	837
375	mmeineke	377	#ifdef IS_MPI
376			if(worldRank == 0 ){
377			#endif // is_mpi
378	tim	837
379	mmeineke	377	strcpy( finalName, entry_plug->finalName );
380	tim	837
381	mmeineke	377	finalOut.open( finalName, ios::out \| ios::trunc );
382			if( !finalOut ){
383			sprintf( painCave.errMsg,
384			"Could not open \"%s\" for final dump output.\n",
385			finalName );
386			painCave.isFatal = 1;
387			simError();
388			}
389	tim	837
390	mmeineke	377	// finalOut.setf( ios::scientific );
391	tim	837
392	mmeineke	377	#ifdef IS_MPI
393			}
394	tim	837
395	mmeineke	377	sprintf(checkPointMsg,"Opened file for final configuration\n");
396	tim	837	MPIcheckPoint();
397
398	mmeineke	377	#endif //is_mpi
399
400	tim	837
401	mmeineke	377	#ifndef IS_MPI
402	tim	837
403	mmeineke	377	finalOut << nAtoms << "\n";
404	tim	837
405	gezelter	591	finalOut << finalTime << ";\t"
406	mmeineke	590	<< entry_plug->Hmat[0][0] << "\t"
407			<< entry_plug->Hmat[1][0] << "\t"
408	gezelter	591	<< entry_plug->Hmat[2][0] << ";\t"
409	tim	837
410	mmeineke	590	<< entry_plug->Hmat[0][1] << "\t"
411			<< entry_plug->Hmat[1][1] << "\t"
412	gezelter	591	<< entry_plug->Hmat[2][1] << ";\t"
413	tim	837
414	mmeineke	590	<< entry_plug->Hmat[0][2] << "\t"
415			<< entry_plug->Hmat[1][2] << "\t"
416	tim	837	<< entry_plug->Hmat[2][2] << ";";
417
418			//write out additional parameters, such as chi and eta
419			finalOut << entry_plug->the_integrator->getAdditionalParameters();
420			finalOut << endl;
421
422	mmeineke	377	for( i=0; i<nAtoms; i++ ){
423	tim	837
424	mmeineke	670	atoms[i]->getPos(pos);
425			atoms[i]->getVel(vel);
426	tim	837
427	mmeineke	377	sprintf( tempBuffer,
428			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
429			atoms[i]->getType(),
430	mmeineke	670	pos[0],
431			pos[1],
432			pos[2],
433			vel[0],
434			vel[1],
435			vel[2]);
436	mmeineke	377	strcpy( writeLine, tempBuffer );
437
438			if( atoms[i]->isDirectional() ){
439	tim	837
440	mmeineke	377	dAtom = (DirectionalAtom *)atoms[i];
441			dAtom->getQ( q );
442	tim	837
443	mmeineke	377	sprintf( tempBuffer,
444			"%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
445			q[0],
446			q[1],
447			q[2],
448			q[3],
449			dAtom->getJx(),
450			dAtom->getJy(),
451			dAtom->getJz());
452			strcat( writeLine, tempBuffer );
453			}
454			else
455			strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
456	tim	837
457	mmeineke	377	finalOut << writeLine;
458			}
459			finalOut.flush();
460	gezelter	415	finalOut.close();
461	mmeineke	377
462			#else // is_mpi
463	tim	837
464	mmeineke	440	// first thing first, suspend fatalities.
465			painCave.isEventLoop = 1;
466
467			int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone
468			int haveError;
469
470	mmeineke	447	MPI_Status istatus;
471	gezelter	416	int *AtomToProcMap = mpiSim->getAtomToProcMap();
472
473	mmeineke	377	// write out header and node 0's coordinates
474	tim	837
475	mmeineke	440	haveError = 0;
476	mmeineke	377	if( worldRank == 0 ){
477			finalOut << mpiSim->getTotAtoms() << "\n";
478	tim	837
479	gezelter	591	finalOut << finalTime << ";\t"
480			<< entry_plug->Hmat[0][0] << "\t"
481			<< entry_plug->Hmat[1][0] << "\t"
482			<< entry_plug->Hmat[2][0] << ";\t"
483	tim	837
484	gezelter	591	<< entry_plug->Hmat[0][1] << "\t"
485			<< entry_plug->Hmat[1][1] << "\t"
486			<< entry_plug->Hmat[2][1] << ";\t"
487	tim	837
488	gezelter	591	<< entry_plug->Hmat[0][2] << "\t"
489			<< entry_plug->Hmat[1][2] << "\t"
490	tim	837	<< entry_plug->Hmat[2][2] << ";";
491
492			//write out additional parameters, such as chi and eta
493			finalOut << entry_plug->the_integrator->getAdditionalParameters();
494			finalOut << endl;
495
496	gezelter	416	for (i = 0 ; i < mpiSim->getTotAtoms(); i++ ) {
497	gezelter	415	// Get the Node number which has this molecule:
498	tim	837
499			which_node = AtomToProcMap[i];
500
501	gezelter	416	if (which_node == mpiSim->getMyNode()) {
502	chuckv	437
503			which_atom = i;
504	tim	837	local_index=-1;
505	chuckv	437	for (j=0; (j<mpiSim->getMyNlocal()) && (local_index < 0); j++) {
506			if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
507			}
508	tim	837	if (local_index != -1) {
509	mmeineke	670
510			atoms[local_index]->getPos(pos);
511			atoms[local_index]->getVel(vel);
512	tim	837
513	chuckv	437	sprintf( tempBuffer,
514			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
515			atoms[local_index]->getType(),
516	mmeineke	670	pos[0],
517			pos[1],
518			pos[2],
519			vel[0],
520			vel[1],
521			vel[2]);
522	chuckv	437	strcpy( writeLine, tempBuffer );
523	tim	837
524	chuckv	437	if( atoms[local_index]->isDirectional() ){
525	tim	837
526	chuckv	437	dAtom = (DirectionalAtom *)atoms[local_index];
527			dAtom->getQ( q );
528	tim	837
529	chuckv	437	sprintf( tempBuffer,
530			"%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
531			q[0],
532			q[1],
533			q[2],
534			q[3],
535			dAtom->getJx(),
536			dAtom->getJy(),
537			dAtom->getJz());
538			strcat( writeLine, tempBuffer );
539			}
540			else
541	tim	837	strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
542	chuckv	437	}
543			else {
544	mmeineke	440	sprintf(painCave.errMsg,
545			"Atom %d not found on processor %d\n",
546			i, worldRank );
547			haveError= 1;
548			simError();
549	tim	837	}
550	mmeineke	440
551			if(haveError) nodeZeroError();
552	tim	837
553			}
554	mmeineke	440	else {
555	tim	837
556	mmeineke	440	myStatus = 1;
557	tim	837	MPI_Send(&myStatus, 1, MPI_INT, which_node,
558	mmeineke	447	TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
559			MPI_Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG_INT,
560			MPI_COMM_WORLD);
561			MPI_Recv(writeLine, BUFFERSIZE, MPI_CHAR, which_node,
562			TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD, &istatus);
563	tim	837	MPI_Recv(&myStatus, 1, MPI_INT, which_node,
564	mmeineke	447	TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
565	tim	837
566	mmeineke	440	if(!myStatus) nodeZeroError();
567	mmeineke	377	}
568	tim	837
569	mmeineke	377	finalOut << writeLine;
570			}
571	tim	837
572	gezelter	415	// kill everyone off:
573	mmeineke	440	myStatus = -1;
574	tim	905	for (j = 1; j < mpiSim->getNumberProcessors(); j++) {
575	tim	837	MPI_Send(&myStatus, 1, MPI_INT, j,
576	mmeineke	447	TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
577	mmeineke	377	}
578
579	gezelter	415	} else {
580	tim	837
581	gezelter	415	done = 0;
582			while (!done) {
583	mmeineke	440
584	tim	837	MPI_Recv(&myStatus, 1, MPI_INT, 0,
585	mmeineke	447	TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
586	tim	837
587	mmeineke	440	if(!myStatus) anonymousNodeDie();
588	tim	837
589	mmeineke	440	if(myStatus < 0) break;
590	tim	837
591			MPI_Recv(&which_atom, 1, MPI_INT, 0,
592	mmeineke	447	TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);
593	tim	837
594	mmeineke	440	myStatus = 1;
595	tim	837	local_index=-1;
596	mmeineke	440	for (j=0; j < mpiSim->getMyNlocal(); j++) {
597			if (atoms[j]->getGlobalIndex() == which_atom) local_index = j;
598			}
599			if (local_index != -1) {
600	mmeineke	377
601	mmeineke	670	atoms[local_index]->getPos(pos);
602			atoms[local_index]->getVel(vel);
603
604	mmeineke	440	//format the line
605			sprintf( tempBuffer,
606			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
607			atoms[local_index]->getType(),
608	mmeineke	670	pos[0],
609			pos[1],
610			pos[2],
611			vel[0],
612			vel[1],
613			vel[2]); // check here.
614	mmeineke	440	strcpy( writeLine, tempBuffer );
615	tim	837
616	mmeineke	440	if( atoms[local_index]->isDirectional() ){
617	tim	837
618	mmeineke	440	dAtom = (DirectionalAtom *)atoms[local_index];
619			dAtom->getQ( q );
620	tim	837
621	mmeineke	440	sprintf( tempBuffer,
622			"%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
623			q[0],
624			q[1],
625			q[2],
626			q[3],
627			dAtom->getJx(),
628			dAtom->getJy(),
629			dAtom->getJz());
630			strcat( writeLine, tempBuffer );
631			}
632			else{
633			strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
634			}
635	tim	837	}
636	mmeineke	440	else {
637			sprintf(painCave.errMsg,
638			"Atom %d not found on processor %d\n",
639			which_atom, worldRank );
640			myStatus = 0;
641			simError();
642	tim	837
643	mmeineke	440	strcpy( writeLine, "Hello, I'm an error.\n");
644	mmeineke	377	}
645	mmeineke	440
646	tim	837	MPI_Send(writeLine, BUFFERSIZE, MPI_CHAR, 0,
647	mmeineke	447	TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD);
648	tim	837	MPI_Send( &myStatus, 1, MPI_INT, 0,
649	mmeineke	447	TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
650	mmeineke	377	}
651	gezelter	419	}
652	gezelter	415	finalOut.flush();
653			sprintf( checkPointMsg,
654			"Sucessfully took a dump.\n");
655			MPIcheckPoint();
656	tim	837
657			if( worldRank == 0 ) finalOut.close();
658	mmeineke	377	#endif // is_mpi
659			}
660	mmeineke	440
661
662
663			#ifdef IS_MPI
664
665			// a couple of functions to let us escape the write loop
666
667			void dWrite::nodeZeroError( void ){
668			int j, myStatus;
669	tim	837
670	mmeineke	440	myStatus = 0;
671	tim	837	for (j = 0; j < mpiSim->getNumberProcessors(); j++) {
672			MPI_Send( &myStatus, 1, MPI_INT, j,
673	mmeineke	447	TAKE_THIS_TAG_INT, MPI_COMM_WORLD);
674	tim	837	}
675	mmeineke	440
676	tim	837
677	mmeineke	440	MPI_Finalize();
678			exit (0);
679	tim	837
680	mmeineke	440	}
681
682			void dWrite::anonymousNodeDie( void ){
683
684			MPI_Finalize();
685			exit (0);
686			}
687
688			#endif //is_mpi