src/io/DumpWriter.cpp

#include "io/DumpWrite.hpp"
#include "utils/simError.h"

#ifdef IS_MPI
#include <mpi.h>
#endif //is_mpi
namespace oopse {
DumpWriter::DumpWriter(SimInfo* info, const std::string& filename) 
                   : info_(info), filename_(filename){
#ifdef IS_MPI

    if (worldRank == 0) {
#endif // is_mpi

        dumpFile.open(filename_.c_str(), ios::out | ios::trunc);

        if (!dumpFile) {
            sprintf(painCave.errMsg, "Could not open \"%s\" for dump output.\n",
                    info_->filename_.c_str());
            painCave.isFatal = 1;
            simError();
        }

#ifdef IS_MPI

    }

    sprintf(checkPointMsg, "Sucessfully opened output file for dumping.\n");
    MPIcheckPoint();

#endif // is_mpi

}

DumpWriter::~DumpWriter() {

#ifdef IS_MPI

    if (worldRank == 0) {
#endif // is_mpi

        dumpFile.close();

#ifdef IS_MPI

    }

#endif // is_mpi

}

void writeCommentLine(Snapshot* s) {

    double currentTime;
    Mat3x3d hmat;
    double chi;
    double integralOfChiDt;
    Mat3x3d eta;
    
    currentTime = s->getTime();
    hmat = s->getHmat();
    chi = s->getChi();
    integralOfChiDt = s->getIntegralOfChiDt();
    eta = s->getEta();
    
    os << currentTime << ";\t" 
         << hmat[0][0] << "\t" << hmat[1][0] << "\t" << hmat[2][0] << ";\t" 
         << hmat[0][1] << "\t" << hmat[1][1] << "\t" << hmat[2][1] << ";\t"
         << hmat[0][2] << "\t" << hmat[1][2] << "\t" << hmat[2][2] << ";";

    //write out additional parameters, such as chi and eta

    os << chi << "\t" << integralOfChiDt << "\t;"

    os << eta[0][0] << "\t" << eta[1][0] << "\t" << eta[2][0] << ";\t" 
         << eta[0][1] << "\t" << eta[1][1] << "\t" << eta[2][1] << ";\t"
         << eta[0][2] << "\t" << eta[1][2] << "\t" << eta[2][2] << ";";
        
    os << endl;
}

void DumpWriter::writeFrame(std::ostream& os) {
    const int BUFFERSIZE = 2000;
    const int MINIBUFFERSIZE = 100;

    char tempBuffer[BUFFERSIZE];
    char writeLine[BUFFERSIZE];

    Quat4d q;
    Vector3d ji;
    Vector3d pos;
    Vector3d vel;

    Molecule* mol;
    StuntDouble* integrableObject;
    typename SimInfo::MoleculeIterator mi;
    typename Molecule::IntegrableObjectIterator ii;
  
    int nTotObjects;    
    nTotObjects = info_->getNGlobalIntegrableObjects();

#ifndef IS_MPI


    os << nTotObjects << "\n";
        
    writeCommentLine(info_->getSnapshotManager()->getCurrentSnapshot());

    for (mol = info_->beginMolecule(mi); mol != NULL; mol = info_->nextMolecule(mi)) {

        for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL; 
            integrableObject = mol->nextIntegrableObject(ii)) { 
                

            pos = integrableObject->getPos();
            vel = integrableObject->getVel();

            sprintf(tempBuffer, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                    integrableObject->getType(), 
                    pos[0], pos[1], pos[2],
                    vel[0], vel[1], vel[2]);

            strcpy(writeLine, tempBuffer);

            if (integrableObject->isDirectional()) {
                q = integrableObject->getQ();
                ji = integrableObject->getJ();

                sprintf(tempBuffer, "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", 
                        q[0], q[1], q[2], q[3],
                        ji[0], ji[1], ji[2]);
                strcat(writeLine, tempBuffer);
            } else {
                strcat(writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
            }

            os << writeLine;

        }
    }

#else // is_mpi
    /*********************************************************************
     * Documentation?  You want DOCUMENTATION?
     * 
     * Why all the potatoes below?  
     *
     * To make a long story short, the original version of DumpWriter
     * worked in the most inefficient way possible.  Node 0 would 
     * poke each of the node for an individual atom's formatted data 
     * as node 0 worked its way down the global index. This was particularly 
     * inefficient since the method blocked all processors at every atom 
     * (and did it twice!).
     *
     * An intermediate version of DumpWriter could be described from Node
     * zero's perspective as follows:
     * 
     *  1) Have 100 of your friends stand in a circle.
     *  2) When you say go, have all of them start tossing potatoes at
     *     you (one at a time).
     *  3) Catch the potatoes.
     *
     * It was an improvement, but MPI has buffers and caches that could 
     * best be described in this analogy as "potato nets", so there's no 
     * need to block the processors atom-by-atom.
     * 
     * This new and improved DumpWriter works in an even more efficient 
     * way:
     * 
     *  1) Have 100 of your friend stand in a circle.
     *  2) When you say go, have them start tossing 5-pound bags of 
     *     potatoes at you.
     *  3) Once you've caught a friend's bag of potatoes,
     *     toss them a spud to let them know they can toss another bag.
     *
     * How's THAT for documentation?
     *
     *********************************************************************/
    const int masterNode = 0;

    int * potatoes;
    int myPotato;
    int nProc;
    int which_node;
    double atomData[13];
    int isDirectional;
    const char * atomTypeString;
    char MPIatomTypeString[MINIBUFFERSIZE];
    int msgLen; // the length of message actually recieved at master nodes
    int haveError;
    MPI_Status istatus;
    int nCurObj;
    
    // code to find maximum tag value
    int * tagub;
    int flag;
    int MAXTAG;
    MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);

    if (flag) {
        MAXTAG = *tagub;
    } else {
        MAXTAG = 32767;
    }

    if (worldRank == masterNode) { //master node (node 0) is responsible for writing the dump file

        // Node 0 needs a list of the magic potatoes for each processor;

        MPI_Comm_size(MPI_COMM_WORLD, &nProc);
        potatoes = new int[nProc];

        //write out the comment lines
        for(i = 0; i < nProc; i++) {
            potatoes[i] = 0;
        }


        os << nTotObjects << "\n";
        writeCommentLine(info_->getSnapshotManager()->getCurrentSnapshot());

        for(i = 0; i < info_->getNGlobalMolecules(); i++) {

            // Get the Node number which has this atom;

            which_node = info_->getMolToProc(i);

            if (which_node != masterNode) { //current molecule is in slave node
                if (potatoes[which_node] + 1 >= MAXTAG) {
                    // The potato was going to exceed the maximum value, 
                    // so wrap this processor potato back to 0:         

                    potatoes[which_node] = 0;
                    MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0,
                             MPI_COMM_WORLD);
                }

                myPotato = potatoes[which_node];

                //recieve the number of integrableObject in current molecule
                MPI_Recv(&nCurObj, 1, MPI_INT, which_node, myPotato,
                         MPI_COMM_WORLD, &istatus);
                myPotato++;

                for(int l = 0; l < nCurObj; l++) {
                    if (potatoes[which_node] + 2 >= MAXTAG) {
                        // The potato was going to exceed the maximum value, 
                        // so wrap this processor potato back to 0:         

                        potatoes[which_node] = 0;
                        MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node,
                                 0, MPI_COMM_WORLD);
                    }

                    MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR,
                             which_node, myPotato, MPI_COMM_WORLD,
                             &istatus);

                    atomTypeString = MPIatomTypeString;

                    myPotato++;

                    MPI_Recv(atomData, 13, MPI_DOUBLE, which_node, myPotato,
                             MPI_COMM_WORLD, &istatus);
                    myPotato++;

                    MPI_Get_count(&istatus, MPI_DOUBLE, &msgLen);

                    if (msgLen == 13)
                        isDirectional = 1;
                    else
                        isDirectional = 0;

                    // If we've survived to here, format the line:

                    if (!isDirectional) {
                        sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                                atomTypeString, atomData[0],
                                atomData[1], atomData[2],
                                atomData[3], atomData[4],
                                atomData[5]);

                        strcat(writeLine,
                               "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
                    } else {
                        sprintf(writeLine,
                                "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
                                atomTypeString,
                                atomData[0],
                                atomData[1],
                                atomData[2],
                                atomData[3],
                                atomData[4],
                                atomData[5],
                                atomData[6],
                                atomData[7],
                                atomData[8],
                                atomData[9],
                                atomData[10],
                                atomData[11],
                                atomData[12]);
                    }

                    os << writeLine;

                } // end for(int l =0)

                potatoes[which_node] = myPotato;
            } else { //master node has current molecule

                mol = info_->getMoleculeByGlobalIndex(i);

                if (mol == NULL) {
                    strcpy(painCave.errMsg, "Molecule not found on node %d!", worldRank);
                    painCave.isFatal = 1;
                    simError();
                }
                
                for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL; 
                    integrableObject = mol->nextIntegrableObject(ii)) {
                        
                    atomTypeString = integrableObject->getType().c_str();

                    pos = integrableObject->getPos();
                    vel = integrableObject->getVel();

                    atomData[0] = pos[0];
                    atomData[1] = pos[1];
                    atomData[2] = pos[2];

                    atomData[3] = vel[0];
                    atomData[4] = vel[1];
                    atomData[5] = vel[2];

                    isDirectional = 0;

                    if (integrableObject->isDirectional()) {
                        isDirectional = 1;

                        q = integrableObject->getQ();
                        ji = integrableObject->getJ();

                        for(int j = 0; j < 6; j++) {
                            atomData[j] = atomData[j];
                        }

                        atomData[6] = q[0];
                        atomData[7] = q[1];
                        atomData[8] = q[2];
                        atomData[9] = q[3];

                        atomData[10] = ji[0];
                        atomData[11] = ji[1];
                        atomData[12] = ji[2];
                    }

                    // If we've survived to here, format the line:

                    if (!isDirectional) {
                        sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                                atomTypeString, atomData[0],
                                atomData[1], atomData[2],
                                atomData[3], atomData[4],
                                atomData[5]);

                        strcat(writeLine,
                               "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
                    } else {
                        sprintf(writeLine,
                                "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
                                atomTypeString,
                                atomData[0],
                                atomData[1],
                                atomData[2],
                                atomData[3],
                                atomData[4],
                                atomData[5],
                                atomData[6],
                                atomData[7],
                                atomData[8],
                                atomData[9],
                                atomData[10],
                                atomData[11],
                                atomData[12]);
                    }


                    os << writeLine;

                } //end for(iter = integrableObject.begin())
            }
        } //end for(i = 0; i < mpiSim->getNmol())

        os.flush();

        sprintf(checkPointMsg, "Sucessfully took a dump.\n");

        MPIcheckPoint();

        delete [] potatoes;
    } else {

        // worldRank != 0, so I'm a remote node.  

        // Set my magic potato to 0:

        myPotato = 0;

        for(i = 0; i < info_->getNGlobalMolecules(); i++) {

            // Am I the node which has this integrableObject?
            whichNode = info_->getMolToProc(i);
            if (whichNode == worldRank) {
                if (myPotato + 1 >= MAXTAG) {

                    // The potato was going to exceed the maximum value, 
                    // so wrap this processor potato back to 0 (and block until
                    // node 0 says we can go:

                    MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD,
                             &istatus);
                }

                mol = info_->getMoleculeByGlobalIndex(i);

                
                nCurObj = mol->getNIntegrableObjects();

                MPI_Send(&nCurObj, 1, MPI_INT, 0, myPotato, MPI_COMM_WORLD);
                myPotato++;

                for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL; 
                    integrableObject = mol->nextIntegrableObject(ii)) {

                    if (myPotato + 2 >= MAXTAG) {

                        // The potato was going to exceed the maximum value, 
                        // so wrap this processor potato back to 0 (and block until
                        // node 0 says we can go:

                        MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD,
                                 &istatus);
                    }

                    atomTypeString = integrableObject->getType().c_str();

                    pos = integrableObject->getPos();
                    vel = integrableObject->getVel();

                    atomData[0] = pos[0];
                    atomData[1] = pos[1];
                    atomData[2] = pos[2];

                    atomData[3] = vel[0];
                    atomData[4] = vel[1];
                    atomData[5] = vel[2];

                    isDirectional = 0;

                    if (integrableObject->isDirectional()) {
                        isDirectional = 1;

                        q = integrableObject->getQ();
                        ji = integrableObject->getJ();

                        atomData[6] = q[0];
                        atomData[7] = q[1];
                        atomData[8] = q[2];
                        atomData[9] = q[3];

                        atomData[10] = ji[0];
                        atomData[11] = ji[1];
                        atomData[12] = ji[2];
                    }

                    strncpy(MPIatomTypeString, atomTypeString, MINIBUFFERSIZE);

                    // null terminate the string before sending (just in case):
                    MPIatomTypeString[MINIBUFFERSIZE - 1] = '\0';

                    MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
                             myPotato, MPI_COMM_WORLD);

                    myPotato++;

                    if (isDirectional) {
                        MPI_Send(atomData, 13, MPI_DOUBLE, 0, myPotato,
                                 MPI_COMM_WORLD);
                    } else {
                        MPI_Send(atomData, 6, MPI_DOUBLE, 0, myPotato,
                                 MPI_COMM_WORLD);
                    }

                    myPotato++;
                }
                    
            }
            
        }

        sprintf(checkPointMsg, "Sucessfully took a dump.\n");
        MPIcheckPoint();
    }

#endif // is_mpi

}

}//end namespace oopse
Revision:	1739
Committed:	Mon Nov 15 18:02:15 2004 UTC (19 years, 9 months ago) by tim
File size:	16723 byte(s)
Log Message:	finish DumpReader, DumpWriter.Next Step is LJFF and integrators
#	User	Rev	Content
1	tim	1739	#include "io/DumpWrite.hpp"
2			#include "utils/simError.h"
3	tim	1727
4			#ifdef IS_MPI
5			#include <mpi.h>
6			#endif //is_mpi
7	tim	1739	namespace oopse {
8			DumpWriter::DumpWriter(SimInfo* info, const std::string& filename)
9			: info_(info), filename_(filename){
10	tim	1727	#ifdef IS_MPI
11
12			if (worldRank == 0) {
13			#endif // is_mpi
14
15	tim	1739	dumpFile.open(filename_.c_str(), ios::out \| ios::trunc);
16	tim	1727
17			if (!dumpFile) {
18			sprintf(painCave.errMsg, "Could not open \"%s\" for dump output.\n",
19	tim	1739	info_->filename_.c_str());
20	tim	1727	painCave.isFatal = 1;
21			simError();
22			}
23
24			#ifdef IS_MPI
25
26			}
27
28			sprintf(checkPointMsg, "Sucessfully opened output file for dumping.\n");
29			MPIcheckPoint();
30
31			#endif // is_mpi
32
33			}
34
35			DumpWriter::~DumpWriter() {
36
37			#ifdef IS_MPI
38
39			if (worldRank == 0) {
40			#endif // is_mpi
41
42			dumpFile.close();
43
44			#ifdef IS_MPI
45
46			}
47
48			#endif // is_mpi
49
50			}
51
52	tim	1739	void writeCommentLine(Snapshot* s) {
53	tim	1727
54	tim	1739	double currentTime;
55			Mat3x3d hmat;
56			double chi;
57			double integralOfChiDt;
58			Mat3x3d eta;
59
60			currentTime = s->getTime();
61			hmat = s->getHmat();
62			chi = s->getChi();
63			integralOfChiDt = s->getIntegralOfChiDt();
64			eta = s->getEta();
65
66			os << currentTime << ";\t"
67			<< hmat[0][0] << "\t" << hmat[1][0] << "\t" << hmat[2][0] << ";\t"
68			<< hmat[0][1] << "\t" << hmat[1][1] << "\t" << hmat[2][1] << ";\t"
69			<< hmat[0][2] << "\t" << hmat[1][2] << "\t" << hmat[2][2] << ";";
70	tim	1727
71	tim	1739	//write out additional parameters, such as chi and eta
72	tim	1727
73	tim	1739	os << chi << "\t" << integralOfChiDt << "\t;"
74	tim	1727
75	tim	1739	os << eta[0][0] << "\t" << eta[1][0] << "\t" << eta[2][0] << ";\t"
76			<< eta[0][1] << "\t" << eta[1][1] << "\t" << eta[2][1] << ";\t"
77			<< eta[0][2] << "\t" << eta[1][2] << "\t" << eta[2][2] << ";";
78
79			os << endl;
80	tim	1727	}
81
82	tim	1739	void DumpWriter::writeFrame(std::ostream& os) {
83			const int BUFFERSIZE = 2000;
84			const int MINIBUFFERSIZE = 100;
85	tim	1727
86	tim	1739	char tempBuffer[BUFFERSIZE];
87			char writeLine[BUFFERSIZE];
88	tim	1727
89	tim	1739	Quat4d q;
90			Vector3d ji;
91			Vector3d pos;
92			Vector3d vel;
93	tim	1727
94	tim	1739	Molecule* mol;
95			StuntDouble* integrableObject;
96			typename SimInfo::MoleculeIterator mi;
97			typename Molecule::IntegrableObjectIterator ii;
98
99			int nTotObjects;
100			nTotObjects = info_->getNGlobalIntegrableObjects();
101	tim	1727
102	tim	1739	#ifndef IS_MPI
103	tim	1727
104
105	tim	1739	os << nTotObjects << "\n";
106
107			writeCommentLine(info_->getSnapshotManager()->getCurrentSnapshot());
108	tim	1727
109	tim	1739	for (mol = info_->beginMolecule(mi); mol != NULL; mol = info_->nextMolecule(mi)) {
110	tim	1727
111	tim	1739	for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL;
112			integrableObject = mol->nextIntegrableObject(ii)) {
113
114	tim	1727
115	tim	1739	pos = integrableObject->getPos();
116			vel = integrableObject->getVel();
117	tim	1727
118	tim	1739	sprintf(tempBuffer, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
119			integrableObject->getType(),
120			pos[0], pos[1], pos[2],
121			vel[0], vel[1], vel[2]);
122	tim	1727
123	tim	1739	strcpy(writeLine, tempBuffer);
124	tim	1727
125	tim	1739	if (integrableObject->isDirectional()) {
126			q = integrableObject->getQ();
127			ji = integrableObject->getJ();
128	tim	1727
129	tim	1739	sprintf(tempBuffer, "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
130			q[0], q[1], q[2], q[3],
131			ji[0], ji[1], ji[2]);
132			strcat(writeLine, tempBuffer);
133			} else {
134			strcat(writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
135			}
136	tim	1727
137	tim	1739	os << writeLine;
138	tim	1727
139			}
140			}
141
142	tim	1739	#else // is_mpi
143	tim	1727	/*********************************************************************
144			* Documentation? You want DOCUMENTATION?
145			*
146			* Why all the potatoes below?
147			*
148			* To make a long story short, the original version of DumpWriter
149			* worked in the most inefficient way possible. Node 0 would
150			* poke each of the node for an individual atom's formatted data
151			* as node 0 worked its way down the global index. This was particularly
152			* inefficient since the method blocked all processors at every atom
153			* (and did it twice!).
154			*
155			* An intermediate version of DumpWriter could be described from Node
156			* zero's perspective as follows:
157			*
158			* 1) Have 100 of your friends stand in a circle.
159			* 2) When you say go, have all of them start tossing potatoes at
160			* you (one at a time).
161			* 3) Catch the potatoes.
162			*
163			* It was an improvement, but MPI has buffers and caches that could
164			* best be described in this analogy as "potato nets", so there's no
165			* need to block the processors atom-by-atom.
166			*
167			* This new and improved DumpWriter works in an even more efficient
168			* way:
169			*
170			* 1) Have 100 of your friend stand in a circle.
171			* 2) When you say go, have them start tossing 5-pound bags of
172			* potatoes at you.
173			* 3) Once you've caught a friend's bag of potatoes,
174			* toss them a spud to let them know they can toss another bag.
175			*
176			* How's THAT for documentation?
177			*
178			*********************************************************************/
179	tim	1739	const int masterNode = 0;
180	tim	1727
181			int * potatoes;
182			int myPotato;
183			int nProc;
184			int which_node;
185			double atomData[13];
186			int isDirectional;
187	tim	1739	const char * atomTypeString;
188	tim	1727	char MPIatomTypeString[MINIBUFFERSIZE];
189			int msgLen; // the length of message actually recieved at master nodes
190	tim	1739	int haveError;
191			MPI_Status istatus;
192			int nCurObj;
193
194			// code to find maximum tag value
195			int * tagub;
196			int flag;
197			int MAXTAG;
198	tim	1727	MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);
199
200			if (flag) {
201			MAXTAG = *tagub;
202			} else {
203			MAXTAG = 32767;
204			}
205
206	tim	1739	if (worldRank == masterNode) { //master node (node 0) is responsible for writing the dump file
207	tim	1727
208			// Node 0 needs a list of the magic potatoes for each processor;
209
210	tim	1739	MPI_Comm_size(MPI_COMM_WORLD, &nProc);
211	tim	1727	potatoes = new int[nProc];
212
213			//write out the comment lines
214			for(i = 0; i < nProc; i++) {
215			potatoes[i] = 0;
216			}
217
218
219	tim	1739	os << nTotObjects << "\n";
220			writeCommentLine(info_->getSnapshotManager()->getCurrentSnapshot());
221	tim	1727
222	tim	1739	for(i = 0; i < info_->getNGlobalMolecules(); i++) {
223	tim	1727
224			// Get the Node number which has this atom;
225
226	tim	1739	which_node = info_->getMolToProc(i);
227	tim	1727
228	tim	1739	if (which_node != masterNode) { //current molecule is in slave node
229	tim	1727	if (potatoes[which_node] + 1 >= MAXTAG) {
230			// The potato was going to exceed the maximum value,
231			// so wrap this processor potato back to 0:
232
233			potatoes[which_node] = 0;
234			MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0,
235			MPI_COMM_WORLD);
236			}
237
238			myPotato = potatoes[which_node];
239
240			//recieve the number of integrableObject in current molecule
241			MPI_Recv(&nCurObj, 1, MPI_INT, which_node, myPotato,
242			MPI_COMM_WORLD, &istatus);
243			myPotato++;
244
245			for(int l = 0; l < nCurObj; l++) {
246			if (potatoes[which_node] + 2 >= MAXTAG) {
247			// The potato was going to exceed the maximum value,
248			// so wrap this processor potato back to 0:
249
250			potatoes[which_node] = 0;
251			MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node,
252			0, MPI_COMM_WORLD);
253			}
254
255			MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR,
256			which_node, myPotato, MPI_COMM_WORLD,
257			&istatus);
258
259			atomTypeString = MPIatomTypeString;
260
261			myPotato++;
262
263			MPI_Recv(atomData, 13, MPI_DOUBLE, which_node, myPotato,
264			MPI_COMM_WORLD, &istatus);
265			myPotato++;
266
267			MPI_Get_count(&istatus, MPI_DOUBLE, &msgLen);
268
269			if (msgLen == 13)
270			isDirectional = 1;
271			else
272			isDirectional = 0;
273
274			// If we've survived to here, format the line:
275
276			if (!isDirectional) {
277			sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
278			atomTypeString, atomData[0],
279			atomData[1], atomData[2],
280			atomData[3], atomData[4],
281			atomData[5]);
282
283			strcat(writeLine,
284			"0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
285			} else {
286			sprintf(writeLine,
287			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
288			atomTypeString,
289			atomData[0],
290			atomData[1],
291			atomData[2],
292			atomData[3],
293			atomData[4],
294			atomData[5],
295			atomData[6],
296			atomData[7],
297			atomData[8],
298			atomData[9],
299			atomData[10],
300			atomData[11],
301			atomData[12]);
302			}
303
304	tim	1739	os << writeLine;
305
306	tim	1727	} // end for(int l =0)
307
308			potatoes[which_node] = myPotato;
309	tim	1739	} else { //master node has current molecule
310	tim	1727
311	tim	1739	mol = info_->getMoleculeByGlobalIndex(i);
312	tim	1727
313	tim	1739	if (mol == NULL) {
314			strcpy(painCave.errMsg, "Molecule not found on node %d!", worldRank);
315			painCave.isFatal = 1;
316			simError();
317			}
318
319			for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL;
320			integrableObject = mol->nextIntegrableObject(ii)) {
321
322			atomTypeString = integrableObject->getType().c_str();
323	tim	1727
324	tim	1739	pos = integrableObject->getPos();
325			vel = integrableObject->getVel();
326	tim	1727
327			atomData[0] = pos[0];
328			atomData[1] = pos[1];
329			atomData[2] = pos[2];
330
331			atomData[3] = vel[0];
332			atomData[4] = vel[1];
333			atomData[5] = vel[2];
334
335			isDirectional = 0;
336
337	tim	1739	if (integrableObject->isDirectional()) {
338	tim	1727	isDirectional = 1;
339
340	tim	1739	q = integrableObject->getQ();
341			ji = integrableObject->getJ();
342	tim	1727
343			for(int j = 0; j < 6; j++) {
344			atomData[j] = atomData[j];
345			}
346
347			atomData[6] = q[0];
348			atomData[7] = q[1];
349			atomData[8] = q[2];
350			atomData[9] = q[3];
351
352			atomData[10] = ji[0];
353			atomData[11] = ji[1];
354			atomData[12] = ji[2];
355			}
356
357			// If we've survived to here, format the line:
358
359			if (!isDirectional) {
360			sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
361			atomTypeString, atomData[0],
362			atomData[1], atomData[2],
363			atomData[3], atomData[4],
364			atomData[5]);
365
366			strcat(writeLine,
367			"0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
368			} else {
369			sprintf(writeLine,
370			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
371			atomTypeString,
372			atomData[0],
373			atomData[1],
374			atomData[2],
375			atomData[3],
376			atomData[4],
377			atomData[5],
378			atomData[6],
379			atomData[7],
380			atomData[8],
381			atomData[9],
382			atomData[10],
383			atomData[11],
384			atomData[12]);
385			}
386
387	tim	1739
388			os << writeLine;
389
390	tim	1727	} //end for(iter = integrableObject.begin())
391			}
392			} //end for(i = 0; i < mpiSim->getNmol())
393
394	tim	1739	os.flush();
395	tim	1727
396			sprintf(checkPointMsg, "Sucessfully took a dump.\n");
397
398			MPIcheckPoint();
399
400			delete [] potatoes;
401			} else {
402
403			// worldRank != 0, so I'm a remote node.
404
405			// Set my magic potato to 0:
406
407			myPotato = 0;
408
409	tim	1739	for(i = 0; i < info_->getNGlobalMolecules(); i++) {
410	tim	1727
411			// Am I the node which has this integrableObject?
412	tim	1739	whichNode = info_->getMolToProc(i);
413			if (whichNode == worldRank) {
414	tim	1727	if (myPotato + 1 >= MAXTAG) {
415
416			// The potato was going to exceed the maximum value,
417			// so wrap this processor potato back to 0 (and block until
418			// node 0 says we can go:
419
420			MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD,
421			&istatus);
422			}
423
424	tim	1739	mol = info_->getMoleculeByGlobalIndex(i);
425	tim	1727
426	tim	1739
427			nCurObj = mol->getNIntegrableObjects();
428	tim	1727
429			MPI_Send(&nCurObj, 1, MPI_INT, 0, myPotato, MPI_COMM_WORLD);
430			myPotato++;
431
432	tim	1739	for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL;
433			integrableObject = mol->nextIntegrableObject(ii)) {
434
435	tim	1727	if (myPotato + 2 >= MAXTAG) {
436
437			// The potato was going to exceed the maximum value,
438			// so wrap this processor potato back to 0 (and block until
439			// node 0 says we can go:
440
441			MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD,
442			&istatus);
443			}
444
445	tim	1739	atomTypeString = integrableObject->getType().c_str();
446	tim	1727
447	tim	1739	pos = integrableObject->getPos();
448			vel = integrableObject->getVel();
449	tim	1727
450			atomData[0] = pos[0];
451			atomData[1] = pos[1];
452			atomData[2] = pos[2];
453
454			atomData[3] = vel[0];
455			atomData[4] = vel[1];
456			atomData[5] = vel[2];
457
458			isDirectional = 0;
459
460	tim	1739	if (integrableObject->isDirectional()) {
461	tim	1727	isDirectional = 1;
462
463	tim	1739	q = integrableObject->getQ();
464			ji = integrableObject->getJ();
465	tim	1727
466			atomData[6] = q[0];
467			atomData[7] = q[1];
468			atomData[8] = q[2];
469			atomData[9] = q[3];
470
471			atomData[10] = ji[0];
472			atomData[11] = ji[1];
473			atomData[12] = ji[2];
474			}
475
476			strncpy(MPIatomTypeString, atomTypeString, MINIBUFFERSIZE);
477
478			// null terminate the string before sending (just in case):
479			MPIatomTypeString[MINIBUFFERSIZE - 1] = '\0';
480
481			MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
482			myPotato, MPI_COMM_WORLD);
483
484			myPotato++;
485
486			if (isDirectional) {
487			MPI_Send(atomData, 13, MPI_DOUBLE, 0, myPotato,
488			MPI_COMM_WORLD);
489			} else {
490			MPI_Send(atomData, 6, MPI_DOUBLE, 0, myPotato,
491			MPI_COMM_WORLD);
492			}
493
494			myPotato++;
495			}
496	tim	1739
497	tim	1727	}
498	tim	1739
499	tim	1727	}
500
501			sprintf(checkPointMsg, "Sucessfully took a dump.\n");
502			MPIcheckPoint();
503			}
504
505			#endif // is_mpi
506
507			}
508
509	tim	1739	}//end namespace oopse