| 1 | < | #define _LARGEFILE_SOURCE64 | 
| 2 | < | #define _FILE_OFFSET_BITS 64 | 
| 1 | > | /* | 
| 2 | > | * Copyright (c) 2005 The University of Notre Dame. All Rights Reserved. | 
| 3 | > | * | 
| 4 | > | * The University of Notre Dame grants you ("Licensee") a | 
| 5 | > | * non-exclusive, royalty free, license to use, modify and | 
| 6 | > | * redistribute this software in source and binary code form, provided | 
| 7 | > | * that the following conditions are met: | 
| 8 | > | * | 
| 9 | > | * 1. Acknowledgement of the program authors must be made in any | 
| 10 | > | *    publication of scientific results based in part on use of the | 
| 11 | > | *    program.  An acceptable form of acknowledgement is citation of | 
| 12 | > | *    the article in which the program was described (Matthew | 
| 13 | > | *    A. Meineke, Charles F. Vardeman II, Teng Lin, Christopher | 
| 14 | > | *    J. Fennell and J. Daniel Gezelter, "OOPSE: An Object-Oriented | 
| 15 | > | *    Parallel Simulation Engine for Molecular Dynamics," | 
| 16 | > | *    J. Comput. Chem. 26, pp. 252-271 (2005)) | 
| 17 | > | * | 
| 18 | > | * 2. Redistributions of source code must retain the above copyright | 
| 19 | > | *    notice, this list of conditions and the following disclaimer. | 
| 20 | > | * | 
| 21 | > | * 3. Redistributions in binary form must reproduce the above copyright | 
| 22 | > | *    notice, this list of conditions and the following disclaimer in the | 
| 23 | > | *    documentation and/or other materials provided with the | 
| 24 | > | *    distribution. | 
| 25 | > | * | 
| 26 | > | * This software is provided "AS IS," without a warranty of any | 
| 27 | > | * kind. All express or implied conditions, representations and | 
| 28 | > | * warranties, including any implied warranty of merchantability, | 
| 29 | > | * fitness for a particular purpose or non-infringement, are hereby | 
| 30 | > | * excluded.  The University of Notre Dame and its licensors shall not | 
| 31 | > | * be liable for any damages suffered by licensee as a result of | 
| 32 | > | * using, modifying or distributing the software or its | 
| 33 | > | * derivatives. In no event will the University of Notre Dame or its | 
| 34 | > | * licensors be liable for any lost revenue, profit or data, or for | 
| 35 | > | * direct, indirect, special, consequential, incidental or punitive | 
| 36 | > | * damages, however caused and regardless of the theory of liability, | 
| 37 | > | * arising out of the use of or inability to use software, even if the | 
| 38 | > | * University of Notre Dame has been advised of the possibility of | 
| 39 | > | * such damages. | 
| 40 | > | */ | 
| 41 | > |  | 
| 42 | > | #include "io/DumpWriter.hpp" | 
| 43 | > | #include "primitives/Molecule.hpp" | 
| 44 | > | #include "utils/simError.h" | 
| 45 | > | #include "io/basic_teebuf.hpp" | 
| 46 | > | #include "io/gzstream.hpp" | 
| 47 | > | #include "io/Globals.hpp" | 
| 48 |  |  | 
| 4 | – | #include <string.h> | 
| 5 | – | #include <iostream> | 
| 6 | – | #include <fstream> | 
| 7 | – | #include <algorithm> | 
| 8 | – | #include <utility> | 
| 9 | – |  | 
| 49 |  | #ifdef IS_MPI | 
| 50 |  | #include <mpi.h> | 
| 12 | – | #include "brains/mpiSimulation.hpp" | 
| 13 | – |  | 
| 14 | – | namespace dWrite{ | 
| 15 | – | void DieDieDie( void ); | 
| 16 | – | } | 
| 17 | – |  | 
| 18 | – | using namespace dWrite; | 
| 51 |  | #endif //is_mpi | 
| 52 |  |  | 
| 53 | < | #include "io/ReadWrite.hpp" | 
| 22 | < | #include "utils/simError.h" | 
| 53 | > | namespace oopse { | 
| 54 |  |  | 
| 55 | < | DumpWriter::DumpWriter( SimInfo* the_entry_plug ){ | 
| 55 | > | DumpWriter::DumpWriter(SimInfo* info) | 
| 56 | > | : info_(info), filename_(info->getDumpFileName()), eorFilename_(info->getFinalConfigFileName()){ | 
| 57 |  |  | 
| 58 | < | entry_plug = the_entry_plug; | 
| 58 | > | Globals* simParams = info->getSimParams(); | 
| 59 | > | needCompression_ = simParams->getCompressDumpFile(); | 
| 60 | > | needForceVector_ = simParams->getDumpForceVector(); | 
| 61 |  |  | 
| 62 | + | #ifdef HAVE_LIBZ | 
| 63 | + | if (needCompression_) { | 
| 64 | + | filename_ += ".gz"; | 
| 65 | + | eorFilename_ += ".gz"; | 
| 66 | + | } | 
| 67 | + | #endif | 
| 68 | + |  | 
| 69 |  | #ifdef IS_MPI | 
| 70 | < | if(worldRank == 0 ){ | 
| 70 | > |  | 
| 71 | > | if (worldRank == 0) { | 
| 72 |  | #endif // is_mpi | 
| 73 |  |  | 
| 32 | – | dumpFile.open(entry_plug->sampleName.c_str(), ios::out | ios::trunc ); | 
| 74 |  |  | 
| 75 | < | if( !dumpFile ){ | 
| 75 | > | dumpFile_ = createOStream(filename_); | 
| 76 |  |  | 
| 77 | < | sprintf( painCave.errMsg, | 
| 78 | < | "Could not open \"%s\" for dump output.\n", | 
| 79 | < | entry_plug->sampleName.c_str()); | 
| 80 | < | painCave.isFatal = 1; | 
| 81 | < | simError(); | 
| 82 | < | } | 
| 77 | > | if (!dumpFile_) { | 
| 78 | > | sprintf(painCave.errMsg, "Could not open \"%s\" for dump output.\n", | 
| 79 | > | filename_.c_str()); | 
| 80 | > | painCave.isFatal = 1; | 
| 81 | > | simError(); | 
| 82 | > | } | 
| 83 |  |  | 
| 84 |  | #ifdef IS_MPI | 
| 44 | – | } | 
| 85 |  |  | 
| 86 | < | //sort the local atoms by global index | 
| 47 | < | sortByGlobalIndex(); | 
| 48 | < |  | 
| 49 | < | sprintf( checkPointMsg, | 
| 50 | < | "Sucessfully opened output file for dumping.\n"); | 
| 51 | < | MPIcheckPoint(); | 
| 52 | < | #endif // is_mpi | 
| 53 | < | } | 
| 86 | > | } | 
| 87 |  |  | 
| 88 | < | DumpWriter::~DumpWriter( ){ | 
| 88 | > | sprintf(checkPointMsg, "Sucessfully opened output file for dumping.\n"); | 
| 89 | > | MPIcheckPoint(); | 
| 90 |  |  | 
| 57 | – | #ifdef IS_MPI | 
| 58 | – | if(worldRank == 0 ){ | 
| 91 |  | #endif // is_mpi | 
| 92 |  |  | 
| 93 | < | dumpFile.close(); | 
| 93 | > | } | 
| 94 |  |  | 
| 63 | – | #ifdef IS_MPI | 
| 64 | – | } | 
| 65 | – | #endif // is_mpi | 
| 66 | – | } | 
| 95 |  |  | 
| 96 | < | #ifdef IS_MPI | 
| 96 | > | DumpWriter::DumpWriter(SimInfo* info, const std::string& filename) | 
| 97 | > | : info_(info), filename_(filename){ | 
| 98 |  |  | 
| 99 | < | /** | 
| 100 | < | * A hook function to load balancing | 
| 72 | < | */ | 
| 99 | > | Globals* simParams = info->getSimParams(); | 
| 100 | > | eorFilename_ = filename_.substr(0, filename_.rfind(".")) + ".eor"; | 
| 101 |  |  | 
| 102 | < | void DumpWriter::update(){ | 
| 103 | < | sortByGlobalIndex(); | 
| 76 | < | } | 
| 77 | < |  | 
| 78 | < | /** | 
| 79 | < | * Auxiliary sorting function | 
| 80 | < | */ | 
| 81 | < |  | 
| 82 | < | bool indexSortingCriterion(const pair<int, int>& p1, const pair<int, int>& p2){ | 
| 83 | < | return p1.second < p2.second; | 
| 84 | < | } | 
| 102 | > | needCompression_ = simParams->getCompressDumpFile(); | 
| 103 | > | needForceVector_ = simParams->getDumpForceVector(); | 
| 104 |  |  | 
| 105 | < | /** | 
| 106 | < | * Sorting the local index by global index | 
| 107 | < | */ | 
| 108 | < |  | 
| 109 | < | void DumpWriter::sortByGlobalIndex(){ | 
| 91 | < | Molecule* mols = entry_plug->molecules; | 
| 92 | < | indexArray.clear(); | 
| 93 | < |  | 
| 94 | < | for(int i = 0; i < entry_plug->n_mol;i++) | 
| 95 | < | indexArray.push_back(make_pair(i, mols[i].getGlobalIndex())); | 
| 96 | < |  | 
| 97 | < | sort(indexArray.begin(), indexArray.end(), indexSortingCriterion); | 
| 98 | < | } | 
| 99 | < |  | 
| 105 | > | #ifdef HAVE_LIBZ | 
| 106 | > | if (needCompression_) { | 
| 107 | > | filename_ += ".gz"; | 
| 108 | > | eorFilename_ += ".gz"; | 
| 109 | > | } | 
| 110 |  | #endif | 
| 111 | < |  | 
| 102 | < | void DumpWriter::writeDump(double currentTime){ | 
| 103 | < |  | 
| 104 | < | ofstream finalOut; | 
| 105 | < | vector<ofstream*> fileStreams; | 
| 106 | < |  | 
| 111 | > |  | 
| 112 |  | #ifdef IS_MPI | 
| 113 | < | if(worldRank == 0 ){ | 
| 114 | < | #endif | 
| 110 | < | finalOut.open( entry_plug->finalName.c_str(), ios::out | ios::trunc ); | 
| 111 | < | if( !finalOut ){ | 
| 112 | < | sprintf( painCave.errMsg, | 
| 113 | < | "Could not open \"%s\" for final dump output.\n", | 
| 114 | < | entry_plug->finalName.c_str() ); | 
| 115 | < | painCave.isFatal = 1; | 
| 116 | < | simError(); | 
| 117 | < | } | 
| 118 | < | #ifdef IS_MPI | 
| 119 | < | } | 
| 113 | > |  | 
| 114 | > | if (worldRank == 0) { | 
| 115 |  | #endif // is_mpi | 
| 116 |  |  | 
| 122 | – | fileStreams.push_back(&finalOut); | 
| 123 | – | fileStreams.push_back(&dumpFile); | 
| 117 |  |  | 
| 118 | < | writeFrame(fileStreams, currentTime); | 
| 118 | > | dumpFile_ = createOStream(filename_); | 
| 119 |  |  | 
| 120 | + | if (!dumpFile_) { | 
| 121 | + | sprintf(painCave.errMsg, "Could not open \"%s\" for dump output.\n", | 
| 122 | + | filename_.c_str()); | 
| 123 | + | painCave.isFatal = 1; | 
| 124 | + | simError(); | 
| 125 | + | } | 
| 126 | + |  | 
| 127 |  | #ifdef IS_MPI | 
| 128 | – | finalOut.close(); | 
| 129 | – | #endif | 
| 130 | – |  | 
| 131 | – | } | 
| 128 |  |  | 
| 129 | < | void DumpWriter::writeFinal(double currentTime){ | 
| 129 | > | } | 
| 130 |  |  | 
| 131 | < | ofstream finalOut; | 
| 132 | < | vector<ofstream*> fileStreams; | 
| 131 | > | sprintf(checkPointMsg, "Sucessfully opened output file for dumping.\n"); | 
| 132 | > | MPIcheckPoint(); | 
| 133 |  |  | 
| 138 | – | #ifdef IS_MPI | 
| 139 | – | if(worldRank == 0 ){ | 
| 134 |  | #endif // is_mpi | 
| 135 |  |  | 
| 142 | – | finalOut.open( entry_plug->finalName.c_str(), ios::out | ios::trunc ); | 
| 143 | – |  | 
| 144 | – | if( !finalOut ){ | 
| 145 | – | sprintf( painCave.errMsg, | 
| 146 | – | "Could not open \"%s\" for final dump output.\n", | 
| 147 | – | entry_plug->finalName.c_str() ); | 
| 148 | – | painCave.isFatal = 1; | 
| 149 | – | simError(); | 
| 136 |  | } | 
| 137 |  |  | 
| 138 | < | #ifdef IS_MPI | 
| 153 | < | } | 
| 154 | < | #endif // is_mpi | 
| 155 | < |  | 
| 156 | < | fileStreams.push_back(&finalOut); | 
| 157 | < | writeFrame(fileStreams, currentTime); | 
| 138 | > | DumpWriter::~DumpWriter() { | 
| 139 |  |  | 
| 140 |  | #ifdef IS_MPI | 
| 160 | – | finalOut.close(); | 
| 161 | – | #endif | 
| 162 | – |  | 
| 163 | – | } | 
| 141 |  |  | 
| 142 | < | void DumpWriter::writeFrame( vector<ofstream*>& outFile, double currentTime ){ | 
| 142 | > | if (worldRank == 0) { | 
| 143 | > | #endif // is_mpi | 
| 144 |  |  | 
| 145 | < | const int BUFFERSIZE = 2000; | 
| 168 | < | const int MINIBUFFERSIZE = 100; | 
| 145 | > | delete dumpFile_; | 
| 146 |  |  | 
| 170 | – | char tempBuffer[BUFFERSIZE]; | 
| 171 | – | char writeLine[BUFFERSIZE]; | 
| 172 | – |  | 
| 173 | – | int i; | 
| 174 | – | unsigned int k; | 
| 175 | – |  | 
| 147 |  | #ifdef IS_MPI | 
| 177 | – |  | 
| 178 | – | /********************************************************************* | 
| 179 | – | * Documentation?  You want DOCUMENTATION? | 
| 180 | – | * | 
| 181 | – | * Why all the potatoes below? | 
| 182 | – | * | 
| 183 | – | * To make a long story short, the original version of DumpWriter | 
| 184 | – | * worked in the most inefficient way possible.  Node 0 would | 
| 185 | – | * poke each of the node for an individual atom's formatted data | 
| 186 | – | * as node 0 worked its way down the global index. This was particularly | 
| 187 | – | * inefficient since the method blocked all processors at every atom | 
| 188 | – | * (and did it twice!). | 
| 189 | – | * | 
| 190 | – | * An intermediate version of DumpWriter could be described from Node | 
| 191 | – | * zero's perspective as follows: | 
| 192 | – | * | 
| 193 | – | *  1) Have 100 of your friends stand in a circle. | 
| 194 | – | *  2) When you say go, have all of them start tossing potatoes at | 
| 195 | – | *     you (one at a time). | 
| 196 | – | *  3) Catch the potatoes. | 
| 197 | – | * | 
| 198 | – | * It was an improvement, but MPI has buffers and caches that could | 
| 199 | – | * best be described in this analogy as "potato nets", so there's no | 
| 200 | – | * need to block the processors atom-by-atom. | 
| 201 | – | * | 
| 202 | – | * This new and improved DumpWriter works in an even more efficient | 
| 203 | – | * way: | 
| 204 | – | * | 
| 205 | – | *  1) Have 100 of your friend stand in a circle. | 
| 206 | – | *  2) When you say go, have them start tossing 5-pound bags of | 
| 207 | – | *     potatoes at you. | 
| 208 | – | *  3) Once you've caught a friend's bag of potatoes, | 
| 209 | – | *     toss them a spud to let them know they can toss another bag. | 
| 210 | – | * | 
| 211 | – | * How's THAT for documentation? | 
| 212 | – | * | 
| 213 | – | *********************************************************************/ | 
| 148 |  |  | 
| 149 | < | int *potatoes; | 
| 216 | < | int myPotato; | 
| 149 | > | } | 
| 150 |  |  | 
| 151 | < | int nProc; | 
| 219 | < | int j, which_node, done, which_atom, local_index, currentIndex; | 
| 220 | < | double atomData[13]; | 
| 221 | < | int isDirectional; | 
| 222 | < | char* atomTypeString; | 
| 223 | < | char MPIatomTypeString[MINIBUFFERSIZE]; | 
| 224 | < | int nObjects; | 
| 225 | < | int msgLen; // the length of message actually recieved at master nodes | 
| 226 | < | #endif //is_mpi | 
| 151 | > | #endif // is_mpi | 
| 152 |  |  | 
| 153 | < | double q[4], ji[3]; | 
| 229 | < | DirectionalAtom* dAtom; | 
| 230 | < | double pos[3], vel[3]; | 
| 231 | < | int nTotObjects; | 
| 232 | < | StuntDouble* sd; | 
| 233 | < | char* molName; | 
| 234 | < | vector<StuntDouble*> integrableObjects; | 
| 235 | < | vector<StuntDouble*>::iterator iter; | 
| 236 | < | nTotObjects = entry_plug->getTotIntegrableObjects(); | 
| 237 | < | #ifndef IS_MPI | 
| 238 | < |  | 
| 239 | < | for(k = 0; k < outFile.size(); k++){ | 
| 240 | < | *outFile[k] << nTotObjects << "\n"; | 
| 153 | > | } | 
| 154 |  |  | 
| 155 | < | *outFile[k] << currentTime << ";\t" | 
| 243 | < | << entry_plug->Hmat[0][0] << "\t" | 
| 244 | < | << entry_plug->Hmat[1][0] << "\t" | 
| 245 | < | << entry_plug->Hmat[2][0] << ";\t" | 
| 246 | < |  | 
| 247 | < | << entry_plug->Hmat[0][1] << "\t" | 
| 248 | < | << entry_plug->Hmat[1][1] << "\t" | 
| 249 | < | << entry_plug->Hmat[2][1] << ";\t" | 
| 155 | > | void DumpWriter::writeCommentLine(std::ostream& os, Snapshot* s) { | 
| 156 |  |  | 
| 157 | < | << entry_plug->Hmat[0][2] << "\t" | 
| 158 | < | << entry_plug->Hmat[1][2] << "\t" | 
| 159 | < | << entry_plug->Hmat[2][2] << ";"; | 
| 157 | > | double currentTime; | 
| 158 | > | Mat3x3d hmat; | 
| 159 | > | double chi; | 
| 160 | > | double integralOfChiDt; | 
| 161 | > | Mat3x3d eta; | 
| 162 | > |  | 
| 163 | > | currentTime = s->getTime(); | 
| 164 | > | hmat = s->getHmat(); | 
| 165 | > | chi = s->getChi(); | 
| 166 | > | integralOfChiDt = s->getIntegralOfChiDt(); | 
| 167 | > | eta = s->getEta(); | 
| 168 | > |  | 
| 169 | > | os << currentTime << ";\t" | 
| 170 | > | << hmat(0, 0) << "\t" << hmat(1, 0) << "\t" << hmat(2, 0) << ";\t" | 
| 171 | > | << hmat(0, 1) << "\t" << hmat(1, 1) << "\t" << hmat(2, 1) << ";\t" | 
| 172 | > | << hmat(0, 2) << "\t" << hmat(1, 2) << "\t" << hmat(2, 2) << ";\t"; | 
| 173 |  |  | 
| 174 |  | //write out additional parameters, such as chi and eta | 
| 256 | – | *outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl; | 
| 257 | – | } | 
| 258 | – |  | 
| 259 | – | for( i=0; i< entry_plug->n_mol; i++ ){ | 
| 175 |  |  | 
| 176 | < | integrableObjects = entry_plug->molecules[i].getIntegrableObjects(); | 
| 262 | < | molName = (entry_plug->compStamps[entry_plug->molecules[i].getStampID()])->getID(); | 
| 263 | < |  | 
| 264 | < | for( iter = integrableObjects.begin();iter !=  integrableObjects.end(); ++iter){ | 
| 265 | < | sd = *iter; | 
| 266 | < | sd->getPos(pos); | 
| 267 | < | sd->getVel(vel); | 
| 176 | > | os << chi << "\t" << integralOfChiDt << "\t;"; | 
| 177 |  |  | 
| 178 | < | sprintf( tempBuffer, | 
| 179 | < | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", | 
| 180 | < | sd->getType(), | 
| 181 | < | pos[0], | 
| 182 | < | pos[1], | 
| 183 | < | pos[2], | 
| 275 | < | vel[0], | 
| 276 | < | vel[1], | 
| 277 | < | vel[2]); | 
| 278 | < | strcpy( writeLine, tempBuffer ); | 
| 178 | > | os << eta(0, 0) << "\t" << eta(1, 0) << "\t" << eta(2, 0) << ";\t" | 
| 179 | > | << eta(0, 1) << "\t" << eta(1, 1) << "\t" << eta(2, 1) << ";\t" | 
| 180 | > | << eta(0, 2) << "\t" << eta(1, 2) << "\t" << eta(2, 2) << ";"; | 
| 181 | > |  | 
| 182 | > | os << "\n"; | 
| 183 | > | } | 
| 184 |  |  | 
| 185 | < | if( sd->isDirectional() ){ | 
| 185 | > | void DumpWriter::writeFrame(std::ostream& os) { | 
| 186 | > | const int BUFFERSIZE = 2000; | 
| 187 | > | const int MINIBUFFERSIZE = 100; | 
| 188 |  |  | 
| 189 | < | sd->getQ( q ); | 
| 190 | < | sd->getJ( ji ); | 
| 189 | > | char tempBuffer[BUFFERSIZE]; | 
| 190 | > | char writeLine[BUFFERSIZE]; | 
| 191 |  |  | 
| 192 | < | sprintf( tempBuffer, | 
| 193 | < | "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", | 
| 194 | < | q[0], | 
| 195 | < | q[1], | 
| 196 | < | q[2], | 
| 197 | < | q[3], | 
| 198 | < | ji[0], | 
| 199 | < | ji[1], | 
| 200 | < | ji[2]); | 
| 201 | < | strcat( writeLine, tempBuffer ); | 
| 192 | > | Quat4d q; | 
| 193 | > | Vector3d ji; | 
| 194 | > | Vector3d pos; | 
| 195 | > | Vector3d vel; | 
| 196 | > | Vector3d frc; | 
| 197 | > | Vector3d trq; | 
| 198 | > |  | 
| 199 | > | Molecule* mol; | 
| 200 | > | StuntDouble* integrableObject; | 
| 201 | > | SimInfo::MoleculeIterator mi; | 
| 202 | > | Molecule::IntegrableObjectIterator ii; | 
| 203 | > |  | 
| 204 | > | int nTotObjects; | 
| 205 | > | nTotObjects = info_->getNGlobalIntegrableObjects(); | 
| 206 | > |  | 
| 207 | > | #ifndef IS_MPI | 
| 208 | > |  | 
| 209 | > |  | 
| 210 | > | os << nTotObjects << "\n"; | 
| 211 | > |  | 
| 212 | > | writeCommentLine(os, info_->getSnapshotManager()->getCurrentSnapshot()); | 
| 213 | > |  | 
| 214 | > | for (mol = info_->beginMolecule(mi); mol != NULL; mol = info_->nextMolecule(mi)) { | 
| 215 | > |  | 
| 216 | > | for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL; | 
| 217 | > | integrableObject = mol->nextIntegrableObject(ii)) { | 
| 218 | > |  | 
| 219 | > |  | 
| 220 | > | pos = integrableObject->getPos(); | 
| 221 | > | vel = integrableObject->getVel(); | 
| 222 | > |  | 
| 223 | > | sprintf(tempBuffer, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", | 
| 224 | > | integrableObject->getType().c_str(), | 
| 225 | > | pos[0], pos[1], pos[2], | 
| 226 | > | vel[0], vel[1], vel[2]); | 
| 227 | > |  | 
| 228 | > | strcpy(writeLine, tempBuffer); | 
| 229 | > |  | 
| 230 | > | if (integrableObject->isDirectional()) { | 
| 231 | > | q = integrableObject->getQ(); | 
| 232 | > | ji = integrableObject->getJ(); | 
| 233 | > |  | 
| 234 | > | sprintf(tempBuffer, "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf", | 
| 235 | > | q[0], q[1], q[2], q[3], | 
| 236 | > | ji[0], ji[1], ji[2]); | 
| 237 | > | strcat(writeLine, tempBuffer); | 
| 238 | > | } else { | 
| 239 | > | strcat(writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0"); | 
| 240 | > | } | 
| 241 | > |  | 
| 242 | > | if (needForceVector_) { | 
| 243 | > | frc = integrableObject->getFrc(); | 
| 244 | > | trq = integrableObject->getTrq(); | 
| 245 | > |  | 
| 246 | > | sprintf(tempBuffer, "\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf", | 
| 247 | > | frc[0], frc[1], frc[2], | 
| 248 | > | trq[0], trq[1], trq[2]); | 
| 249 | > | strcat(writeLine, tempBuffer); | 
| 250 | > | } | 
| 251 | > |  | 
| 252 | > | strcat(writeLine, "\n"); | 
| 253 | > | os << writeLine; | 
| 254 | > |  | 
| 255 |  | } | 
| 296 | – | else | 
| 297 | – | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); | 
| 298 | – |  | 
| 299 | – | for(k = 0; k < outFile.size(); k++) | 
| 300 | – | *outFile[k] << writeLine; | 
| 256 |  | } | 
| 257 |  |  | 
| 258 | < | } | 
| 304 | < |  | 
| 258 | > | os.flush(); | 
| 259 |  | #else // is_mpi | 
| 260 | + | /********************************************************************* | 
| 261 | + | * Documentation?  You want DOCUMENTATION? | 
| 262 | + | * | 
| 263 | + | * Why all the potatoes below? | 
| 264 | + | * | 
| 265 | + | * To make a long story short, the original version of DumpWriter | 
| 266 | + | * worked in the most inefficient way possible.  Node 0 would | 
| 267 | + | * poke each of the node for an individual atom's formatted data | 
| 268 | + | * as node 0 worked its way down the global index. This was particularly | 
| 269 | + | * inefficient since the method blocked all processors at every atom | 
| 270 | + | * (and did it twice!). | 
| 271 | + | * | 
| 272 | + | * An intermediate version of DumpWriter could be described from Node | 
| 273 | + | * zero's perspective as follows: | 
| 274 | + | * | 
| 275 | + | *  1) Have 100 of your friends stand in a circle. | 
| 276 | + | *  2) When you say go, have all of them start tossing potatoes at | 
| 277 | + | *     you (one at a time). | 
| 278 | + | *  3) Catch the potatoes. | 
| 279 | + | * | 
| 280 | + | * It was an improvement, but MPI has buffers and caches that could | 
| 281 | + | * best be described in this analogy as "potato nets", so there's no | 
| 282 | + | * need to block the processors atom-by-atom. | 
| 283 | + | * | 
| 284 | + | * This new and improved DumpWriter works in an even more efficient | 
| 285 | + | * way: | 
| 286 | + | * | 
| 287 | + | *  1) Have 100 of your friend stand in a circle. | 
| 288 | + | *  2) When you say go, have them start tossing 5-pound bags of | 
| 289 | + | *     potatoes at you. | 
| 290 | + | *  3) Once you've caught a friend's bag of potatoes, | 
| 291 | + | *     toss them a spud to let them know they can toss another bag. | 
| 292 | + | * | 
| 293 | + | * How's THAT for documentation? | 
| 294 | + | * | 
| 295 | + | *********************************************************************/ | 
| 296 | + | const int masterNode = 0; | 
| 297 |  |  | 
| 298 | < | /* code to find maximum tag value */ | 
| 299 | < |  | 
| 300 | < | int *tagub, flag, MAXTAG; | 
| 301 | < | MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag); | 
| 302 | < | if (flag) { | 
| 303 | < | MAXTAG = *tagub; | 
| 304 | < | } else { | 
| 305 | < | MAXTAG = 32767; | 
| 306 | < | } | 
| 298 | > | int * potatoes; | 
| 299 | > | int myPotato; | 
| 300 | > | int nProc; | 
| 301 | > | int which_node; | 
| 302 | > | double atomData[19]; | 
| 303 | > | int isDirectional; | 
| 304 | > | char MPIatomTypeString[MINIBUFFERSIZE]; | 
| 305 | > | int msgLen; // the length of message actually recieved at master nodes | 
| 306 | > | int haveError; | 
| 307 | > | MPI_Status istatus; | 
| 308 | > | int nCurObj; | 
| 309 | > |  | 
| 310 | > | // code to find maximum tag value | 
| 311 | > | int * tagub; | 
| 312 | > | int flag; | 
| 313 | > | int MAXTAG; | 
| 314 | > | MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag); | 
| 315 |  |  | 
| 316 | < | int haveError; | 
| 316 | > | if (flag) { | 
| 317 | > | MAXTAG = *tagub; | 
| 318 | > | } else { | 
| 319 | > | MAXTAG = 32767; | 
| 320 | > | } | 
| 321 |  |  | 
| 322 | < | MPI_Status istatus; | 
| 320 | < | int nCurObj; | 
| 321 | < | int *MolToProcMap = mpiSim->getMolToProcMap(); | 
| 322 | > | if (worldRank == masterNode) { //master node (node 0) is responsible for writing the dump file | 
| 323 |  |  | 
| 324 | < | // write out header and node 0's coordinates | 
| 324 | > | // Node 0 needs a list of the magic potatoes for each processor; | 
| 325 |  |  | 
| 326 | < | if( worldRank == 0 ){ | 
| 326 | > | MPI_Comm_size(MPI_COMM_WORLD, &nProc); | 
| 327 | > | potatoes = new int[nProc]; | 
| 328 |  |  | 
| 329 | < | // Node 0 needs a list of the magic potatoes for each processor; | 
| 329 | > | //write out the comment lines | 
| 330 | > | for(int i = 0; i < nProc; i++) { | 
| 331 | > | potatoes[i] = 0; | 
| 332 | > | } | 
| 333 |  |  | 
| 329 | – | nProc = mpiSim->getNProcessors(); | 
| 330 | – | potatoes = new int[nProc]; | 
| 334 |  |  | 
| 335 | < | //write out the comment lines | 
| 336 | < | for (i = 0; i < nProc; i++) | 
| 334 | < | potatoes[i] = 0; | 
| 335 | < |  | 
| 336 | < | for(k = 0; k < outFile.size(); k++){ | 
| 337 | < | *outFile[k] << nTotObjects << "\n"; | 
| 335 | > | os << nTotObjects << "\n"; | 
| 336 | > | writeCommentLine(os, info_->getSnapshotManager()->getCurrentSnapshot()); | 
| 337 |  |  | 
| 338 | < | *outFile[k] << currentTime << ";\t" | 
| 340 | < | << entry_plug->Hmat[0][0] << "\t" | 
| 341 | < | << entry_plug->Hmat[1][0] << "\t" | 
| 342 | < | << entry_plug->Hmat[2][0] << ";\t" | 
| 338 | > | for(int i = 0; i < info_->getNGlobalMolecules(); i++) { | 
| 339 |  |  | 
| 340 | < | << entry_plug->Hmat[0][1] << "\t" | 
| 345 | < | << entry_plug->Hmat[1][1] << "\t" | 
| 346 | < | << entry_plug->Hmat[2][1] << ";\t" | 
| 340 | > | // Get the Node number which has this atom; | 
| 341 |  |  | 
| 342 | < | << entry_plug->Hmat[0][2] << "\t" | 
| 349 | < | << entry_plug->Hmat[1][2] << "\t" | 
| 350 | < | << entry_plug->Hmat[2][2] << ";"; | 
| 351 | < |  | 
| 352 | < | *outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl; | 
| 353 | < | } | 
| 342 | > | which_node = info_->getMolToProc(i); | 
| 343 |  |  | 
| 344 | < | currentIndex = 0; | 
| 344 | > | if (which_node != masterNode) { //current molecule is in slave node | 
| 345 | > | if (potatoes[which_node] + 1 >= MAXTAG) { | 
| 346 | > | // The potato was going to exceed the maximum value, | 
| 347 | > | // so wrap this processor potato back to 0: | 
| 348 |  |  | 
| 349 | < | for (i = 0 ; i < mpiSim->getNMolGlobal(); i++ ) { | 
| 350 | < |  | 
| 351 | < | // Get the Node number which has this atom; | 
| 352 | < |  | 
| 361 | < | which_node = MolToProcMap[i]; | 
| 362 | < |  | 
| 363 | < | if (which_node != 0) { | 
| 364 | < |  | 
| 365 | < | if (potatoes[which_node] + 1 >= MAXTAG) { | 
| 366 | < | // The potato was going to exceed the maximum value, | 
| 367 | < | // so wrap this processor potato back to 0: | 
| 349 | > | potatoes[which_node] = 0; | 
| 350 | > | MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0, | 
| 351 | > | MPI_COMM_WORLD); | 
| 352 | > | } | 
| 353 |  |  | 
| 354 | < | potatoes[which_node] = 0; | 
| 370 | < | MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0, MPI_COMM_WORLD); | 
| 371 | < |  | 
| 372 | < | } | 
| 354 | > | myPotato = potatoes[which_node]; | 
| 355 |  |  | 
| 356 | < | myPotato = potatoes[which_node]; | 
| 356 | > | //recieve the number of integrableObject in current molecule | 
| 357 | > | MPI_Recv(&nCurObj, 1, MPI_INT, which_node, myPotato, | 
| 358 | > | MPI_COMM_WORLD, &istatus); | 
| 359 | > | myPotato++; | 
| 360 |  |  | 
| 361 | < | //recieve the number of integrableObject in current molecule | 
| 362 | < | MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 363 | < | myPotato, MPI_COMM_WORLD, &istatus); | 
| 364 | < | myPotato++; | 
| 380 | < |  | 
| 381 | < | for(int l = 0; l < nCurObj; l++){ | 
| 361 | > | for(int l = 0; l < nCurObj; l++) { | 
| 362 | > | if (potatoes[which_node] + 2 >= MAXTAG) { | 
| 363 | > | // The potato was going to exceed the maximum value, | 
| 364 | > | // so wrap this processor potato back to 0: | 
| 365 |  |  | 
| 366 | < | if (potatoes[which_node] + 2 >= MAXTAG) { | 
| 367 | < | // The potato was going to exceed the maximum value, | 
| 368 | < | // so wrap this processor potato back to 0: | 
| 366 | > | potatoes[which_node] = 0; | 
| 367 | > | MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, | 
| 368 | > | 0, MPI_COMM_WORLD); | 
| 369 | > | } | 
| 370 |  |  | 
| 371 | < | potatoes[which_node] = 0; | 
| 372 | < | MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0, MPI_COMM_WORLD); | 
| 373 | < |  | 
| 390 | < | } | 
| 371 | > | MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, | 
| 372 | > | which_node, myPotato, MPI_COMM_WORLD, | 
| 373 | > | &istatus); | 
| 374 |  |  | 
| 375 | < | MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, which_node, | 
| 393 | < | myPotato, MPI_COMM_WORLD, &istatus); | 
| 375 | > | myPotato++; | 
| 376 |  |  | 
| 377 | < | atomTypeString = MPIatomTypeString; | 
| 377 | > | MPI_Recv(atomData, 19, MPI_DOUBLE, which_node, myPotato, | 
| 378 | > | MPI_COMM_WORLD, &istatus); | 
| 379 | > | myPotato++; | 
| 380 |  |  | 
| 381 | < | myPotato++; | 
| 381 | > | MPI_Get_count(&istatus, MPI_DOUBLE, &msgLen); | 
| 382 |  |  | 
| 383 | < | MPI_Recv(atomData, 13, MPI_DOUBLE, which_node, myPotato, MPI_COMM_WORLD, &istatus); | 
| 384 | < | myPotato++; | 
| 383 | > | if (msgLen == 13 || msgLen == 19) | 
| 384 | > | isDirectional = 1; | 
| 385 | > | else | 
| 386 | > | isDirectional = 0; | 
| 387 |  |  | 
| 388 | < | MPI_Get_count(&istatus, MPI_DOUBLE, &msgLen); | 
| 388 | > | // If we've survived to here, format the line: | 
| 389 |  |  | 
| 390 | < | if(msgLen  == 13) | 
| 391 | < | isDirectional = 1; | 
| 392 | < | else | 
| 393 | < | isDirectional = 0; | 
| 394 | < |  | 
| 395 | < | // If we've survived to here, format the line: | 
| 396 | < |  | 
| 397 | < | if (!isDirectional) { | 
| 398 | < |  | 
| 399 | < | sprintf( writeLine, | 
| 400 | < | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", | 
| 401 | < | atomTypeString, | 
| 402 | < | atomData[0], | 
| 403 | < | atomData[1], | 
| 404 | < | atomData[2], | 
| 405 | < | atomData[3], | 
| 406 | < | atomData[4], | 
| 407 | < | atomData[5]); | 
| 408 | < |  | 
| 409 | < | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); | 
| 410 | < |  | 
| 411 | < | } | 
| 412 | < | else { | 
| 413 | < |  | 
| 414 | < | sprintf( writeLine, | 
| 415 | < | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", | 
| 416 | < | atomTypeString, | 
| 417 | < | atomData[0], | 
| 418 | < | atomData[1], | 
| 419 | < | atomData[2], | 
| 420 | < | atomData[3], | 
| 421 | < | atomData[4], | 
| 422 | < | atomData[5], | 
| 423 | < | atomData[6], | 
| 424 | < | atomData[7], | 
| 425 | < | atomData[8], | 
| 426 | < | atomData[9], | 
| 427 | < | atomData[10], | 
| 428 | < | atomData[11], | 
| 429 | < | atomData[12]); | 
| 430 | < |  | 
| 431 | < | } | 
| 432 | < |  | 
| 433 | < | for(k = 0; k < outFile.size(); k++) | 
| 434 | < | *outFile[k] << writeLine; | 
| 390 | > | if (!isDirectional) { | 
| 391 | > | sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", | 
| 392 | > | MPIatomTypeString, atomData[0], | 
| 393 | > | atomData[1], atomData[2], | 
| 394 | > | atomData[3], atomData[4], | 
| 395 | > | atomData[5]); | 
| 396 | > |  | 
| 397 | > | strcat(writeLine, | 
| 398 | > | "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0"); | 
| 399 | > | } else { | 
| 400 | > | sprintf(writeLine, | 
| 401 | > | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf", | 
| 402 | > | MPIatomTypeString, | 
| 403 | > | atomData[0], | 
| 404 | > | atomData[1], | 
| 405 | > | atomData[2], | 
| 406 | > | atomData[3], | 
| 407 | > | atomData[4], | 
| 408 | > | atomData[5], | 
| 409 | > | atomData[6], | 
| 410 | > | atomData[7], | 
| 411 | > | atomData[8], | 
| 412 | > | atomData[9], | 
| 413 | > | atomData[10], | 
| 414 | > | atomData[11], | 
| 415 | > | atomData[12]); | 
| 416 | > | } | 
| 417 | > |  | 
| 418 | > | if (needForceVector_) { | 
| 419 | > | if (!isDirectional) { | 
| 420 | > | sprintf(writeLine, "\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf", | 
| 421 | > | atomData[6], | 
| 422 | > | atomData[7], | 
| 423 | > | atomData[8], | 
| 424 | > | atomData[9], | 
| 425 | > | atomData[10], | 
| 426 | > | atomData[11]); | 
| 427 | > | } else { | 
| 428 | > | sprintf(writeLine, "\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf", | 
| 429 | > | atomData[13], | 
| 430 | > | atomData[14], | 
| 431 | > | atomData[15], | 
| 432 | > | atomData[16], | 
| 433 | > | atomData[17], | 
| 434 | > | atomData[18]); | 
| 435 | > | } | 
| 436 | > | } | 
| 437 |  |  | 
| 438 | < | }// end for(int l =0) | 
| 439 | < | potatoes[which_node] = myPotato; | 
| 438 | > | sprintf(writeLine, "\n"); | 
| 439 | > | os << writeLine; | 
| 440 |  |  | 
| 441 | < | } | 
| 454 | < | else { | 
| 455 | < |  | 
| 456 | < | haveError = 0; | 
| 457 | < |  | 
| 458 | < | local_index = indexArray[currentIndex].first; | 
| 441 | > | } // end for(int l =0) | 
| 442 |  |  | 
| 443 | < | integrableObjects = (entry_plug->molecules[local_index]).getIntegrableObjects(); | 
| 443 | > | potatoes[which_node] = myPotato; | 
| 444 | > | } else { //master node has current molecule | 
| 445 |  |  | 
| 446 | < | for(iter= integrableObjects.begin(); iter != integrableObjects.end(); ++iter){ | 
| 463 | < | sd = *iter; | 
| 464 | < | atomTypeString = sd->getType(); | 
| 465 | < |  | 
| 466 | < | sd->getPos(pos); | 
| 467 | < | sd->getVel(vel); | 
| 468 | < |  | 
| 469 | < | atomData[0] = pos[0]; | 
| 470 | < | atomData[1] = pos[1]; | 
| 471 | < | atomData[2] = pos[2]; | 
| 446 | > | mol = info_->getMoleculeByGlobalIndex(i); | 
| 447 |  |  | 
| 448 | < | atomData[3] = vel[0]; | 
| 449 | < | atomData[4] = vel[1]; | 
| 450 | < | atomData[5] = vel[2]; | 
| 451 | < |  | 
| 452 | < | isDirectional = 0; | 
| 478 | < |  | 
| 479 | < | if( sd->isDirectional() ){ | 
| 480 | < |  | 
| 481 | < | isDirectional = 1; | 
| 448 | > | if (mol == NULL) { | 
| 449 | > | sprintf(painCave.errMsg, "Molecule not found on node %d!", worldRank); | 
| 450 | > | painCave.isFatal = 1; | 
| 451 | > | simError(); | 
| 452 | > | } | 
| 453 |  |  | 
| 454 | < | sd->getQ( q ); | 
| 455 | < | sd->getJ( ji ); | 
| 454 | > | for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL; | 
| 455 | > | integrableObject = mol->nextIntegrableObject(ii)) { | 
| 456 |  |  | 
| 457 | < | for (int j = 0; j < 6 ; j++) | 
| 458 | < | atomData[j] = atomData[j]; | 
| 488 | < |  | 
| 489 | < | atomData[6] = q[0]; | 
| 490 | < | atomData[7] = q[1]; | 
| 491 | < | atomData[8] = q[2]; | 
| 492 | < | atomData[9] = q[3]; | 
| 493 | < |  | 
| 494 | < | atomData[10] = ji[0]; | 
| 495 | < | atomData[11] = ji[1]; | 
| 496 | < | atomData[12] = ji[2]; | 
| 497 | < | } | 
| 498 | < |  | 
| 499 | < | // If we've survived to here, format the line: | 
| 500 | < |  | 
| 501 | < | if (!isDirectional) { | 
| 502 | < |  | 
| 503 | < | sprintf( writeLine, | 
| 504 | < | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", | 
| 505 | < | atomTypeString, | 
| 506 | < | atomData[0], | 
| 507 | < | atomData[1], | 
| 508 | < | atomData[2], | 
| 509 | < | atomData[3], | 
| 510 | < | atomData[4], | 
| 511 | < | atomData[5]); | 
| 512 | < |  | 
| 513 | < | strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" ); | 
| 514 | < |  | 
| 515 | < | } | 
| 516 | < | else { | 
| 517 | < |  | 
| 518 | < | sprintf( writeLine, | 
| 519 | < | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", | 
| 520 | < | atomTypeString, | 
| 521 | < | atomData[0], | 
| 522 | < | atomData[1], | 
| 523 | < | atomData[2], | 
| 524 | < | atomData[3], | 
| 525 | < | atomData[4], | 
| 526 | < | atomData[5], | 
| 527 | < | atomData[6], | 
| 528 | < | atomData[7], | 
| 529 | < | atomData[8], | 
| 530 | < | atomData[9], | 
| 531 | < | atomData[10], | 
| 532 | < | atomData[11], | 
| 533 | < | atomData[12]); | 
| 534 | < |  | 
| 535 | < | } | 
| 536 | < |  | 
| 537 | < | for(k = 0; k < outFile.size(); k++) | 
| 538 | < | *outFile[k] << writeLine; | 
| 539 | < |  | 
| 540 | < |  | 
| 541 | < | }//end for(iter = integrableObject.begin()) | 
| 542 | < |  | 
| 543 | < | currentIndex++; | 
| 544 | < | } | 
| 457 | > | pos = integrableObject->getPos(); | 
| 458 | > | vel = integrableObject->getVel(); | 
| 459 |  |  | 
| 460 | < | }//end for(i = 0; i < mpiSim->getNmol()) | 
| 461 | < |  | 
| 462 | < | for(k = 0; k < outFile.size(); k++) | 
| 549 | < | outFile[k]->flush(); | 
| 550 | < |  | 
| 551 | < | sprintf( checkPointMsg, | 
| 552 | < | "Sucessfully took a dump.\n"); | 
| 553 | < |  | 
| 554 | < | MPIcheckPoint(); | 
| 555 | < |  | 
| 556 | < | delete[] potatoes; | 
| 557 | < |  | 
| 558 | < | } else { | 
| 460 | > | atomData[0] = pos[0]; | 
| 461 | > | atomData[1] = pos[1]; | 
| 462 | > | atomData[2] = pos[2]; | 
| 463 |  |  | 
| 464 | < | // worldRank != 0, so I'm a remote node. | 
| 464 | > | atomData[3] = vel[0]; | 
| 465 | > | atomData[4] = vel[1]; | 
| 466 | > | atomData[5] = vel[2]; | 
| 467 |  |  | 
| 468 | < | // Set my magic potato to 0: | 
| 468 | > | isDirectional = 0; | 
| 469 |  |  | 
| 470 | < | myPotato = 0; | 
| 471 | < | currentIndex = 0; | 
| 566 | < |  | 
| 567 | < | for (i = 0 ; i < mpiSim->getNMolGlobal(); i++ ) { | 
| 568 | < |  | 
| 569 | < | // Am I the node which has this integrableObject? | 
| 570 | < |  | 
| 571 | < | if (MolToProcMap[i] == worldRank) { | 
| 470 | > | if (integrableObject->isDirectional()) { | 
| 471 | > | isDirectional = 1; | 
| 472 |  |  | 
| 473 | + | q = integrableObject->getQ(); | 
| 474 | + | ji = integrableObject->getJ(); | 
| 475 |  |  | 
| 476 | < | if (myPotato + 1 >= MAXTAG) { | 
| 477 | < |  | 
| 478 | < | // The potato was going to exceed the maximum value, | 
| 577 | < | // so wrap this processor potato back to 0 (and block until | 
| 578 | < | // node 0 says we can go: | 
| 579 | < |  | 
| 580 | < | MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus); | 
| 581 | < |  | 
| 582 | < | } | 
| 476 | > | for(int j = 0; j < 6; j++) { | 
| 477 | > | atomData[j] = atomData[j]; | 
| 478 | > | } | 
| 479 |  |  | 
| 480 | < | local_index = indexArray[currentIndex].first; | 
| 481 | < | integrableObjects = entry_plug->molecules[local_index].getIntegrableObjects(); | 
| 482 | < |  | 
| 483 | < | nCurObj = integrableObjects.size(); | 
| 588 | < |  | 
| 589 | < | MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 590 | < | myPotato, MPI_COMM_WORLD); | 
| 591 | < | myPotato++; | 
| 480 | > | atomData[6] = q[0]; | 
| 481 | > | atomData[7] = q[1]; | 
| 482 | > | atomData[8] = q[2]; | 
| 483 | > | atomData[9] = q[3]; | 
| 484 |  |  | 
| 485 | < | for( iter = integrableObjects.begin(); iter  != integrableObjects.end(); iter++){ | 
| 485 | > | atomData[10] = ji[0]; | 
| 486 | > | atomData[11] = ji[1]; | 
| 487 | > | atomData[12] = ji[2]; | 
| 488 | > | } | 
| 489 |  |  | 
| 490 | < | if (myPotato + 2 >= MAXTAG) { | 
| 491 | < |  | 
| 492 | < | // The potato was going to exceed the maximum value, | 
| 598 | < | // so wrap this processor potato back to 0 (and block until | 
| 599 | < | // node 0 says we can go: | 
| 600 | < |  | 
| 601 | < | MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus); | 
| 602 | < |  | 
| 603 | < | } | 
| 604 | < |  | 
| 605 | < | sd = *iter; | 
| 606 | < |  | 
| 607 | < | atomTypeString = sd->getType(); | 
| 490 | > | if (needForceVector_) { | 
| 491 | > | frc = integrableObject->getFrc(); | 
| 492 | > | trq = integrableObject->getTrq(); | 
| 493 |  |  | 
| 494 | < | sd->getPos(pos); | 
| 495 | < | sd->getVel(vel); | 
| 494 | > | if (!isDirectional) { | 
| 495 | > | atomData[6] = frc[0]; | 
| 496 | > | atomData[7] = frc[1]; | 
| 497 | > | atomData[8] = frc[2]; | 
| 498 | > | atomData[9] = trq[0]; | 
| 499 | > | atomData[10] = trq[1]; | 
| 500 | > | atomData[11] = trq[2]; | 
| 501 | > | } else { | 
| 502 | > | atomData[13] = frc[0]; | 
| 503 | > | atomData[14] = frc[1]; | 
| 504 | > | atomData[15] = frc[2]; | 
| 505 | > | atomData[16] = trq[0]; | 
| 506 | > | atomData[17] = trq[1]; | 
| 507 | > | atomData[18] = trq[2]; | 
| 508 | > | } | 
| 509 | > | } | 
| 510 |  |  | 
| 511 | < | atomData[0] = pos[0]; | 
| 613 | < | atomData[1] = pos[1]; | 
| 614 | < | atomData[2] = pos[2]; | 
| 511 | > | // If we've survived to here, format the line: | 
| 512 |  |  | 
| 513 | < | atomData[3] = vel[0]; | 
| 514 | < | atomData[4] = vel[1]; | 
| 515 | < | atomData[5] = vel[2]; | 
| 516 | < |  | 
| 517 | < | isDirectional = 0; | 
| 513 | > | if (!isDirectional) { | 
| 514 | > | sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t", | 
| 515 | > | integrableObject->getType().c_str(), atomData[0], | 
| 516 | > | atomData[1], atomData[2], | 
| 517 | > | atomData[3], atomData[4], | 
| 518 | > | atomData[5]); | 
| 519 |  |  | 
| 520 | < | if( sd->isDirectional() ){ | 
| 521 | < |  | 
| 522 | < | isDirectional = 1; | 
| 523 | < |  | 
| 524 | < | sd->getQ( q ); | 
| 525 | < | sd->getJ( ji ); | 
| 526 | < |  | 
| 527 | < |  | 
| 528 | < | atomData[6] = q[0]; | 
| 529 | < | atomData[7] = q[1]; | 
| 530 | < | atomData[8] = q[2]; | 
| 531 | < | atomData[9] = q[3]; | 
| 532 | < |  | 
| 533 | < | atomData[10] = ji[0]; | 
| 534 | < | atomData[11] = ji[1]; | 
| 535 | < | atomData[12] = ji[2]; | 
| 536 | < | } | 
| 537 | < |  | 
| 538 | < |  | 
| 539 | < | strncpy(MPIatomTypeString, atomTypeString, MINIBUFFERSIZE); | 
| 520 | > | strcat(writeLine, | 
| 521 | > | "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0"); | 
| 522 | > | } else { | 
| 523 | > | sprintf(writeLine, | 
| 524 | > | "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf", | 
| 525 | > | integrableObject->getType().c_str(), | 
| 526 | > | atomData[0], | 
| 527 | > | atomData[1], | 
| 528 | > | atomData[2], | 
| 529 | > | atomData[3], | 
| 530 | > | atomData[4], | 
| 531 | > | atomData[5], | 
| 532 | > | atomData[6], | 
| 533 | > | atomData[7], | 
| 534 | > | atomData[8], | 
| 535 | > | atomData[9], | 
| 536 | > | atomData[10], | 
| 537 | > | atomData[11], | 
| 538 | > | atomData[12]); | 
| 539 | > | } | 
| 540 |  |  | 
| 541 | < | // null terminate the string before sending (just in case): | 
| 542 | < | MPIatomTypeString[MINIBUFFERSIZE-1] = '\0'; | 
| 541 | > | if (needForceVector_) { | 
| 542 | > | if (!isDirectional) { | 
| 543 | > | sprintf(writeLine, "\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf", | 
| 544 | > | atomData[6], | 
| 545 | > | atomData[7], | 
| 546 | > | atomData[8], | 
| 547 | > | atomData[9], | 
| 548 | > | atomData[10], | 
| 549 | > | atomData[11]); | 
| 550 | > | } else { | 
| 551 | > | sprintf(writeLine, "\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf", | 
| 552 | > | atomData[13], | 
| 553 | > | atomData[14], | 
| 554 | > | atomData[15], | 
| 555 | > | atomData[16], | 
| 556 | > | atomData[17], | 
| 557 | > | atomData[18]); | 
| 558 | > | } | 
| 559 | > | } | 
| 560 |  |  | 
| 561 | < | MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0, | 
| 562 | < | myPotato, MPI_COMM_WORLD); | 
| 648 | < |  | 
| 649 | < | myPotato++; | 
| 650 | < |  | 
| 651 | < | if (isDirectional) { | 
| 561 | > | sprintf(writeLine, "\n"); | 
| 562 | > | os << writeLine; | 
| 563 |  |  | 
| 564 | < | MPI_Send(atomData, 13, MPI_DOUBLE, 0, | 
| 565 | < | myPotato, MPI_COMM_WORLD); | 
| 566 | < |  | 
| 656 | < | } else { | 
| 564 | > | } //end for(iter = integrableObject.begin()) | 
| 565 | > | } | 
| 566 | > | } //end for(i = 0; i < mpiSim->getNmol()) | 
| 567 |  |  | 
| 568 | < | MPI_Send(atomData, 6, MPI_DOUBLE, 0, | 
| 569 | < | myPotato, MPI_COMM_WORLD); | 
| 570 | < | } | 
| 568 | > | os.flush(); | 
| 569 | > |  | 
| 570 | > | sprintf(checkPointMsg, "Sucessfully took a dump.\n"); | 
| 571 | > | MPIcheckPoint(); | 
| 572 |  |  | 
| 573 | < | myPotato++; | 
| 573 | > | delete [] potatoes; | 
| 574 | > | } else { | 
| 575 |  |  | 
| 576 | < | } | 
| 576 | > | // worldRank != 0, so I'm a remote node. | 
| 577 |  |  | 
| 578 | < | currentIndex++; | 
| 579 | < |  | 
| 580 | < | } | 
| 581 | < |  | 
| 578 | > | // Set my magic potato to 0: | 
| 579 | > |  | 
| 580 | > | myPotato = 0; | 
| 581 | > |  | 
| 582 | > | for(int i = 0; i < info_->getNGlobalMolecules(); i++) { | 
| 583 | > |  | 
| 584 | > | // Am I the node which has this integrableObject? | 
| 585 | > | int whichNode = info_->getMolToProc(i); | 
| 586 | > | if (whichNode == worldRank) { | 
| 587 | > | if (myPotato + 1 >= MAXTAG) { | 
| 588 | > |  | 
| 589 | > | // The potato was going to exceed the maximum value, | 
| 590 | > | // so wrap this processor potato back to 0 (and block until | 
| 591 | > | // node 0 says we can go: | 
| 592 | > |  | 
| 593 | > | MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, | 
| 594 | > | &istatus); | 
| 595 | > | } | 
| 596 | > |  | 
| 597 | > | mol = info_->getMoleculeByGlobalIndex(i); | 
| 598 | > |  | 
| 599 | > |  | 
| 600 | > | nCurObj = mol->getNIntegrableObjects(); | 
| 601 | > |  | 
| 602 | > | MPI_Send(&nCurObj, 1, MPI_INT, 0, myPotato, MPI_COMM_WORLD); | 
| 603 | > | myPotato++; | 
| 604 | > |  | 
| 605 | > | for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL; | 
| 606 | > | integrableObject = mol->nextIntegrableObject(ii)) { | 
| 607 | > |  | 
| 608 | > | if (myPotato + 2 >= MAXTAG) { | 
| 609 | > |  | 
| 610 | > | // The potato was going to exceed the maximum value, | 
| 611 | > | // so wrap this processor potato back to 0 (and block until | 
| 612 | > | // node 0 says we can go: | 
| 613 | > |  | 
| 614 | > | MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, | 
| 615 | > | &istatus); | 
| 616 | > | } | 
| 617 | > |  | 
| 618 | > | pos = integrableObject->getPos(); | 
| 619 | > | vel = integrableObject->getVel(); | 
| 620 | > |  | 
| 621 | > | atomData[0] = pos[0]; | 
| 622 | > | atomData[1] = pos[1]; | 
| 623 | > | atomData[2] = pos[2]; | 
| 624 | > |  | 
| 625 | > | atomData[3] = vel[0]; | 
| 626 | > | atomData[4] = vel[1]; | 
| 627 | > | atomData[5] = vel[2]; | 
| 628 | > |  | 
| 629 | > | isDirectional = 0; | 
| 630 | > |  | 
| 631 | > | if (integrableObject->isDirectional()) { | 
| 632 | > | isDirectional = 1; | 
| 633 | > |  | 
| 634 | > | q = integrableObject->getQ(); | 
| 635 | > | ji = integrableObject->getJ(); | 
| 636 | > |  | 
| 637 | > | atomData[6] = q[0]; | 
| 638 | > | atomData[7] = q[1]; | 
| 639 | > | atomData[8] = q[2]; | 
| 640 | > | atomData[9] = q[3]; | 
| 641 | > |  | 
| 642 | > | atomData[10] = ji[0]; | 
| 643 | > | atomData[11] = ji[1]; | 
| 644 | > | atomData[12] = ji[2]; | 
| 645 | > | } | 
| 646 | > |  | 
| 647 | > | if (needForceVector_) { | 
| 648 | > | frc = integrableObject->getFrc(); | 
| 649 | > | trq = integrableObject->getTrq(); | 
| 650 | > |  | 
| 651 | > | if (!isDirectional) { | 
| 652 | > | atomData[6] = frc[0]; | 
| 653 | > | atomData[7] = frc[1]; | 
| 654 | > | atomData[8] = frc[2]; | 
| 655 | > |  | 
| 656 | > | atomData[9] = trq[0]; | 
| 657 | > | atomData[10] = trq[1]; | 
| 658 | > | atomData[11] = trq[2]; | 
| 659 | > | } else { | 
| 660 | > | atomData[13] = frc[0]; | 
| 661 | > | atomData[14] = frc[1]; | 
| 662 | > | atomData[15] = frc[2]; | 
| 663 | > |  | 
| 664 | > | atomData[16] = trq[0]; | 
| 665 | > | atomData[17] = trq[1]; | 
| 666 | > | atomData[18] = trq[2]; | 
| 667 | > | } | 
| 668 | > | } | 
| 669 | > |  | 
| 670 | > | strncpy(MPIatomTypeString, integrableObject->getType().c_str(), MINIBUFFERSIZE); | 
| 671 | > |  | 
| 672 | > | // null terminate the  std::string before sending (just in case): | 
| 673 | > | MPIatomTypeString[MINIBUFFERSIZE - 1] = '\0'; | 
| 674 | > |  | 
| 675 | > | MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0, | 
| 676 | > | myPotato, MPI_COMM_WORLD); | 
| 677 | > |  | 
| 678 | > | myPotato++; | 
| 679 | > |  | 
| 680 | > | if (isDirectional && needForceVector_) { | 
| 681 | > | MPI_Send(atomData, 19, MPI_DOUBLE, 0, myPotato, | 
| 682 | > | MPI_COMM_WORLD); | 
| 683 | > | } else if (isDirectional) { | 
| 684 | > | MPI_Send(atomData, 13, MPI_DOUBLE, 0, myPotato, | 
| 685 | > | MPI_COMM_WORLD); | 
| 686 | > | } else if (needForceVector_) { | 
| 687 | > | MPI_Send(atomData, 12, MPI_DOUBLE, 0, myPotato, | 
| 688 | > | MPI_COMM_WORLD); | 
| 689 | > | } else { | 
| 690 | > | MPI_Send(atomData, 6, MPI_DOUBLE, 0, myPotato, | 
| 691 | > | MPI_COMM_WORLD); | 
| 692 | > | } | 
| 693 | > |  | 
| 694 | > | myPotato++; | 
| 695 | > | } | 
| 696 | > |  | 
| 697 | > | } | 
| 698 | > |  | 
| 699 |  | } | 
| 700 | + | sprintf(checkPointMsg, "Sucessfully took a dump.\n"); | 
| 701 | + | MPIcheckPoint(); | 
| 702 | + | } | 
| 703 |  |  | 
| 704 | < | sprintf( checkPointMsg, | 
| 705 | < | "Sucessfully took a dump.\n"); | 
| 706 | < | MPIcheckPoint(); | 
| 704 | > | #endif // is_mpi | 
| 705 | > |  | 
| 706 | > | } | 
| 707 | > |  | 
| 708 | > | void DumpWriter::writeDump() { | 
| 709 | > | writeFrame(*dumpFile_); | 
| 710 | > | } | 
| 711 | > |  | 
| 712 | > | void DumpWriter::writeEor() { | 
| 713 | > | std::ostream* eorStream; | 
| 714 |  |  | 
| 715 | + | #ifdef IS_MPI | 
| 716 | + | if (worldRank == 0) { | 
| 717 | + | #endif // is_mpi | 
| 718 | + |  | 
| 719 | + | eorStream = createOStream(eorFilename_); | 
| 720 | + |  | 
| 721 | + | #ifdef IS_MPI | 
| 722 |  | } | 
| 723 | + | #endif // is_mpi | 
| 724 |  |  | 
| 725 | + | writeFrame(*eorStream); | 
| 726 |  |  | 
| 727 | < |  | 
| 727 | > | #ifdef IS_MPI | 
| 728 | > | if (worldRank == 0) { | 
| 729 |  | #endif // is_mpi | 
| 730 | < | } | 
| 730 | > | delete eorStream; | 
| 731 |  |  | 
| 732 |  | #ifdef IS_MPI | 
| 733 | + | } | 
| 734 | + | #endif // is_mpi | 
| 735 |  |  | 
| 736 | < | // a couple of functions to let us escape the write loop | 
| 736 | > | } | 
| 737 |  |  | 
| 687 | – | void dWrite::DieDieDie( void ){ | 
| 738 |  |  | 
| 739 | < | MPI_Finalize(); | 
| 740 | < | exit (0); | 
| 739 | > | void DumpWriter::writeDumpAndEor() { | 
| 740 | > | std::vector<std::streambuf*> buffers; | 
| 741 | > | std::ostream* eorStream; | 
| 742 | > | #ifdef IS_MPI | 
| 743 | > | if (worldRank == 0) { | 
| 744 | > | #endif // is_mpi | 
| 745 | > |  | 
| 746 | > | buffers.push_back(dumpFile_->rdbuf()); | 
| 747 | > |  | 
| 748 | > | eorStream = createOStream(eorFilename_); | 
| 749 | > |  | 
| 750 | > | buffers.push_back(eorStream->rdbuf()); | 
| 751 | > |  | 
| 752 | > | #ifdef IS_MPI | 
| 753 | > | } | 
| 754 | > | #endif // is_mpi | 
| 755 | > |  | 
| 756 | > | TeeBuf tbuf(buffers.begin(), buffers.end()); | 
| 757 | > | std::ostream os(&tbuf); | 
| 758 | > |  | 
| 759 | > | writeFrame(os); | 
| 760 | > |  | 
| 761 | > | #ifdef IS_MPI | 
| 762 | > | if (worldRank == 0) { | 
| 763 | > | #endif // is_mpi | 
| 764 | > | delete eorStream; | 
| 765 | > |  | 
| 766 | > | #ifdef IS_MPI | 
| 767 | > | } | 
| 768 | > | #endif // is_mpi | 
| 769 | > |  | 
| 770 | > | } | 
| 771 | > |  | 
| 772 | > | std::ostream* DumpWriter::createOStream(const std::string& filename) { | 
| 773 | > |  | 
| 774 | > | std::ostream* newOStream; | 
| 775 | > | #ifdef HAVE_LIBZ | 
| 776 | > | if (needCompression_) { | 
| 777 | > | newOStream = new ogzstream(filename.c_str()); | 
| 778 | > | } else { | 
| 779 | > | newOStream = new std::ofstream(filename.c_str()); | 
| 780 | > | } | 
| 781 | > | #else | 
| 782 | > | newOStream = new std::ofstream(filename.c_str()); | 
| 783 | > | #endif | 
| 784 | > | return newOStream; | 
| 785 |  | } | 
| 786 |  |  | 
| 787 | < | #endif //is_mpi | 
| 787 | > | }//end namespace oopse |