| 60 | 
  | 
 | 
| 61 | 
  | 
#ifdef IS_MPI | 
| 62 | 
  | 
#include <mpi.h> | 
| 63 | 
– | 
#include "brains/mpiSimulation.hpp" | 
| 63 | 
  | 
#define TAKE_THIS_TAG_CHAR 0 | 
| 64 | 
  | 
#define TAKE_THIS_TAG_INT 1 | 
| 65 | 
  | 
#define TAKE_THIS_TAG_DOUBLE 2 | 
| 68 | 
  | 
namespace oopse { | 
| 69 | 
  | 
   | 
| 70 | 
  | 
  RestReader::RestReader( SimInfo* info ) : info_(info){ | 
| 71 | 
< | 
         | 
| 71 | 
> | 
     | 
| 72 | 
  | 
    idealName = "idealCrystal.in"; | 
| 73 | 
  | 
     | 
| 74 | 
  | 
    isScanned = false; | 
| 92 | 
  | 
            "File \"idealCrystal.in\" opened successfully for reading." ); | 
| 93 | 
  | 
    MPIcheckPoint(); | 
| 94 | 
  | 
#endif | 
| 95 | 
+ | 
 | 
| 96 | 
  | 
    return;   | 
| 97 | 
  | 
  } | 
| 98 | 
  | 
   | 
| 122 | 
  | 
   | 
| 123 | 
  | 
   | 
| 124 | 
  | 
  void RestReader :: readIdealCrystal(){ | 
| 125 | 
< | 
     | 
| 125 | 
> | 
         | 
| 126 | 
  | 
    int i; | 
| 127 | 
  | 
    unsigned int j; | 
| 128 | 
< | 
     | 
| 128 | 
> | 
 | 
| 129 | 
  | 
#ifdef IS_MPI | 
| 130 | 
  | 
    int done, which_node, which_atom; // loop counter | 
| 131 | 
  | 
#endif //is_mpi | 
| 220 | 
  | 
    int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone | 
| 221 | 
  | 
     | 
| 222 | 
  | 
    MPI_Status istatus; | 
| 223 | 
– | 
    int localIndex; | 
| 223 | 
  | 
    int nCurObj; | 
| 224 | 
  | 
    int nitems; | 
| 225 | 
< | 
     | 
| 226 | 
< | 
    nTotObjs = info_->getTotIntegrableObjects(); | 
| 225 | 
> | 
    int haveError; | 
| 226 | 
> | 
 | 
| 227 | 
> | 
    nTotObjs = info_->getNGlobalIntegrableObjects(); | 
| 228 | 
  | 
    haveError = 0; | 
| 229 | 
< | 
     | 
| 229 | 
> | 
 | 
| 230 | 
  | 
    if (worldRank == masterNode) { | 
| 231 | 
  | 
      eof_test = fgets(read_buffer, sizeof(read_buffer), inIdealFile); | 
| 232 | 
  | 
      if( eof_test == NULL ){ | 
| 259 | 
  | 
        painCave.isFatal = 1; | 
| 260 | 
  | 
        simError(); | 
| 261 | 
  | 
      } | 
| 262 | 
< | 
       | 
| 262 | 
> | 
 | 
| 263 | 
> | 
      MPI_Bcast(read_buffer, BUFFERSIZE, MPI_CHAR, masterNode, MPI_COMM_WORLD); | 
| 264 | 
> | 
 | 
| 265 | 
  | 
      for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 266 | 
  | 
        int which_node = info_->getMolToProc(i); | 
| 267 | 
  | 
         | 
| 268 | 
  | 
        if(which_node == masterNode){ | 
| 269 | 
  | 
          //molecules belong to master node | 
| 270 | 
  | 
           | 
| 271 | 
< | 
          localIndex = info_->getMoleculeByGlobalIndex(i); | 
| 271 | 
> | 
          mol = info_->getMoleculeByGlobalIndex(i); | 
| 272 | 
  | 
           | 
| 273 | 
< | 
          if(localIndex == NULL) { | 
| 274 | 
< | 
            strcpy(painCave.errMsg,  | 
| 275 | 
< | 
                   "RestReader Error: Molecule not found on node %d!", | 
| 276 | 
< | 
                   worldRank); | 
| 273 | 
> | 
          if(mol == NULL) { | 
| 274 | 
> | 
            sprintf(painCave.errMsg,  | 
| 275 | 
> | 
                    "RestReader Error: Molecule not found on node %d!\n", | 
| 276 | 
> | 
                    worldRank); | 
| 277 | 
  | 
            painCave.isFatal = 1; | 
| 278 | 
  | 
            simError(); | 
| 279 | 
  | 
          } | 
| 293 | 
  | 
              painCave.isFatal = 1; | 
| 294 | 
  | 
              simError(); | 
| 295 | 
  | 
            } | 
| 296 | 
< | 
             | 
| 297 | 
< | 
            parseIdealLine(read_buffer, integrableObjects[j]); | 
| 296 | 
> | 
         | 
| 297 | 
> | 
            parseIdealLine(read_buffer, integrableObject); | 
| 298 | 
> | 
         | 
| 299 | 
  | 
          } | 
| 300 | 
+ | 
 | 
| 301 | 
  | 
        } else { | 
| 302 | 
  | 
          //molecule belongs to slave nodes | 
| 303 | 
  | 
           | 
| 304 | 
  | 
          MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 305 | 
  | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 306 | 
  | 
           | 
| 307 | 
< | 
          for(j=0; j < nCurObj; j++){ | 
| 307 | 
> | 
          for(j = 0; j < nCurObj; j++){ | 
| 308 | 
  | 
             | 
| 309 | 
  | 
            eof_test = fgets(read_buffer, sizeof(read_buffer), inIdealFile); | 
| 310 | 
  | 
            if(eof_test == NULL){ | 
| 317 | 
  | 
              simError(); | 
| 318 | 
  | 
            } | 
| 319 | 
  | 
             | 
| 320 | 
< | 
            if(haveError) nodeZeroError(); | 
| 317 | 
< | 
             | 
| 318 | 
< | 
            MPI_Send(read_buffer, BUFFERSIZE, MPI_CHAR, which_node, | 
| 320 | 
> | 
            MPI_Send(read_buffer, BUFFERSIZE, MPI_CHAR, which_node, | 
| 321 | 
  | 
                     TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD); | 
| 322 | 
  | 
          } | 
| 323 | 
  | 
        } | 
| 324 | 
  | 
      } | 
| 325 | 
  | 
    } else { | 
| 326 | 
< | 
      //actions taken at slave nodes | 
| 326 | 
> | 
      //actions taken at slave nodes  | 
| 327 | 
> | 
      MPI_Bcast(read_buffer, BUFFERSIZE, MPI_CHAR, masterNode, MPI_COMM_WORLD); | 
| 328 | 
> | 
 | 
| 329 | 
  | 
      for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 330 | 
< | 
        int which_node = info_->getMolToProc(i); | 
| 331 | 
< | 
         | 
| 330 | 
> | 
        int which_node = info_->getMolToProc(i);  | 
| 331 | 
> | 
 | 
| 332 | 
  | 
        if(which_node == worldRank){ | 
| 333 | 
  | 
          //molecule with global index i belongs to this processor | 
| 334 | 
  | 
           | 
| 335 | 
< | 
          localIndex = info_->getMoleculeByGlobalIndex(i); | 
| 335 | 
> | 
          mol = info_->getMoleculeByGlobalIndex(i); | 
| 336 | 
  | 
           | 
| 337 | 
< | 
          if(localIndex == NULL) { | 
| 337 | 
> | 
          if(mol == NULL) { | 
| 338 | 
  | 
            sprintf(painCave.errMsg,  | 
| 339 | 
  | 
                    "RestReader Error: molecule not found on node %d\n",  | 
| 340 | 
  | 
                    worldRank); | 
| 342 | 
  | 
            simError(); | 
| 343 | 
  | 
          } | 
| 344 | 
  | 
           | 
| 345 | 
< | 
          nCurObj = localIndex->getNIntegrableObjects(); | 
| 345 | 
> | 
          nCurObj = mol->getNIntegrableObjects(); | 
| 346 | 
  | 
           | 
| 347 | 
  | 
          MPI_Send(&nCurObj, 1, MPI_INT, masterNode, | 
| 348 | 
  | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 369 | 
  | 
   | 
| 370 | 
  | 
  char* RestReader::parseIdealLine(char* readLine, StuntDouble* sd){ | 
| 371 | 
  | 
     | 
| 372 | 
< | 
    char *foo; // the pointer to the current string token | 
| 373 | 
< | 
     | 
| 374 | 
< | 
    double pos[3];        // position place holders | 
| 375 | 
< | 
    double q[4];          // the quaternions | 
| 376 | 
< | 
    double RfromQ[3][3];  // the rotation matrix  | 
| 377 | 
< | 
    double normalize;     // to normalize the reference unit vector | 
| 374 | 
< | 
    double uX, uY, uZ;    // reference unit vector place holders | 
| 375 | 
< | 
    double uselessToken; | 
| 372 | 
> | 
    RealType pos[3];        // position place holders | 
| 373 | 
> | 
    RealType q[4];          // the quaternions | 
| 374 | 
> | 
    RealType RfromQ[3][3];  // the rotation matrix  | 
| 375 | 
> | 
    RealType normalize;     // to normalize the reference unit vector | 
| 376 | 
> | 
    RealType uX, uY, uZ;    // reference unit vector place holders | 
| 377 | 
> | 
    RealType uselessToken; | 
| 378 | 
  | 
    StringTokenizer tokenizer(readLine); | 
| 379 | 
  | 
    int nTokens; | 
| 380 | 
  | 
     | 
| 381 | 
  | 
    nTokens = tokenizer.countTokens(); | 
| 382 | 
< | 
     | 
| 382 | 
> | 
 | 
| 383 | 
  | 
    if (nTokens < 14) { | 
| 384 | 
  | 
      sprintf(painCave.errMsg, | 
| 385 | 
  | 
              "RestReader Error: Not enough Tokens.\n"); | 
| 388 | 
  | 
    } | 
| 389 | 
  | 
     | 
| 390 | 
  | 
    std::string name = tokenizer.nextToken(); | 
| 391 | 
< | 
     | 
| 391 | 
> | 
 | 
| 392 | 
  | 
    if (name != sd->getType()) { | 
| 393 | 
  | 
       | 
| 394 | 
  | 
      sprintf(painCave.errMsg, | 
| 403 | 
  | 
    pos[0] = tokenizer.nextTokenAsDouble(); | 
| 404 | 
  | 
    pos[1] = tokenizer.nextTokenAsDouble(); | 
| 405 | 
  | 
    pos[2] = tokenizer.nextTokenAsDouble(); | 
| 406 | 
< | 
     | 
| 406 | 
> | 
 | 
| 407 | 
  | 
    // store the positions in the stuntdouble as generic data doubles | 
| 408 | 
  | 
    DoubleGenericData* refPosX = new DoubleGenericData(); | 
| 409 | 
  | 
    refPosX->setID("refPosX"); | 
| 410 | 
  | 
    refPosX->setData(pos[0]); | 
| 411 | 
  | 
    sd->addProperty(refPosX); | 
| 412 | 
< | 
     | 
| 412 | 
> | 
 | 
| 413 | 
  | 
    DoubleGenericData* refPosY = new DoubleGenericData(); | 
| 414 | 
  | 
    refPosY->setID("refPosY"); | 
| 415 | 
  | 
    refPosY->setData(pos[1]); | 
| 419 | 
  | 
    refPosZ->setID("refPosZ"); | 
| 420 | 
  | 
    refPosZ->setData(pos[2]); | 
| 421 | 
  | 
    sd->addProperty(refPosZ); | 
| 422 | 
< | 
     | 
| 422 | 
> | 
 | 
| 423 | 
  | 
    // we don't need the velocities | 
| 424 | 
  | 
    uselessToken = tokenizer.nextTokenAsDouble(); | 
| 425 | 
  | 
    uselessToken = tokenizer.nextTokenAsDouble(); | 
| 495 | 
  | 
    char *parseErr; | 
| 496 | 
  | 
     | 
| 497 | 
  | 
    std::vector<StuntDouble*> vecParticles; | 
| 498 | 
< | 
    std::vector<double> tempZangs; | 
| 498 | 
> | 
    std::vector<RealType> tempZangs; | 
| 499 | 
  | 
       | 
| 500 | 
  | 
    inAngFileName = info_->getRestFileName(); | 
| 501 | 
  | 
     | 
| 596 | 
  | 
     | 
| 597 | 
  | 
    // first thing first, suspend fatalities. | 
| 598 | 
  | 
    painCave.isEventLoop = 1; | 
| 599 | 
< | 
     | 
| 599 | 
> | 
 | 
| 600 | 
> | 
    int masterNode = 0; | 
| 601 | 
  | 
    int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone | 
| 602 | 
< | 
    int haveError, index; | 
| 603 | 
< | 
     | 
| 604 | 
< | 
    int *MolToProcMap = mpiSim->getMolToProcMap(); | 
| 605 | 
< | 
    int localIndex; | 
| 602 | 
> | 
    int haveError; | 
| 603 | 
> | 
    int intObjIndex;     | 
| 604 | 
> | 
    int intObjIndexTransfer;     | 
| 605 | 
> | 
 | 
| 606 | 
  | 
    int nCurObj; | 
| 607 | 
< | 
    double angleTranfer; | 
| 607 | 
> | 
    RealType angleTranfer; | 
| 608 | 
  | 
     | 
| 609 | 
< | 
    nTotObjs = info_->getTotIntegrableObjects(); | 
| 609 | 
> | 
    nTotObjs = info_->getNGlobalIntegrableObjects(); | 
| 610 | 
  | 
    haveError = 0; | 
| 611 | 
< | 
    if (worldRank == 0) { | 
| 611 | 
> | 
 | 
| 612 | 
> | 
    if (worldRank == masterNode) { | 
| 613 | 
  | 
       | 
| 614 | 
  | 
      eof_test = fgets(read_buffer, sizeof(read_buffer), inAngFile); | 
| 615 | 
  | 
      if( eof_test == NULL ){ | 
| 627 | 
  | 
          tempZangs.push_back( atof(read_buffer) ); | 
| 628 | 
  | 
        eof_test = fgets(read_buffer, sizeof(read_buffer), inAngFile); | 
| 629 | 
  | 
      } | 
| 630 | 
< | 
       | 
| 630 | 
> | 
 | 
| 631 | 
  | 
      // Check to see that the number of integrable objects in the | 
| 632 | 
  | 
      // intial configuration file is the same as derived from the | 
| 633 | 
  | 
      // meta-data file. | 
| 640 | 
  | 
        simError(); | 
| 641 | 
  | 
      } | 
| 642 | 
  | 
       | 
| 643 | 
< | 
    } | 
| 644 | 
< | 
    // At this point, node 0 has a tempZangs vector completed, and  | 
| 641 | 
< | 
    // everyone else has nada | 
| 642 | 
< | 
    index = 0; | 
| 643 | 
< | 
     | 
| 644 | 
< | 
    for (i=0 ; i < mpiSim->getNMolGlobal(); i++) { | 
| 645 | 
< | 
      // Get the Node number which has this atom | 
| 646 | 
< | 
      which_node = MolToProcMap[i]; | 
| 643 | 
> | 
      // At this point, node 0 has a tempZangs vector completed, and  | 
| 644 | 
> | 
      // everyone else has nada | 
| 645 | 
  | 
       | 
| 646 | 
< | 
      if (worldRank == 0) { | 
| 647 | 
< | 
        if (which_node == 0) { | 
| 648 | 
< | 
          localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 649 | 
< | 
           | 
| 650 | 
< | 
          if(localIndex == -1) { | 
| 651 | 
< | 
            strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 652 | 
< | 
            haveError = 1; | 
| 646 | 
> | 
      for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 647 | 
> | 
        // Get the Node number which has this atom | 
| 648 | 
> | 
        which_node = info_->getMolToProc(i); | 
| 649 | 
> | 
         | 
| 650 | 
> | 
        if (which_node == masterNode) { | 
| 651 | 
> | 
          mol = info_->getMoleculeByGlobalIndex(i); | 
| 652 | 
> | 
 | 
| 653 | 
> | 
          if(mol == NULL) { | 
| 654 | 
> | 
            strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 655 | 
> | 
            haveError = 1; | 
| 656 | 
> | 
            simError(); | 
| 657 | 
> | 
          } | 
| 658 | 
> | 
 | 
| 659 | 
> | 
          for (integrableObject = mol->beginIntegrableObject(ii);  | 
| 660 | 
> | 
               integrableObject != NULL;  | 
| 661 | 
> | 
               integrableObject = mol->nextIntegrableObject(ii)){ | 
| 662 | 
> | 
            intObjIndex = integrableObject->getGlobalIndex(); | 
| 663 | 
> | 
            integrableObject->setZangle(tempZangs[intObjIndex]); | 
| 664 | 
> | 
          }      | 
| 665 | 
> | 
           | 
| 666 | 
> | 
        } else { | 
| 667 | 
> | 
          // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 668 | 
> | 
          // listen for the number of integrableObjects in the molecule | 
| 669 | 
> | 
          MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 670 | 
> | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 671 | 
> | 
           | 
| 672 | 
> | 
          for(j=0; j < nCurObj; j++){            | 
| 673 | 
> | 
            // listen for which integrableObject we need to send the value for | 
| 674 | 
> | 
            MPI_Recv(&intObjIndexTransfer, 1, MPI_INT, which_node, | 
| 675 | 
> | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus);  | 
| 676 | 
> | 
            angleTransfer = tempZangs[intObjIndexTransfer]; | 
| 677 | 
> | 
            // send the value to the node so it can initialize the object | 
| 678 | 
> | 
            MPI_Send(&angleTransfer, 1, MPI_REALTYPE, which_node,  | 
| 679 | 
> | 
                     TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD);                | 
| 680 | 
> | 
          } | 
| 681 | 
> | 
        } | 
| 682 | 
> | 
      } | 
| 683 | 
> | 
    } else { | 
| 684 | 
> | 
      // I am SLAVE TO THE MASTER | 
| 685 | 
> | 
      for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 686 | 
> | 
        int which_node = info_->getMolToProc(i); | 
| 687 | 
> | 
 | 
| 688 | 
> | 
        if (which_node == worldRank) { | 
| 689 | 
> | 
           | 
| 690 | 
> | 
          // BUT I OWN THIS MOLECULE!!! | 
| 691 | 
> | 
           | 
| 692 | 
> | 
          mol = info_->getMoleculeByGlobalIndex(i); | 
| 693 | 
> | 
 | 
| 694 | 
> | 
          if(mol == NULL) { | 
| 695 | 
> | 
            sprintf(painCave.errMsg,  | 
| 696 | 
> | 
                    "RestReader Error: molecule not found on node %d\n",  | 
| 697 | 
> | 
                    worldRank); | 
| 698 | 
> | 
            painCave.isFatal = 1; | 
| 699 | 
  | 
            simError(); | 
| 700 | 
  | 
          } | 
| 701 | 
< | 
           | 
| 702 | 
< | 
          vecParticles = (info_->molecules[localIndex]).getIntegrableObjects();  | 
| 703 | 
< | 
          for(j = 0; j < vecParticles.size(); j++){        | 
| 704 | 
< | 
            vecParticles[j]->setZangle(tempZangs[index]); | 
| 705 | 
< | 
            index++; | 
| 706 | 
< | 
          }      | 
| 707 | 
< | 
           | 
| 708 | 
< | 
        } else { | 
| 709 | 
< | 
          // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 710 | 
< | 
           | 
| 711 | 
< | 
          MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 712 | 
< | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 713 | 
< | 
           | 
| 714 | 
< | 
          for(j=0; j < nCurObj; j++){             | 
| 715 | 
< | 
            angleTransfer = tempZangs[index]; | 
| 716 | 
< | 
            MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node,  | 
| 717 | 
< | 
                     TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD);                | 
| 718 | 
< | 
            index++; | 
| 719 | 
< | 
          } | 
| 720 | 
< | 
           | 
| 677 | 
< | 
        } | 
| 678 | 
< | 
         | 
| 679 | 
< | 
      } else { | 
| 680 | 
< | 
        // I am SLAVE TO THE MASTER | 
| 681 | 
< | 
         | 
| 682 | 
< | 
        if (which_node == worldRank) { | 
| 683 | 
< | 
           | 
| 684 | 
< | 
          // BUT I OWN THIS MOLECULE!!! | 
| 685 | 
< | 
           | 
| 686 | 
< | 
          localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 687 | 
< | 
          vecParticles = (info_->molecules[localIndex]).getIntegrableObjects();  | 
| 688 | 
< | 
          nCurObj = vecParticles.size(); | 
| 689 | 
< | 
           | 
| 690 | 
< | 
          MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 691 | 
< | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 692 | 
< | 
           | 
| 693 | 
< | 
          for(j = 0; j < vecParticles.size(); j++){ | 
| 694 | 
< | 
             | 
| 695 | 
< | 
            MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, | 
| 696 | 
< | 
                     TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); | 
| 697 | 
< | 
            vecParticles[j]->setZangle(angleTransfer); | 
| 698 | 
< | 
          }      | 
| 699 | 
< | 
        } | 
| 701 | 
> | 
 | 
| 702 | 
> | 
          nCurObj = mol->getNIntegrableObjects(); | 
| 703 | 
> | 
          // send the number of integrableObjects in the molecule | 
| 704 | 
> | 
          MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 705 | 
> | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 706 | 
> | 
           | 
| 707 | 
> | 
          for (integrableObject = mol->beginIntegrableObject(ii);  | 
| 708 | 
> | 
               integrableObject != NULL;  | 
| 709 | 
> | 
               integrableObject = mol->nextIntegrableObject(ii)){ | 
| 710 | 
> | 
            intObjIndexTransfer = integrableObject->getGlobalIndex(); | 
| 711 | 
> | 
            // send the global index of the integrableObject | 
| 712 | 
> | 
            MPI_Send(&intObjIndexTransfer, 1, MPI_INT, 0, | 
| 713 | 
> | 
                     TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 714 | 
> | 
            // listen for the value we want to set locally | 
| 715 | 
> | 
            MPI_Recv(&angleTransfer, 1, MPI_REALTYPE, 0, | 
| 716 | 
> | 
                     TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); | 
| 717 | 
> | 
 | 
| 718 | 
> | 
            integrableObject->setZangle(angleTransfer); | 
| 719 | 
> | 
          }      | 
| 720 | 
> | 
        } | 
| 721 | 
  | 
      } | 
| 722 | 
< | 
    } | 
| 722 | 
> | 
    }  | 
| 723 | 
  | 
#endif | 
| 724 | 
  | 
  } | 
| 725 | 
  | 
   | 
| 753 | 
  | 
    // first thing first, suspend fatalities. | 
| 754 | 
  | 
    painCave.isEventLoop = 1; | 
| 755 | 
  | 
     | 
| 756 | 
+ | 
    int masterNode = 0; | 
| 757 | 
  | 
    int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone | 
| 758 | 
< | 
    int haveError, index; | 
| 758 | 
> | 
    int haveError; | 
| 759 | 
  | 
    int which_node; | 
| 760 | 
  | 
     | 
| 761 | 
  | 
    MPI_Status istatus; | 
| 762 | 
< | 
    int *MolToProcMap = mpiSim->getMolToProcMap(); | 
| 741 | 
< | 
    int localIndex; | 
| 762 | 
> | 
     | 
| 763 | 
  | 
    int nCurObj; | 
| 764 | 
< | 
    double angleTranfer; | 
| 764 | 
> | 
    RealType angleTranfer; | 
| 765 | 
  | 
     | 
| 766 | 
< | 
    nTotObjs = info_->getTotIntegrableObjects(); | 
| 766 | 
> | 
    nTotObjs = info_->getNGlobalIntegrableObjects(); | 
| 767 | 
  | 
    haveError = 0; | 
| 768 | 
< | 
     | 
| 769 | 
< | 
    for (i=0 ; i < mpiSim->getNMolGlobal(); i++) { | 
| 770 | 
< | 
      // Get the Node number which has this atom | 
| 771 | 
< | 
      which_node = MolToProcMap[i]; | 
| 772 | 
< | 
       | 
| 773 | 
< | 
      // let's let node 0 pass out constant values to all the processors | 
| 774 | 
< | 
      if (worldRank == 0) { | 
| 775 | 
< | 
        if (which_node == 0) { | 
| 776 | 
< | 
          localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 777 | 
< | 
           | 
| 778 | 
< | 
          if(localIndex == -1) { | 
| 779 | 
< | 
            strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 780 | 
< | 
            haveError = 1; | 
| 781 | 
< | 
            simError(); | 
| 782 | 
< | 
          } | 
| 783 | 
< | 
           | 
| 784 | 
< | 
          vecParticles = (info_->molecules[localIndex]).getIntegrableObjects();  | 
| 785 | 
< | 
          for(j = 0; j < vecParticles.size(); j++){        | 
| 786 | 
< | 
            vecParticles[j]->setZangle( 0.0 ); | 
| 787 | 
< | 
          }      | 
| 788 | 
< | 
           | 
| 789 | 
< | 
        } else { | 
| 790 | 
< | 
          // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 791 | 
< | 
           | 
| 792 | 
< | 
          MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 793 | 
< | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 794 | 
< | 
           | 
| 795 | 
< | 
          for(j=0; j < nCurObj; j++){             | 
| 796 | 
< | 
            angleTransfer = 0.0; | 
| 797 | 
< | 
            MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node,  | 
| 798 | 
< | 
                     TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD);                | 
| 799 | 
< | 
            index++; | 
| 800 | 
< | 
          } | 
| 801 | 
< | 
        } | 
| 802 | 
< | 
      } else { | 
| 803 | 
< | 
        // I am SLAVE TO THE MASTER | 
| 804 | 
< | 
         | 
| 805 | 
< | 
        if (which_node == worldRank) { | 
| 806 | 
< | 
           | 
| 807 | 
< | 
          // BUT I OWN THIS MOLECULE!!! | 
| 808 | 
< | 
           | 
| 809 | 
< | 
          localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 810 | 
< | 
          vecParticles = (info_->molecules[localIndex]).getIntegrableObjects();  | 
| 811 | 
< | 
          nCurObj = vecParticles.size(); | 
| 812 | 
< | 
           | 
| 813 | 
< | 
          MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 814 | 
< | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 815 | 
< | 
           | 
| 816 | 
< | 
          for(j = 0; j < vecParticles.size(); j++){ | 
| 817 | 
< | 
             | 
| 818 | 
< | 
            MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, | 
| 768 | 
> | 
    if (worldRank == masterNode) { | 
| 769 | 
> | 
 | 
| 770 | 
> | 
      for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 771 | 
> | 
        // Get the Node number which has this atom | 
| 772 | 
> | 
        which_node = info_->getMolToProc(i); | 
| 773 | 
> | 
         | 
| 774 | 
> | 
        // let's let node 0 pass out constant values to all the processors | 
| 775 | 
> | 
        if (worldRank == masterNode) { | 
| 776 | 
> | 
          mol = info_->getMoleculeByGlobalIndex(i); | 
| 777 | 
> | 
           | 
| 778 | 
> | 
          if(mol == NULL) { | 
| 779 | 
> | 
            strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 780 | 
> | 
            haveError = 1; | 
| 781 | 
> | 
            simError(); | 
| 782 | 
> | 
          } | 
| 783 | 
> | 
           | 
| 784 | 
> | 
          for (integrableObject = mol->beginIntegrableObject(ii);  | 
| 785 | 
> | 
               integrableObject != NULL;  | 
| 786 | 
> | 
               integrableObject = mol->nextIntegrableObject(ii)){ | 
| 787 | 
> | 
             | 
| 788 | 
> | 
            integrableObject->setZangle( 0.0 ); | 
| 789 | 
> | 
             | 
| 790 | 
> | 
          } | 
| 791 | 
> | 
           | 
| 792 | 
> | 
        } else { | 
| 793 | 
> | 
          // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 794 | 
> | 
           | 
| 795 | 
> | 
          MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 796 | 
> | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 797 | 
> | 
           | 
| 798 | 
> | 
          for(j=0; j < nCurObj; j++){             | 
| 799 | 
> | 
            angleTransfer = 0.0; | 
| 800 | 
> | 
            MPI_Send(&angleTransfer, 1, MPI_REALTYPE, which_node,  | 
| 801 | 
> | 
                     TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD);                | 
| 802 | 
> | 
             | 
| 803 | 
> | 
          } | 
| 804 | 
> | 
        } | 
| 805 | 
> | 
      } | 
| 806 | 
> | 
    } else { | 
| 807 | 
> | 
      // I am SLAVE TO THE MASTER | 
| 808 | 
> | 
      for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 809 | 
> | 
        int which_node = info_->getMolToProc(i); | 
| 810 | 
> | 
         | 
| 811 | 
> | 
        if (which_node == worldRank) { | 
| 812 | 
> | 
           | 
| 813 | 
> | 
          // BUT I OWN THIS MOLECULE!!! | 
| 814 | 
> | 
          mol = info_->getMoleculeByGlobalIndex(i); | 
| 815 | 
> | 
           | 
| 816 | 
> | 
          if(mol == NULL) { | 
| 817 | 
> | 
            sprintf(painCave.errMsg,  | 
| 818 | 
> | 
                    "RestReader Error: molecule not found on node %d\n",  | 
| 819 | 
> | 
                    worldRank); | 
| 820 | 
> | 
            painCave.isFatal = 1; | 
| 821 | 
> | 
            simError(); | 
| 822 | 
> | 
          } | 
| 823 | 
> | 
           | 
| 824 | 
> | 
          nCurObj = mol->getNIntegrableObjects(); | 
| 825 | 
> | 
           | 
| 826 | 
> | 
          MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 827 | 
> | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 828 | 
> | 
           | 
| 829 | 
> | 
          for (integrableObject = mol->beginIntegrableObject(ii);  | 
| 830 | 
> | 
               integrableObject != NULL;  | 
| 831 | 
> | 
               integrableObject = mol->nextIntegrableObject(ii)){ | 
| 832 | 
> | 
             | 
| 833 | 
> | 
            MPI_Recv(&angleTransfer, 1, MPI_REALTYPE, 0, | 
| 834 | 
  | 
                     TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); | 
| 835 | 
  | 
            vecParticles[j]->setZangle(angleTransfer); | 
| 836 | 
  | 
          }      |