| 60 | 
  | 
 | 
| 61 | 
  | 
#ifdef IS_MPI | 
| 62 | 
  | 
#include <mpi.h> | 
| 63 | 
– | 
#include "brains/mpiSimulation.hpp" | 
| 63 | 
  | 
#define TAKE_THIS_TAG_CHAR 0 | 
| 64 | 
  | 
#define TAKE_THIS_TAG_INT 1 | 
| 65 | 
  | 
#define TAKE_THIS_TAG_DOUBLE 2 | 
| 219 | 
  | 
    int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone | 
| 220 | 
  | 
     | 
| 221 | 
  | 
    MPI_Status istatus; | 
| 223 | 
– | 
    int localIndex; | 
| 222 | 
  | 
    int nCurObj; | 
| 223 | 
  | 
    int nitems; | 
| 224 | 
< | 
     | 
| 225 | 
< | 
    nTotObjs = info_->getTotIntegrableObjects(); | 
| 224 | 
> | 
    int haveError; | 
| 225 | 
> | 
 | 
| 226 | 
> | 
    nTotObjs = info_->getNGlobalIntegrableObjects(); | 
| 227 | 
  | 
    haveError = 0; | 
| 228 | 
  | 
     | 
| 229 | 
  | 
    if (worldRank == masterNode) { | 
| 265 | 
  | 
        if(which_node == masterNode){ | 
| 266 | 
  | 
          //molecules belong to master node | 
| 267 | 
  | 
           | 
| 268 | 
< | 
          localIndex = info_->getMoleculeByGlobalIndex(i); | 
| 268 | 
> | 
          mol = info_->getMoleculeByGlobalIndex(i); | 
| 269 | 
  | 
           | 
| 270 | 
< | 
          if(localIndex == NULL) { | 
| 271 | 
< | 
            strcpy(painCave.errMsg,  | 
| 272 | 
< | 
                   "RestReader Error: Molecule not found on node %d!", | 
| 270 | 
> | 
          if(mol == NULL) { | 
| 271 | 
> | 
            sprintf(painCave.errMsg,  | 
| 272 | 
> | 
                   "RestReader Error: Molecule not found on node %d!\n", | 
| 273 | 
  | 
                   worldRank); | 
| 274 | 
  | 
            painCave.isFatal = 1; | 
| 275 | 
  | 
            simError(); | 
| 312 | 
  | 
              simError(); | 
| 313 | 
  | 
            } | 
| 314 | 
  | 
             | 
| 315 | 
< | 
            if(haveError) nodeZeroError(); | 
| 317 | 
< | 
             | 
| 318 | 
< | 
            MPI_Send(read_buffer, BUFFERSIZE, MPI_CHAR, which_node, | 
| 315 | 
> | 
            MPI_Send(read_buffer, BUFFERSIZE, MPI_CHAR, which_node, | 
| 316 | 
  | 
                     TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD); | 
| 317 | 
  | 
          } | 
| 318 | 
  | 
        } | 
| 325 | 
  | 
        if(which_node == worldRank){ | 
| 326 | 
  | 
          //molecule with global index i belongs to this processor | 
| 327 | 
  | 
           | 
| 328 | 
< | 
          localIndex = info_->getMoleculeByGlobalIndex(i); | 
| 328 | 
> | 
          mol = info_->getMoleculeByGlobalIndex(i); | 
| 329 | 
  | 
           | 
| 330 | 
< | 
          if(localIndex == NULL) { | 
| 330 | 
> | 
          if(mol == NULL) { | 
| 331 | 
  | 
            sprintf(painCave.errMsg,  | 
| 332 | 
  | 
                    "RestReader Error: molecule not found on node %d\n",  | 
| 333 | 
  | 
                    worldRank); | 
| 335 | 
  | 
            simError(); | 
| 336 | 
  | 
          } | 
| 337 | 
  | 
           | 
| 338 | 
< | 
          nCurObj = localIndex->getNIntegrableObjects(); | 
| 338 | 
> | 
          nCurObj = mol->getNIntegrableObjects(); | 
| 339 | 
  | 
           | 
| 340 | 
  | 
          MPI_Send(&nCurObj, 1, MPI_INT, masterNode, | 
| 341 | 
  | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 591 | 
  | 
     | 
| 592 | 
  | 
    // first thing first, suspend fatalities. | 
| 593 | 
  | 
    painCave.isEventLoop = 1; | 
| 594 | 
< | 
     | 
| 594 | 
> | 
 | 
| 595 | 
> | 
    int masterNode = 0; | 
| 596 | 
  | 
    int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone | 
| 597 | 
< | 
    int haveError, index; | 
| 598 | 
< | 
     | 
| 599 | 
< | 
    int *MolToProcMap = mpiSim->getMolToProcMap(); | 
| 602 | 
< | 
    int localIndex; | 
| 597 | 
> | 
    int haveError; | 
| 598 | 
> | 
    int index;     | 
| 599 | 
> | 
 | 
| 600 | 
  | 
    int nCurObj; | 
| 601 | 
  | 
    double angleTranfer; | 
| 602 | 
  | 
     | 
| 603 | 
< | 
    nTotObjs = info_->getTotIntegrableObjects(); | 
| 603 | 
> | 
    nTotObjs = info_->getNGlobalIntegrableObjects(); | 
| 604 | 
  | 
    haveError = 0; | 
| 605 | 
< | 
    if (worldRank == 0) { | 
| 605 | 
> | 
 | 
| 606 | 
> | 
    if (worldRank == masterNode) { | 
| 607 | 
  | 
       | 
| 608 | 
  | 
      eof_test = fgets(read_buffer, sizeof(read_buffer), inAngFile); | 
| 609 | 
  | 
      if( eof_test == NULL ){ | 
| 634 | 
  | 
        simError(); | 
| 635 | 
  | 
      } | 
| 636 | 
  | 
       | 
| 637 | 
< | 
    } | 
| 638 | 
< | 
    // At this point, node 0 has a tempZangs vector completed, and  | 
| 639 | 
< | 
    // everyone else has nada | 
| 642 | 
< | 
    index = 0; | 
| 643 | 
< | 
     | 
| 644 | 
< | 
    for (i=0 ; i < mpiSim->getNMolGlobal(); i++) { | 
| 645 | 
< | 
      // Get the Node number which has this atom | 
| 646 | 
< | 
      which_node = MolToProcMap[i]; | 
| 637 | 
> | 
      // At this point, node 0 has a tempZangs vector completed, and  | 
| 638 | 
> | 
      // everyone else has nada | 
| 639 | 
> | 
      index = 0; | 
| 640 | 
  | 
       | 
| 641 | 
< | 
      if (worldRank == 0) { | 
| 642 | 
< | 
        if (which_node == 0) { | 
| 643 | 
< | 
          localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 644 | 
< | 
           | 
| 645 | 
< | 
          if(localIndex == -1) { | 
| 646 | 
< | 
            strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 647 | 
< | 
            haveError = 1; | 
| 641 | 
> | 
      for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 642 | 
> | 
        // Get the Node number which has this atom | 
| 643 | 
> | 
        which_node = info_->getMolToProc(i); | 
| 644 | 
> | 
         | 
| 645 | 
> | 
        if (worldRank == masterNode) { | 
| 646 | 
> | 
          mol = info_->getMoleculeByGlobalIndex(i); | 
| 647 | 
> | 
           | 
| 648 | 
> | 
          if(mol == NULL) { | 
| 649 | 
> | 
            strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 650 | 
> | 
            haveError = 1; | 
| 651 | 
> | 
            simError(); | 
| 652 | 
> | 
          } | 
| 653 | 
> | 
           | 
| 654 | 
> | 
          for (integrableObject = mol->beginIntegrableObject(ii);  | 
| 655 | 
> | 
               integrableObject != NULL;  | 
| 656 | 
> | 
               integrableObject = mol->nextIntegrableObject(ii)){ | 
| 657 | 
> | 
             | 
| 658 | 
> | 
            integrableObject->setZangle(tempZangs[index]); | 
| 659 | 
> | 
            index++; | 
| 660 | 
> | 
          }      | 
| 661 | 
> | 
           | 
| 662 | 
> | 
        } else { | 
| 663 | 
> | 
          // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 664 | 
> | 
           | 
| 665 | 
> | 
          MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 666 | 
> | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 667 | 
> | 
           | 
| 668 | 
> | 
          for(j=0; j < nCurObj; j++){             | 
| 669 | 
> | 
            angleTransfer = tempZangs[index]; | 
| 670 | 
> | 
            MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node,  | 
| 671 | 
> | 
                     TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD);                | 
| 672 | 
> | 
            index++; | 
| 673 | 
> | 
          } | 
| 674 | 
> | 
           | 
| 675 | 
> | 
        } | 
| 676 | 
> | 
      } | 
| 677 | 
> | 
    } else { | 
| 678 | 
> | 
      // I am SLAVE TO THE MASTER | 
| 679 | 
> | 
      for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 680 | 
> | 
        int which_node = info_->getMolToProc(i); | 
| 681 | 
> | 
 | 
| 682 | 
> | 
        if (which_node == worldRank) { | 
| 683 | 
> | 
           | 
| 684 | 
> | 
          // BUT I OWN THIS MOLECULE!!! | 
| 685 | 
> | 
           | 
| 686 | 
> | 
          mol = info_->getMoleculeByGlobalIndex(i); | 
| 687 | 
> | 
 | 
| 688 | 
> | 
          if(mol == NULL) { | 
| 689 | 
> | 
            sprintf(painCave.errMsg,  | 
| 690 | 
> | 
                    "RestReader Error: molecule not found on node %d\n",  | 
| 691 | 
> | 
                    worldRank); | 
| 692 | 
> | 
            painCave.isFatal = 1; | 
| 693 | 
  | 
            simError(); | 
| 694 | 
  | 
          } | 
| 695 | 
< | 
           | 
| 696 | 
< | 
          vecParticles = (info_->molecules[localIndex]).getIntegrableObjects();  | 
| 697 | 
< | 
          for(j = 0; j < vecParticles.size(); j++){        | 
| 698 | 
< | 
            vecParticles[j]->setZangle(tempZangs[index]); | 
| 699 | 
< | 
            index++; | 
| 700 | 
< | 
          }      | 
| 701 | 
< | 
           | 
| 702 | 
< | 
        } else { | 
| 703 | 
< | 
          // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 704 | 
< | 
           | 
| 705 | 
< | 
          MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 706 | 
< | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 707 | 
< | 
           | 
| 708 | 
< | 
          for(j=0; j < nCurObj; j++){             | 
| 709 | 
< | 
            angleTransfer = tempZangs[index]; | 
| 710 | 
< | 
            MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node,  | 
| 673 | 
< | 
                     TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD);                | 
| 674 | 
< | 
            index++; | 
| 675 | 
< | 
          } | 
| 676 | 
< | 
           | 
| 677 | 
< | 
        } | 
| 678 | 
< | 
         | 
| 679 | 
< | 
      } else { | 
| 680 | 
< | 
        // I am SLAVE TO THE MASTER | 
| 681 | 
< | 
         | 
| 682 | 
< | 
        if (which_node == worldRank) { | 
| 683 | 
< | 
           | 
| 684 | 
< | 
          // BUT I OWN THIS MOLECULE!!! | 
| 685 | 
< | 
           | 
| 686 | 
< | 
          localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 687 | 
< | 
          vecParticles = (info_->molecules[localIndex]).getIntegrableObjects();  | 
| 688 | 
< | 
          nCurObj = vecParticles.size(); | 
| 689 | 
< | 
           | 
| 690 | 
< | 
          MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 691 | 
< | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 692 | 
< | 
           | 
| 693 | 
< | 
          for(j = 0; j < vecParticles.size(); j++){ | 
| 694 | 
< | 
             | 
| 695 | 
< | 
            MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, | 
| 696 | 
< | 
                     TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); | 
| 697 | 
< | 
            vecParticles[j]->setZangle(angleTransfer); | 
| 698 | 
< | 
          }      | 
| 699 | 
< | 
        } | 
| 695 | 
> | 
 | 
| 696 | 
> | 
          nCurObj = mol->getNIntegrableObjects(); | 
| 697 | 
> | 
         | 
| 698 | 
> | 
          MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 699 | 
> | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 700 | 
> | 
           | 
| 701 | 
> | 
          for (integrableObject = mol->beginIntegrableObject(ii);  | 
| 702 | 
> | 
               integrableObject != NULL;  | 
| 703 | 
> | 
               integrableObject = mol->nextIntegrableObject(ii)){ | 
| 704 | 
> | 
             | 
| 705 | 
> | 
            MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, | 
| 706 | 
> | 
                     TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); | 
| 707 | 
> | 
 | 
| 708 | 
> | 
            integrableObject->setZangle(angleTransfer); | 
| 709 | 
> | 
          }      | 
| 710 | 
> | 
        } | 
| 711 | 
  | 
      } | 
| 712 | 
< | 
    } | 
| 712 | 
> | 
    }  | 
| 713 | 
  | 
#endif | 
| 714 | 
  | 
  } | 
| 715 | 
  | 
   | 
| 743 | 
  | 
    // first thing first, suspend fatalities. | 
| 744 | 
  | 
    painCave.isEventLoop = 1; | 
| 745 | 
  | 
     | 
| 746 | 
+ | 
    int masterNode = 0; | 
| 747 | 
  | 
    int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone | 
| 748 | 
< | 
    int haveError, index; | 
| 748 | 
> | 
    int haveError; | 
| 749 | 
  | 
    int which_node; | 
| 750 | 
  | 
     | 
| 751 | 
  | 
    MPI_Status istatus; | 
| 752 | 
< | 
    int *MolToProcMap = mpiSim->getMolToProcMap(); | 
| 741 | 
< | 
    int localIndex; | 
| 752 | 
> | 
     | 
| 753 | 
  | 
    int nCurObj; | 
| 754 | 
  | 
    double angleTranfer; | 
| 755 | 
  | 
     | 
| 756 | 
< | 
    nTotObjs = info_->getTotIntegrableObjects(); | 
| 756 | 
> | 
    nTotObjs = info_->getNGlobalIntegrableObjects(); | 
| 757 | 
  | 
    haveError = 0; | 
| 758 | 
< | 
     | 
| 759 | 
< | 
    for (i=0 ; i < mpiSim->getNMolGlobal(); i++) { | 
| 760 | 
< | 
      // Get the Node number which has this atom | 
| 761 | 
< | 
      which_node = MolToProcMap[i]; | 
| 762 | 
< | 
       | 
| 763 | 
< | 
      // let's let node 0 pass out constant values to all the processors | 
| 764 | 
< | 
      if (worldRank == 0) { | 
| 765 | 
< | 
        if (which_node == 0) { | 
| 766 | 
< | 
          localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 767 | 
< | 
           | 
| 768 | 
< | 
          if(localIndex == -1) { | 
| 769 | 
< | 
            strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 770 | 
< | 
            haveError = 1; | 
| 771 | 
< | 
            simError(); | 
| 772 | 
< | 
          } | 
| 773 | 
< | 
           | 
| 774 | 
< | 
          vecParticles = (info_->molecules[localIndex]).getIntegrableObjects();  | 
| 775 | 
< | 
          for(j = 0; j < vecParticles.size(); j++){        | 
| 776 | 
< | 
            vecParticles[j]->setZangle( 0.0 ); | 
| 777 | 
< | 
          }      | 
| 778 | 
< | 
           | 
| 779 | 
< | 
        } else { | 
| 780 | 
< | 
          // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 781 | 
< | 
           | 
| 782 | 
< | 
          MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 783 | 
< | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 784 | 
< | 
           | 
| 785 | 
< | 
          for(j=0; j < nCurObj; j++){             | 
| 786 | 
< | 
            angleTransfer = 0.0; | 
| 787 | 
< | 
            MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node,  | 
| 788 | 
< | 
                     TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD);                | 
| 789 | 
< | 
            index++; | 
| 790 | 
< | 
          } | 
| 791 | 
< | 
        } | 
| 792 | 
< | 
      } else { | 
| 793 | 
< | 
        // I am SLAVE TO THE MASTER | 
| 794 | 
< | 
         | 
| 795 | 
< | 
        if (which_node == worldRank) { | 
| 796 | 
< | 
           | 
| 797 | 
< | 
          // BUT I OWN THIS MOLECULE!!! | 
| 798 | 
< | 
           | 
| 799 | 
< | 
          localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 800 | 
< | 
          vecParticles = (info_->molecules[localIndex]).getIntegrableObjects();  | 
| 801 | 
< | 
          nCurObj = vecParticles.size(); | 
| 802 | 
< | 
           | 
| 803 | 
< | 
          MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 804 | 
< | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 805 | 
< | 
           | 
| 806 | 
< | 
          for(j = 0; j < vecParticles.size(); j++){ | 
| 807 | 
< | 
             | 
| 758 | 
> | 
    if (worldRank == masterNode) { | 
| 759 | 
> | 
 | 
| 760 | 
> | 
      for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 761 | 
> | 
        // Get the Node number which has this atom | 
| 762 | 
> | 
        which_node = info_->getMolToProc(i); | 
| 763 | 
> | 
         | 
| 764 | 
> | 
        // let's let node 0 pass out constant values to all the processors | 
| 765 | 
> | 
        if (worldRank == masterNode) { | 
| 766 | 
> | 
          mol = info_->getMoleculeByGlobalIndex(i); | 
| 767 | 
> | 
           | 
| 768 | 
> | 
          if(mol == NULL) { | 
| 769 | 
> | 
            strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 770 | 
> | 
            haveError = 1; | 
| 771 | 
> | 
            simError(); | 
| 772 | 
> | 
          } | 
| 773 | 
> | 
           | 
| 774 | 
> | 
          for (integrableObject = mol->beginIntegrableObject(ii);  | 
| 775 | 
> | 
               integrableObject != NULL;  | 
| 776 | 
> | 
               integrableObject = mol->nextIntegrableObject(ii)){ | 
| 777 | 
> | 
             | 
| 778 | 
> | 
            integrableObject->setZangle( 0.0 ); | 
| 779 | 
> | 
             | 
| 780 | 
> | 
          } | 
| 781 | 
> | 
           | 
| 782 | 
> | 
        } else { | 
| 783 | 
> | 
          // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 784 | 
> | 
           | 
| 785 | 
> | 
          MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 786 | 
> | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 787 | 
> | 
           | 
| 788 | 
> | 
          for(j=0; j < nCurObj; j++){             | 
| 789 | 
> | 
            angleTransfer = 0.0; | 
| 790 | 
> | 
            MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node,  | 
| 791 | 
> | 
                     TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD);                | 
| 792 | 
> | 
             | 
| 793 | 
> | 
          } | 
| 794 | 
> | 
        } | 
| 795 | 
> | 
      } | 
| 796 | 
> | 
    } else { | 
| 797 | 
> | 
      // I am SLAVE TO THE MASTER | 
| 798 | 
> | 
      for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 799 | 
> | 
        int which_node = info_->getMolToProc(i); | 
| 800 | 
> | 
         | 
| 801 | 
> | 
        if (which_node == worldRank) { | 
| 802 | 
> | 
           | 
| 803 | 
> | 
          // BUT I OWN THIS MOLECULE!!! | 
| 804 | 
> | 
          mol = info_->getMoleculeByGlobalIndex(i); | 
| 805 | 
> | 
           | 
| 806 | 
> | 
          if(mol == NULL) { | 
| 807 | 
> | 
            sprintf(painCave.errMsg,  | 
| 808 | 
> | 
                    "RestReader Error: molecule not found on node %d\n",  | 
| 809 | 
> | 
                    worldRank); | 
| 810 | 
> | 
            painCave.isFatal = 1; | 
| 811 | 
> | 
            simError(); | 
| 812 | 
> | 
          } | 
| 813 | 
> | 
           | 
| 814 | 
> | 
          nCurObj = mol->getNIntegrableObjects(); | 
| 815 | 
> | 
           | 
| 816 | 
> | 
          MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 817 | 
> | 
                   TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 818 | 
> | 
           | 
| 819 | 
> | 
          for (integrableObject = mol->beginIntegrableObject(ii);  | 
| 820 | 
> | 
               integrableObject != NULL;  | 
| 821 | 
> | 
               integrableObject = mol->nextIntegrableObject(ii)){ | 
| 822 | 
> | 
             | 
| 823 | 
  | 
            MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, | 
| 824 | 
  | 
                     TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); | 
| 825 | 
  | 
            vecParticles[j]->setZangle(angleTransfer); |