| 60 |  |  | 
| 61 |  | #ifdef IS_MPI | 
| 62 |  | #include <mpi.h> | 
| 63 | – | #include "brains/mpiSimulation.hpp" | 
| 63 |  | #define TAKE_THIS_TAG_CHAR 0 | 
| 64 |  | #define TAKE_THIS_TAG_INT 1 | 
| 65 |  | #define TAKE_THIS_TAG_DOUBLE 2 | 
| 219 |  | int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone | 
| 220 |  |  | 
| 221 |  | MPI_Status istatus; | 
| 223 | – | int localIndex; | 
| 222 |  | int nCurObj; | 
| 223 |  | int nitems; | 
| 224 | < |  | 
| 225 | < | nTotObjs = info_->getTotIntegrableObjects(); | 
| 224 | > | int haveError; | 
| 225 | > |  | 
| 226 | > | nTotObjs = info_->getNGlobalIntegrableObjects(); | 
| 227 |  | haveError = 0; | 
| 228 |  |  | 
| 229 |  | if (worldRank == masterNode) { | 
| 265 |  | if(which_node == masterNode){ | 
| 266 |  | //molecules belong to master node | 
| 267 |  |  | 
| 268 | < | localIndex = info_->getMoleculeByGlobalIndex(i); | 
| 268 | > | mol = info_->getMoleculeByGlobalIndex(i); | 
| 269 |  |  | 
| 270 | < | if(localIndex == NULL) { | 
| 271 | < | strcpy(painCave.errMsg, | 
| 272 | < | "RestReader Error: Molecule not found on node %d!", | 
| 270 | > | if(mol == NULL) { | 
| 271 | > | sprintf(painCave.errMsg, | 
| 272 | > | "RestReader Error: Molecule not found on node %d!\n", | 
| 273 |  | worldRank); | 
| 274 |  | painCave.isFatal = 1; | 
| 275 |  | simError(); | 
| 312 |  | simError(); | 
| 313 |  | } | 
| 314 |  |  | 
| 315 | < | if(haveError) nodeZeroError(); | 
| 317 | < |  | 
| 318 | < | MPI_Send(read_buffer, BUFFERSIZE, MPI_CHAR, which_node, | 
| 315 | > | MPI_Send(read_buffer, BUFFERSIZE, MPI_CHAR, which_node, | 
| 316 |  | TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD); | 
| 317 |  | } | 
| 318 |  | } | 
| 325 |  | if(which_node == worldRank){ | 
| 326 |  | //molecule with global index i belongs to this processor | 
| 327 |  |  | 
| 328 | < | localIndex = info_->getMoleculeByGlobalIndex(i); | 
| 328 | > | mol = info_->getMoleculeByGlobalIndex(i); | 
| 329 |  |  | 
| 330 | < | if(localIndex == NULL) { | 
| 330 | > | if(mol == NULL) { | 
| 331 |  | sprintf(painCave.errMsg, | 
| 332 |  | "RestReader Error: molecule not found on node %d\n", | 
| 333 |  | worldRank); | 
| 335 |  | simError(); | 
| 336 |  | } | 
| 337 |  |  | 
| 338 | < | nCurObj = localIndex->getNIntegrableObjects(); | 
| 338 | > | nCurObj = mol->getNIntegrableObjects(); | 
| 339 |  |  | 
| 340 |  | MPI_Send(&nCurObj, 1, MPI_INT, masterNode, | 
| 341 |  | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 591 |  |  | 
| 592 |  | // first thing first, suspend fatalities. | 
| 593 |  | painCave.isEventLoop = 1; | 
| 594 | < |  | 
| 594 | > |  | 
| 595 | > | int masterNode = 0; | 
| 596 |  | int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone | 
| 597 | < | int haveError, index; | 
| 598 | < |  | 
| 599 | < | int *MolToProcMap = mpiSim->getMolToProcMap(); | 
| 602 | < | int localIndex; | 
| 597 | > | int haveError; | 
| 598 | > | int index; | 
| 599 | > |  | 
| 600 |  | int nCurObj; | 
| 601 |  | double angleTranfer; | 
| 602 |  |  | 
| 603 | < | nTotObjs = info_->getTotIntegrableObjects(); | 
| 603 | > | nTotObjs = info_->getNGlobalIntegrableObjects(); | 
| 604 |  | haveError = 0; | 
| 605 | < | if (worldRank == 0) { | 
| 605 | > |  | 
| 606 | > | if (worldRank == masterNode) { | 
| 607 |  |  | 
| 608 |  | eof_test = fgets(read_buffer, sizeof(read_buffer), inAngFile); | 
| 609 |  | if( eof_test == NULL ){ | 
| 634 |  | simError(); | 
| 635 |  | } | 
| 636 |  |  | 
| 637 | < | } | 
| 638 | < | // At this point, node 0 has a tempZangs vector completed, and | 
| 639 | < | // everyone else has nada | 
| 642 | < | index = 0; | 
| 643 | < |  | 
| 644 | < | for (i=0 ; i < mpiSim->getNMolGlobal(); i++) { | 
| 645 | < | // Get the Node number which has this atom | 
| 646 | < | which_node = MolToProcMap[i]; | 
| 637 | > | // At this point, node 0 has a tempZangs vector completed, and | 
| 638 | > | // everyone else has nada | 
| 639 | > | index = 0; | 
| 640 |  |  | 
| 641 | < | if (worldRank == 0) { | 
| 642 | < | if (which_node == 0) { | 
| 643 | < | localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 644 | < |  | 
| 645 | < | if(localIndex == -1) { | 
| 646 | < | strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 647 | < | haveError = 1; | 
| 641 | > | for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 642 | > | // Get the Node number which has this atom | 
| 643 | > | which_node = info_->getMolToProc(i); | 
| 644 | > |  | 
| 645 | > | if (worldRank == masterNode) { | 
| 646 | > | mol = info_->getMoleculeByGlobalIndex(i); | 
| 647 | > |  | 
| 648 | > | if(mol == NULL) { | 
| 649 | > | strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 650 | > | haveError = 1; | 
| 651 | > | simError(); | 
| 652 | > | } | 
| 653 | > |  | 
| 654 | > | for (integrableObject = mol->beginIntegrableObject(ii); | 
| 655 | > | integrableObject != NULL; | 
| 656 | > | integrableObject = mol->nextIntegrableObject(ii)){ | 
| 657 | > |  | 
| 658 | > | integrableObject->setZangle(tempZangs[index]); | 
| 659 | > | index++; | 
| 660 | > | } | 
| 661 | > |  | 
| 662 | > | } else { | 
| 663 | > | // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 664 | > |  | 
| 665 | > | MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 666 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 667 | > |  | 
| 668 | > | for(j=0; j < nCurObj; j++){ | 
| 669 | > | angleTransfer = tempZangs[index]; | 
| 670 | > | MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, | 
| 671 | > | TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); | 
| 672 | > | index++; | 
| 673 | > | } | 
| 674 | > |  | 
| 675 | > | } | 
| 676 | > | } | 
| 677 | > | } else { | 
| 678 | > | // I am SLAVE TO THE MASTER | 
| 679 | > | for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 680 | > | int which_node = info_->getMolToProc(i); | 
| 681 | > |  | 
| 682 | > | if (which_node == worldRank) { | 
| 683 | > |  | 
| 684 | > | // BUT I OWN THIS MOLECULE!!! | 
| 685 | > |  | 
| 686 | > | mol = info_->getMoleculeByGlobalIndex(i); | 
| 687 | > |  | 
| 688 | > | if(mol == NULL) { | 
| 689 | > | sprintf(painCave.errMsg, | 
| 690 | > | "RestReader Error: molecule not found on node %d\n", | 
| 691 | > | worldRank); | 
| 692 | > | painCave.isFatal = 1; | 
| 693 |  | simError(); | 
| 694 |  | } | 
| 695 | < |  | 
| 696 | < | vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); | 
| 697 | < | for(j = 0; j < vecParticles.size(); j++){ | 
| 698 | < | vecParticles[j]->setZangle(tempZangs[index]); | 
| 699 | < | index++; | 
| 700 | < | } | 
| 701 | < |  | 
| 702 | < | } else { | 
| 703 | < | // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 704 | < |  | 
| 705 | < | MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 706 | < | TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 707 | < |  | 
| 708 | < | for(j=0; j < nCurObj; j++){ | 
| 709 | < | angleTransfer = tempZangs[index]; | 
| 710 | < | MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, | 
| 673 | < | TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); | 
| 674 | < | index++; | 
| 675 | < | } | 
| 676 | < |  | 
| 677 | < | } | 
| 678 | < |  | 
| 679 | < | } else { | 
| 680 | < | // I am SLAVE TO THE MASTER | 
| 681 | < |  | 
| 682 | < | if (which_node == worldRank) { | 
| 683 | < |  | 
| 684 | < | // BUT I OWN THIS MOLECULE!!! | 
| 685 | < |  | 
| 686 | < | localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 687 | < | vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); | 
| 688 | < | nCurObj = vecParticles.size(); | 
| 689 | < |  | 
| 690 | < | MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 691 | < | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 692 | < |  | 
| 693 | < | for(j = 0; j < vecParticles.size(); j++){ | 
| 694 | < |  | 
| 695 | < | MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, | 
| 696 | < | TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); | 
| 697 | < | vecParticles[j]->setZangle(angleTransfer); | 
| 698 | < | } | 
| 699 | < | } | 
| 695 | > |  | 
| 696 | > | nCurObj = mol->getNIntegrableObjects(); | 
| 697 | > |  | 
| 698 | > | MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 699 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 700 | > |  | 
| 701 | > | for (integrableObject = mol->beginIntegrableObject(ii); | 
| 702 | > | integrableObject != NULL; | 
| 703 | > | integrableObject = mol->nextIntegrableObject(ii)){ | 
| 704 | > |  | 
| 705 | > | MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, | 
| 706 | > | TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); | 
| 707 | > |  | 
| 708 | > | integrableObject->setZangle(angleTransfer); | 
| 709 | > | } | 
| 710 | > | } | 
| 711 |  | } | 
| 712 | < | } | 
| 712 | > | } | 
| 713 |  | #endif | 
| 714 |  | } | 
| 715 |  |  | 
| 743 |  | // first thing first, suspend fatalities. | 
| 744 |  | painCave.isEventLoop = 1; | 
| 745 |  |  | 
| 746 | + | int masterNode = 0; | 
| 747 |  | int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone | 
| 748 | < | int haveError, index; | 
| 748 | > | int haveError; | 
| 749 |  | int which_node; | 
| 750 |  |  | 
| 751 |  | MPI_Status istatus; | 
| 752 | < | int *MolToProcMap = mpiSim->getMolToProcMap(); | 
| 741 | < | int localIndex; | 
| 752 | > |  | 
| 753 |  | int nCurObj; | 
| 754 |  | double angleTranfer; | 
| 755 |  |  | 
| 756 | < | nTotObjs = info_->getTotIntegrableObjects(); | 
| 756 | > | nTotObjs = info_->getNGlobalIntegrableObjects(); | 
| 757 |  | haveError = 0; | 
| 758 | < |  | 
| 759 | < | for (i=0 ; i < mpiSim->getNMolGlobal(); i++) { | 
| 760 | < | // Get the Node number which has this atom | 
| 761 | < | which_node = MolToProcMap[i]; | 
| 762 | < |  | 
| 763 | < | // let's let node 0 pass out constant values to all the processors | 
| 764 | < | if (worldRank == 0) { | 
| 765 | < | if (which_node == 0) { | 
| 766 | < | localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 767 | < |  | 
| 768 | < | if(localIndex == -1) { | 
| 769 | < | strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 770 | < | haveError = 1; | 
| 771 | < | simError(); | 
| 772 | < | } | 
| 773 | < |  | 
| 774 | < | vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); | 
| 775 | < | for(j = 0; j < vecParticles.size(); j++){ | 
| 776 | < | vecParticles[j]->setZangle( 0.0 ); | 
| 777 | < | } | 
| 778 | < |  | 
| 779 | < | } else { | 
| 780 | < | // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 781 | < |  | 
| 782 | < | MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 783 | < | TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 784 | < |  | 
| 785 | < | for(j=0; j < nCurObj; j++){ | 
| 786 | < | angleTransfer = 0.0; | 
| 787 | < | MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, | 
| 788 | < | TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); | 
| 789 | < | index++; | 
| 790 | < | } | 
| 791 | < | } | 
| 792 | < | } else { | 
| 793 | < | // I am SLAVE TO THE MASTER | 
| 794 | < |  | 
| 795 | < | if (which_node == worldRank) { | 
| 796 | < |  | 
| 797 | < | // BUT I OWN THIS MOLECULE!!! | 
| 798 | < |  | 
| 799 | < | localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 800 | < | vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); | 
| 801 | < | nCurObj = vecParticles.size(); | 
| 802 | < |  | 
| 803 | < | MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 804 | < | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 805 | < |  | 
| 806 | < | for(j = 0; j < vecParticles.size(); j++){ | 
| 807 | < |  | 
| 758 | > | if (worldRank == masterNode) { | 
| 759 | > |  | 
| 760 | > | for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 761 | > | // Get the Node number which has this atom | 
| 762 | > | which_node = info_->getMolToProc(i); | 
| 763 | > |  | 
| 764 | > | // let's let node 0 pass out constant values to all the processors | 
| 765 | > | if (worldRank == masterNode) { | 
| 766 | > | mol = info_->getMoleculeByGlobalIndex(i); | 
| 767 | > |  | 
| 768 | > | if(mol == NULL) { | 
| 769 | > | strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 770 | > | haveError = 1; | 
| 771 | > | simError(); | 
| 772 | > | } | 
| 773 | > |  | 
| 774 | > | for (integrableObject = mol->beginIntegrableObject(ii); | 
| 775 | > | integrableObject != NULL; | 
| 776 | > | integrableObject = mol->nextIntegrableObject(ii)){ | 
| 777 | > |  | 
| 778 | > | integrableObject->setZangle( 0.0 ); | 
| 779 | > |  | 
| 780 | > | } | 
| 781 | > |  | 
| 782 | > | } else { | 
| 783 | > | // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 784 | > |  | 
| 785 | > | MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 786 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 787 | > |  | 
| 788 | > | for(j=0; j < nCurObj; j++){ | 
| 789 | > | angleTransfer = 0.0; | 
| 790 | > | MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, | 
| 791 | > | TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); | 
| 792 | > |  | 
| 793 | > | } | 
| 794 | > | } | 
| 795 | > | } | 
| 796 | > | } else { | 
| 797 | > | // I am SLAVE TO THE MASTER | 
| 798 | > | for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 799 | > | int which_node = info_->getMolToProc(i); | 
| 800 | > |  | 
| 801 | > | if (which_node == worldRank) { | 
| 802 | > |  | 
| 803 | > | // BUT I OWN THIS MOLECULE!!! | 
| 804 | > | mol = info_->getMoleculeByGlobalIndex(i); | 
| 805 | > |  | 
| 806 | > | if(mol == NULL) { | 
| 807 | > | sprintf(painCave.errMsg, | 
| 808 | > | "RestReader Error: molecule not found on node %d\n", | 
| 809 | > | worldRank); | 
| 810 | > | painCave.isFatal = 1; | 
| 811 | > | simError(); | 
| 812 | > | } | 
| 813 | > |  | 
| 814 | > | nCurObj = mol->getNIntegrableObjects(); | 
| 815 | > |  | 
| 816 | > | MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 817 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 818 | > |  | 
| 819 | > | for (integrableObject = mol->beginIntegrableObject(ii); | 
| 820 | > | integrableObject != NULL; | 
| 821 | > | integrableObject = mol->nextIntegrableObject(ii)){ | 
| 822 | > |  | 
| 823 |  | MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, | 
| 824 |  | TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); | 
| 825 |  | vecParticles[j]->setZangle(angleTransfer); |