| 60 |
|
|
| 61 |
|
#ifdef IS_MPI |
| 62 |
|
#include <mpi.h> |
| 63 |
– |
#include "brains/mpiSimulation.hpp" |
| 63 |
|
#define TAKE_THIS_TAG_CHAR 0 |
| 64 |
|
#define TAKE_THIS_TAG_INT 1 |
| 65 |
|
#define TAKE_THIS_TAG_DOUBLE 2 |
| 219 |
|
int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone |
| 220 |
|
|
| 221 |
|
MPI_Status istatus; |
| 223 |
– |
int localIndex; |
| 222 |
|
int nCurObj; |
| 223 |
|
int nitems; |
| 224 |
< |
|
| 225 |
< |
nTotObjs = info_->getTotIntegrableObjects(); |
| 224 |
> |
int haveError; |
| 225 |
> |
|
| 226 |
> |
nTotObjs = info_->getNGlobalIntegrableObjects(); |
| 227 |
|
haveError = 0; |
| 228 |
|
|
| 229 |
|
if (worldRank == masterNode) { |
| 265 |
|
if(which_node == masterNode){ |
| 266 |
|
//molecules belong to master node |
| 267 |
|
|
| 268 |
< |
localIndex = info_->getMoleculeByGlobalIndex(i); |
| 268 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
| 269 |
|
|
| 270 |
< |
if(localIndex == NULL) { |
| 271 |
< |
strcpy(painCave.errMsg, |
| 272 |
< |
"RestReader Error: Molecule not found on node %d!", |
| 270 |
> |
if(mol == NULL) { |
| 271 |
> |
sprintf(painCave.errMsg, |
| 272 |
> |
"RestReader Error: Molecule not found on node %d!\n", |
| 273 |
|
worldRank); |
| 274 |
|
painCave.isFatal = 1; |
| 275 |
|
simError(); |
| 312 |
|
simError(); |
| 313 |
|
} |
| 314 |
|
|
| 315 |
< |
if(haveError) nodeZeroError(); |
| 317 |
< |
|
| 318 |
< |
MPI_Send(read_buffer, BUFFERSIZE, MPI_CHAR, which_node, |
| 315 |
> |
MPI_Send(read_buffer, BUFFERSIZE, MPI_CHAR, which_node, |
| 316 |
|
TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD); |
| 317 |
|
} |
| 318 |
|
} |
| 325 |
|
if(which_node == worldRank){ |
| 326 |
|
//molecule with global index i belongs to this processor |
| 327 |
|
|
| 328 |
< |
localIndex = info_->getMoleculeByGlobalIndex(i); |
| 328 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
| 329 |
|
|
| 330 |
< |
if(localIndex == NULL) { |
| 330 |
> |
if(mol == NULL) { |
| 331 |
|
sprintf(painCave.errMsg, |
| 332 |
|
"RestReader Error: molecule not found on node %d\n", |
| 333 |
|
worldRank); |
| 335 |
|
simError(); |
| 336 |
|
} |
| 337 |
|
|
| 338 |
< |
nCurObj = localIndex->getNIntegrableObjects(); |
| 338 |
> |
nCurObj = mol->getNIntegrableObjects(); |
| 339 |
|
|
| 340 |
|
MPI_Send(&nCurObj, 1, MPI_INT, masterNode, |
| 341 |
|
TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
| 591 |
|
|
| 592 |
|
// first thing first, suspend fatalities. |
| 593 |
|
painCave.isEventLoop = 1; |
| 594 |
< |
|
| 594 |
> |
|
| 595 |
> |
int masterNode = 0; |
| 596 |
|
int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone |
| 597 |
< |
int haveError, index; |
| 598 |
< |
|
| 599 |
< |
int *MolToProcMap = mpiSim->getMolToProcMap(); |
| 602 |
< |
int localIndex; |
| 597 |
> |
int haveError; |
| 598 |
> |
int index; |
| 599 |
> |
|
| 600 |
|
int nCurObj; |
| 601 |
|
double angleTranfer; |
| 602 |
|
|
| 603 |
< |
nTotObjs = info_->getTotIntegrableObjects(); |
| 603 |
> |
nTotObjs = info_->getNGlobalIntegrableObjects(); |
| 604 |
|
haveError = 0; |
| 605 |
< |
if (worldRank == 0) { |
| 605 |
> |
|
| 606 |
> |
if (worldRank == masterNode) { |
| 607 |
|
|
| 608 |
|
eof_test = fgets(read_buffer, sizeof(read_buffer), inAngFile); |
| 609 |
|
if( eof_test == NULL ){ |
| 634 |
|
simError(); |
| 635 |
|
} |
| 636 |
|
|
| 637 |
< |
} |
| 638 |
< |
// At this point, node 0 has a tempZangs vector completed, and |
| 639 |
< |
// everyone else has nada |
| 642 |
< |
index = 0; |
| 643 |
< |
|
| 644 |
< |
for (i=0 ; i < mpiSim->getNMolGlobal(); i++) { |
| 645 |
< |
// Get the Node number which has this atom |
| 646 |
< |
which_node = MolToProcMap[i]; |
| 637 |
> |
// At this point, node 0 has a tempZangs vector completed, and |
| 638 |
> |
// everyone else has nada |
| 639 |
> |
index = 0; |
| 640 |
|
|
| 641 |
< |
if (worldRank == 0) { |
| 642 |
< |
if (which_node == 0) { |
| 643 |
< |
localIndex = mpiSim->getGlobalToLocalMol(i); |
| 644 |
< |
|
| 645 |
< |
if(localIndex == -1) { |
| 646 |
< |
strcpy(painCave.errMsg, "Molecule not found on node 0!"); |
| 647 |
< |
haveError = 1; |
| 641 |
> |
for (i=0 ; i < info_->getNGlobalMolecules(); i++) { |
| 642 |
> |
// Get the Node number which has this atom |
| 643 |
> |
which_node = info_->getMolToProc(i); |
| 644 |
> |
|
| 645 |
> |
if (worldRank == masterNode) { |
| 646 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
| 647 |
> |
|
| 648 |
> |
if(mol == NULL) { |
| 649 |
> |
strcpy(painCave.errMsg, "Molecule not found on node 0!"); |
| 650 |
> |
haveError = 1; |
| 651 |
> |
simError(); |
| 652 |
> |
} |
| 653 |
> |
|
| 654 |
> |
for (integrableObject = mol->beginIntegrableObject(ii); |
| 655 |
> |
integrableObject != NULL; |
| 656 |
> |
integrableObject = mol->nextIntegrableObject(ii)){ |
| 657 |
> |
|
| 658 |
> |
integrableObject->setZangle(tempZangs[index]); |
| 659 |
> |
index++; |
| 660 |
> |
} |
| 661 |
> |
|
| 662 |
> |
} else { |
| 663 |
> |
// I am MASTER OF THE UNIVERSE, but I don't own this molecule |
| 664 |
> |
|
| 665 |
> |
MPI_Recv(&nCurObj, 1, MPI_INT, which_node, |
| 666 |
> |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
| 667 |
> |
|
| 668 |
> |
for(j=0; j < nCurObj; j++){ |
| 669 |
> |
angleTransfer = tempZangs[index]; |
| 670 |
> |
MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, |
| 671 |
> |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); |
| 672 |
> |
index++; |
| 673 |
> |
} |
| 674 |
> |
|
| 675 |
> |
} |
| 676 |
> |
} |
| 677 |
> |
} else { |
| 678 |
> |
// I am SLAVE TO THE MASTER |
| 679 |
> |
for (i=0 ; i < info_->getNGlobalMolecules(); i++) { |
| 680 |
> |
int which_node = info_->getMolToProc(i); |
| 681 |
> |
|
| 682 |
> |
if (which_node == worldRank) { |
| 683 |
> |
|
| 684 |
> |
// BUT I OWN THIS MOLECULE!!! |
| 685 |
> |
|
| 686 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
| 687 |
> |
|
| 688 |
> |
if(mol == NULL) { |
| 689 |
> |
sprintf(painCave.errMsg, |
| 690 |
> |
"RestReader Error: molecule not found on node %d\n", |
| 691 |
> |
worldRank); |
| 692 |
> |
painCave.isFatal = 1; |
| 693 |
|
simError(); |
| 694 |
|
} |
| 695 |
< |
|
| 696 |
< |
vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); |
| 697 |
< |
for(j = 0; j < vecParticles.size(); j++){ |
| 698 |
< |
vecParticles[j]->setZangle(tempZangs[index]); |
| 699 |
< |
index++; |
| 700 |
< |
} |
| 701 |
< |
|
| 702 |
< |
} else { |
| 703 |
< |
// I am MASTER OF THE UNIVERSE, but I don't own this molecule |
| 704 |
< |
|
| 705 |
< |
MPI_Recv(&nCurObj, 1, MPI_INT, which_node, |
| 706 |
< |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
| 707 |
< |
|
| 708 |
< |
for(j=0; j < nCurObj; j++){ |
| 709 |
< |
angleTransfer = tempZangs[index]; |
| 710 |
< |
MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, |
| 673 |
< |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); |
| 674 |
< |
index++; |
| 675 |
< |
} |
| 676 |
< |
|
| 677 |
< |
} |
| 678 |
< |
|
| 679 |
< |
} else { |
| 680 |
< |
// I am SLAVE TO THE MASTER |
| 681 |
< |
|
| 682 |
< |
if (which_node == worldRank) { |
| 683 |
< |
|
| 684 |
< |
// BUT I OWN THIS MOLECULE!!! |
| 685 |
< |
|
| 686 |
< |
localIndex = mpiSim->getGlobalToLocalMol(i); |
| 687 |
< |
vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); |
| 688 |
< |
nCurObj = vecParticles.size(); |
| 689 |
< |
|
| 690 |
< |
MPI_Send(&nCurObj, 1, MPI_INT, 0, |
| 691 |
< |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
| 692 |
< |
|
| 693 |
< |
for(j = 0; j < vecParticles.size(); j++){ |
| 694 |
< |
|
| 695 |
< |
MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, |
| 696 |
< |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); |
| 697 |
< |
vecParticles[j]->setZangle(angleTransfer); |
| 698 |
< |
} |
| 699 |
< |
} |
| 695 |
> |
|
| 696 |
> |
nCurObj = mol->getNIntegrableObjects(); |
| 697 |
> |
|
| 698 |
> |
MPI_Send(&nCurObj, 1, MPI_INT, 0, |
| 699 |
> |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
| 700 |
> |
|
| 701 |
> |
for (integrableObject = mol->beginIntegrableObject(ii); |
| 702 |
> |
integrableObject != NULL; |
| 703 |
> |
integrableObject = mol->nextIntegrableObject(ii)){ |
| 704 |
> |
|
| 705 |
> |
MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, |
| 706 |
> |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); |
| 707 |
> |
|
| 708 |
> |
integrableObject->setZangle(angleTransfer); |
| 709 |
> |
} |
| 710 |
> |
} |
| 711 |
|
} |
| 712 |
< |
} |
| 712 |
> |
} |
| 713 |
|
#endif |
| 714 |
|
} |
| 715 |
|
|
| 743 |
|
// first thing first, suspend fatalities. |
| 744 |
|
painCave.isEventLoop = 1; |
| 745 |
|
|
| 746 |
+ |
int masterNode = 0; |
| 747 |
|
int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone |
| 748 |
< |
int haveError, index; |
| 748 |
> |
int haveError; |
| 749 |
|
int which_node; |
| 750 |
|
|
| 751 |
|
MPI_Status istatus; |
| 752 |
< |
int *MolToProcMap = mpiSim->getMolToProcMap(); |
| 741 |
< |
int localIndex; |
| 752 |
> |
|
| 753 |
|
int nCurObj; |
| 754 |
|
double angleTranfer; |
| 755 |
|
|
| 756 |
< |
nTotObjs = info_->getTotIntegrableObjects(); |
| 756 |
> |
nTotObjs = info_->getNGlobalIntegrableObjects(); |
| 757 |
|
haveError = 0; |
| 758 |
< |
|
| 759 |
< |
for (i=0 ; i < mpiSim->getNMolGlobal(); i++) { |
| 760 |
< |
// Get the Node number which has this atom |
| 761 |
< |
which_node = MolToProcMap[i]; |
| 762 |
< |
|
| 763 |
< |
// let's let node 0 pass out constant values to all the processors |
| 764 |
< |
if (worldRank == 0) { |
| 765 |
< |
if (which_node == 0) { |
| 766 |
< |
localIndex = mpiSim->getGlobalToLocalMol(i); |
| 767 |
< |
|
| 768 |
< |
if(localIndex == -1) { |
| 769 |
< |
strcpy(painCave.errMsg, "Molecule not found on node 0!"); |
| 770 |
< |
haveError = 1; |
| 771 |
< |
simError(); |
| 772 |
< |
} |
| 773 |
< |
|
| 774 |
< |
vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); |
| 775 |
< |
for(j = 0; j < vecParticles.size(); j++){ |
| 776 |
< |
vecParticles[j]->setZangle( 0.0 ); |
| 777 |
< |
} |
| 778 |
< |
|
| 779 |
< |
} else { |
| 780 |
< |
// I am MASTER OF THE UNIVERSE, but I don't own this molecule |
| 781 |
< |
|
| 782 |
< |
MPI_Recv(&nCurObj, 1, MPI_INT, which_node, |
| 783 |
< |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
| 784 |
< |
|
| 785 |
< |
for(j=0; j < nCurObj; j++){ |
| 786 |
< |
angleTransfer = 0.0; |
| 787 |
< |
MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, |
| 788 |
< |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); |
| 789 |
< |
index++; |
| 790 |
< |
} |
| 791 |
< |
} |
| 792 |
< |
} else { |
| 793 |
< |
// I am SLAVE TO THE MASTER |
| 794 |
< |
|
| 795 |
< |
if (which_node == worldRank) { |
| 796 |
< |
|
| 797 |
< |
// BUT I OWN THIS MOLECULE!!! |
| 798 |
< |
|
| 799 |
< |
localIndex = mpiSim->getGlobalToLocalMol(i); |
| 800 |
< |
vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); |
| 801 |
< |
nCurObj = vecParticles.size(); |
| 802 |
< |
|
| 803 |
< |
MPI_Send(&nCurObj, 1, MPI_INT, 0, |
| 804 |
< |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
| 805 |
< |
|
| 806 |
< |
for(j = 0; j < vecParticles.size(); j++){ |
| 807 |
< |
|
| 758 |
> |
if (worldRank == masterNode) { |
| 759 |
> |
|
| 760 |
> |
for (i=0 ; i < info_->getNGlobalMolecules(); i++) { |
| 761 |
> |
// Get the Node number which has this atom |
| 762 |
> |
which_node = info_->getMolToProc(i); |
| 763 |
> |
|
| 764 |
> |
// let's let node 0 pass out constant values to all the processors |
| 765 |
> |
if (worldRank == masterNode) { |
| 766 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
| 767 |
> |
|
| 768 |
> |
if(mol == NULL) { |
| 769 |
> |
strcpy(painCave.errMsg, "Molecule not found on node 0!"); |
| 770 |
> |
haveError = 1; |
| 771 |
> |
simError(); |
| 772 |
> |
} |
| 773 |
> |
|
| 774 |
> |
for (integrableObject = mol->beginIntegrableObject(ii); |
| 775 |
> |
integrableObject != NULL; |
| 776 |
> |
integrableObject = mol->nextIntegrableObject(ii)){ |
| 777 |
> |
|
| 778 |
> |
integrableObject->setZangle( 0.0 ); |
| 779 |
> |
|
| 780 |
> |
} |
| 781 |
> |
|
| 782 |
> |
} else { |
| 783 |
> |
// I am MASTER OF THE UNIVERSE, but I don't own this molecule |
| 784 |
> |
|
| 785 |
> |
MPI_Recv(&nCurObj, 1, MPI_INT, which_node, |
| 786 |
> |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
| 787 |
> |
|
| 788 |
> |
for(j=0; j < nCurObj; j++){ |
| 789 |
> |
angleTransfer = 0.0; |
| 790 |
> |
MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, |
| 791 |
> |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); |
| 792 |
> |
|
| 793 |
> |
} |
| 794 |
> |
} |
| 795 |
> |
} |
| 796 |
> |
} else { |
| 797 |
> |
// I am SLAVE TO THE MASTER |
| 798 |
> |
for (i=0 ; i < info_->getNGlobalMolecules(); i++) { |
| 799 |
> |
int which_node = info_->getMolToProc(i); |
| 800 |
> |
|
| 801 |
> |
if (which_node == worldRank) { |
| 802 |
> |
|
| 803 |
> |
// BUT I OWN THIS MOLECULE!!! |
| 804 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
| 805 |
> |
|
| 806 |
> |
if(mol == NULL) { |
| 807 |
> |
sprintf(painCave.errMsg, |
| 808 |
> |
"RestReader Error: molecule not found on node %d\n", |
| 809 |
> |
worldRank); |
| 810 |
> |
painCave.isFatal = 1; |
| 811 |
> |
simError(); |
| 812 |
> |
} |
| 813 |
> |
|
| 814 |
> |
nCurObj = mol->getNIntegrableObjects(); |
| 815 |
> |
|
| 816 |
> |
MPI_Send(&nCurObj, 1, MPI_INT, 0, |
| 817 |
> |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
| 818 |
> |
|
| 819 |
> |
for (integrableObject = mol->beginIntegrableObject(ii); |
| 820 |
> |
integrableObject != NULL; |
| 821 |
> |
integrableObject = mol->nextIntegrableObject(ii)){ |
| 822 |
> |
|
| 823 |
|
MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, |
| 824 |
|
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); |
| 825 |
|
vecParticles[j]->setZangle(angleTransfer); |