60 |
|
|
61 |
|
#ifdef IS_MPI |
62 |
|
#include <mpi.h> |
63 |
– |
#include "brains/mpiSimulation.hpp" |
63 |
|
#define TAKE_THIS_TAG_CHAR 0 |
64 |
|
#define TAKE_THIS_TAG_INT 1 |
65 |
|
#define TAKE_THIS_TAG_DOUBLE 2 |
219 |
|
int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone |
220 |
|
|
221 |
|
MPI_Status istatus; |
223 |
– |
int localIndex; |
222 |
|
int nCurObj; |
223 |
|
int nitems; |
224 |
< |
|
225 |
< |
nTotObjs = info_->getTotIntegrableObjects(); |
224 |
> |
int haveError; |
225 |
> |
|
226 |
> |
nTotObjs = info_->getNGlobalIntegrableObjects(); |
227 |
|
haveError = 0; |
228 |
|
|
229 |
|
if (worldRank == masterNode) { |
265 |
|
if(which_node == masterNode){ |
266 |
|
//molecules belong to master node |
267 |
|
|
268 |
< |
localIndex = info_->getMoleculeByGlobalIndex(i); |
268 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
269 |
|
|
270 |
< |
if(localIndex == NULL) { |
271 |
< |
strcpy(painCave.errMsg, |
272 |
< |
"RestReader Error: Molecule not found on node %d!", |
270 |
> |
if(mol == NULL) { |
271 |
> |
sprintf(painCave.errMsg, |
272 |
> |
"RestReader Error: Molecule not found on node %d!\n", |
273 |
|
worldRank); |
274 |
|
painCave.isFatal = 1; |
275 |
|
simError(); |
312 |
|
simError(); |
313 |
|
} |
314 |
|
|
315 |
< |
if(haveError) nodeZeroError(); |
317 |
< |
|
318 |
< |
MPI_Send(read_buffer, BUFFERSIZE, MPI_CHAR, which_node, |
315 |
> |
MPI_Send(read_buffer, BUFFERSIZE, MPI_CHAR, which_node, |
316 |
|
TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD); |
317 |
|
} |
318 |
|
} |
325 |
|
if(which_node == worldRank){ |
326 |
|
//molecule with global index i belongs to this processor |
327 |
|
|
328 |
< |
localIndex = info_->getMoleculeByGlobalIndex(i); |
328 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
329 |
|
|
330 |
< |
if(localIndex == NULL) { |
330 |
> |
if(mol == NULL) { |
331 |
|
sprintf(painCave.errMsg, |
332 |
|
"RestReader Error: molecule not found on node %d\n", |
333 |
|
worldRank); |
335 |
|
simError(); |
336 |
|
} |
337 |
|
|
338 |
< |
nCurObj = localIndex->getNIntegrableObjects(); |
338 |
> |
nCurObj = mol->getNIntegrableObjects(); |
339 |
|
|
340 |
|
MPI_Send(&nCurObj, 1, MPI_INT, masterNode, |
341 |
|
TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
591 |
|
|
592 |
|
// first thing first, suspend fatalities. |
593 |
|
painCave.isEventLoop = 1; |
594 |
< |
|
594 |
> |
|
595 |
> |
int masterNode = 0; |
596 |
|
int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone |
597 |
< |
int haveError, index; |
598 |
< |
|
599 |
< |
int *MolToProcMap = mpiSim->getMolToProcMap(); |
602 |
< |
int localIndex; |
597 |
> |
int haveError; |
598 |
> |
int index; |
599 |
> |
|
600 |
|
int nCurObj; |
601 |
|
double angleTranfer; |
602 |
|
|
603 |
< |
nTotObjs = info_->getTotIntegrableObjects(); |
603 |
> |
nTotObjs = info_->getNGlobalIntegrableObjects(); |
604 |
|
haveError = 0; |
605 |
< |
if (worldRank == 0) { |
605 |
> |
|
606 |
> |
if (worldRank == masterNode) { |
607 |
|
|
608 |
|
eof_test = fgets(read_buffer, sizeof(read_buffer), inAngFile); |
609 |
|
if( eof_test == NULL ){ |
634 |
|
simError(); |
635 |
|
} |
636 |
|
|
637 |
< |
} |
638 |
< |
// At this point, node 0 has a tempZangs vector completed, and |
639 |
< |
// everyone else has nada |
642 |
< |
index = 0; |
643 |
< |
|
644 |
< |
for (i=0 ; i < mpiSim->getNMolGlobal(); i++) { |
645 |
< |
// Get the Node number which has this atom |
646 |
< |
which_node = MolToProcMap[i]; |
637 |
> |
// At this point, node 0 has a tempZangs vector completed, and |
638 |
> |
// everyone else has nada |
639 |
> |
index = 0; |
640 |
|
|
641 |
< |
if (worldRank == 0) { |
642 |
< |
if (which_node == 0) { |
643 |
< |
localIndex = mpiSim->getGlobalToLocalMol(i); |
644 |
< |
|
645 |
< |
if(localIndex == -1) { |
646 |
< |
strcpy(painCave.errMsg, "Molecule not found on node 0!"); |
647 |
< |
haveError = 1; |
641 |
> |
for (i=0 ; i < info_->getNGlobalMolecules(); i++) { |
642 |
> |
// Get the Node number which has this atom |
643 |
> |
which_node = info_->getMolToProc(i); |
644 |
> |
|
645 |
> |
if (worldRank == masterNode) { |
646 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
647 |
> |
|
648 |
> |
if(mol == NULL) { |
649 |
> |
strcpy(painCave.errMsg, "Molecule not found on node 0!"); |
650 |
> |
haveError = 1; |
651 |
> |
simError(); |
652 |
> |
} |
653 |
> |
|
654 |
> |
for (integrableObject = mol->beginIntegrableObject(ii); |
655 |
> |
integrableObject != NULL; |
656 |
> |
integrableObject = mol->nextIntegrableObject(ii)){ |
657 |
> |
|
658 |
> |
integrableObject->setZangle(tempZangs[index]); |
659 |
> |
index++; |
660 |
> |
} |
661 |
> |
|
662 |
> |
} else { |
663 |
> |
// I am MASTER OF THE UNIVERSE, but I don't own this molecule |
664 |
> |
|
665 |
> |
MPI_Recv(&nCurObj, 1, MPI_INT, which_node, |
666 |
> |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
667 |
> |
|
668 |
> |
for(j=0; j < nCurObj; j++){ |
669 |
> |
angleTransfer = tempZangs[index]; |
670 |
> |
MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, |
671 |
> |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); |
672 |
> |
index++; |
673 |
> |
} |
674 |
> |
|
675 |
> |
} |
676 |
> |
} |
677 |
> |
} else { |
678 |
> |
// I am SLAVE TO THE MASTER |
679 |
> |
for (i=0 ; i < info_->getNGlobalMolecules(); i++) { |
680 |
> |
int which_node = info_->getMolToProc(i); |
681 |
> |
|
682 |
> |
if (which_node == worldRank) { |
683 |
> |
|
684 |
> |
// BUT I OWN THIS MOLECULE!!! |
685 |
> |
|
686 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
687 |
> |
|
688 |
> |
if(mol == NULL) { |
689 |
> |
sprintf(painCave.errMsg, |
690 |
> |
"RestReader Error: molecule not found on node %d\n", |
691 |
> |
worldRank); |
692 |
> |
painCave.isFatal = 1; |
693 |
|
simError(); |
694 |
|
} |
695 |
< |
|
696 |
< |
vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); |
697 |
< |
for(j = 0; j < vecParticles.size(); j++){ |
698 |
< |
vecParticles[j]->setZangle(tempZangs[index]); |
699 |
< |
index++; |
700 |
< |
} |
701 |
< |
|
702 |
< |
} else { |
703 |
< |
// I am MASTER OF THE UNIVERSE, but I don't own this molecule |
704 |
< |
|
705 |
< |
MPI_Recv(&nCurObj, 1, MPI_INT, which_node, |
706 |
< |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
707 |
< |
|
708 |
< |
for(j=0; j < nCurObj; j++){ |
709 |
< |
angleTransfer = tempZangs[index]; |
710 |
< |
MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, |
673 |
< |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); |
674 |
< |
index++; |
675 |
< |
} |
676 |
< |
|
677 |
< |
} |
678 |
< |
|
679 |
< |
} else { |
680 |
< |
// I am SLAVE TO THE MASTER |
681 |
< |
|
682 |
< |
if (which_node == worldRank) { |
683 |
< |
|
684 |
< |
// BUT I OWN THIS MOLECULE!!! |
685 |
< |
|
686 |
< |
localIndex = mpiSim->getGlobalToLocalMol(i); |
687 |
< |
vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); |
688 |
< |
nCurObj = vecParticles.size(); |
689 |
< |
|
690 |
< |
MPI_Send(&nCurObj, 1, MPI_INT, 0, |
691 |
< |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
692 |
< |
|
693 |
< |
for(j = 0; j < vecParticles.size(); j++){ |
694 |
< |
|
695 |
< |
MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, |
696 |
< |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); |
697 |
< |
vecParticles[j]->setZangle(angleTransfer); |
698 |
< |
} |
699 |
< |
} |
695 |
> |
|
696 |
> |
nCurObj = mol->getNIntegrableObjects(); |
697 |
> |
|
698 |
> |
MPI_Send(&nCurObj, 1, MPI_INT, 0, |
699 |
> |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
700 |
> |
|
701 |
> |
for (integrableObject = mol->beginIntegrableObject(ii); |
702 |
> |
integrableObject != NULL; |
703 |
> |
integrableObject = mol->nextIntegrableObject(ii)){ |
704 |
> |
|
705 |
> |
MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, |
706 |
> |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); |
707 |
> |
|
708 |
> |
integrableObject->setZangle(angleTransfer); |
709 |
> |
} |
710 |
> |
} |
711 |
|
} |
712 |
< |
} |
712 |
> |
} |
713 |
|
#endif |
714 |
|
} |
715 |
|
|
743 |
|
// first thing first, suspend fatalities. |
744 |
|
painCave.isEventLoop = 1; |
745 |
|
|
746 |
+ |
int masterNode = 0; |
747 |
|
int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone |
748 |
< |
int haveError, index; |
748 |
> |
int haveError; |
749 |
|
int which_node; |
750 |
|
|
751 |
|
MPI_Status istatus; |
752 |
< |
int *MolToProcMap = mpiSim->getMolToProcMap(); |
741 |
< |
int localIndex; |
752 |
> |
|
753 |
|
int nCurObj; |
754 |
|
double angleTranfer; |
755 |
|
|
756 |
< |
nTotObjs = info_->getTotIntegrableObjects(); |
756 |
> |
nTotObjs = info_->getNGlobalIntegrableObjects(); |
757 |
|
haveError = 0; |
758 |
< |
|
759 |
< |
for (i=0 ; i < mpiSim->getNMolGlobal(); i++) { |
760 |
< |
// Get the Node number which has this atom |
761 |
< |
which_node = MolToProcMap[i]; |
762 |
< |
|
763 |
< |
// let's let node 0 pass out constant values to all the processors |
764 |
< |
if (worldRank == 0) { |
765 |
< |
if (which_node == 0) { |
766 |
< |
localIndex = mpiSim->getGlobalToLocalMol(i); |
767 |
< |
|
768 |
< |
if(localIndex == -1) { |
769 |
< |
strcpy(painCave.errMsg, "Molecule not found on node 0!"); |
770 |
< |
haveError = 1; |
771 |
< |
simError(); |
772 |
< |
} |
773 |
< |
|
774 |
< |
vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); |
775 |
< |
for(j = 0; j < vecParticles.size(); j++){ |
776 |
< |
vecParticles[j]->setZangle( 0.0 ); |
777 |
< |
} |
778 |
< |
|
779 |
< |
} else { |
780 |
< |
// I am MASTER OF THE UNIVERSE, but I don't own this molecule |
781 |
< |
|
782 |
< |
MPI_Recv(&nCurObj, 1, MPI_INT, which_node, |
783 |
< |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
784 |
< |
|
785 |
< |
for(j=0; j < nCurObj; j++){ |
786 |
< |
angleTransfer = 0.0; |
787 |
< |
MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, |
788 |
< |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); |
789 |
< |
index++; |
790 |
< |
} |
791 |
< |
} |
792 |
< |
} else { |
793 |
< |
// I am SLAVE TO THE MASTER |
794 |
< |
|
795 |
< |
if (which_node == worldRank) { |
796 |
< |
|
797 |
< |
// BUT I OWN THIS MOLECULE!!! |
798 |
< |
|
799 |
< |
localIndex = mpiSim->getGlobalToLocalMol(i); |
800 |
< |
vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); |
801 |
< |
nCurObj = vecParticles.size(); |
802 |
< |
|
803 |
< |
MPI_Send(&nCurObj, 1, MPI_INT, 0, |
804 |
< |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
805 |
< |
|
806 |
< |
for(j = 0; j < vecParticles.size(); j++){ |
807 |
< |
|
758 |
> |
if (worldRank == masterNode) { |
759 |
> |
|
760 |
> |
for (i=0 ; i < info_->getNGlobalMolecules(); i++) { |
761 |
> |
// Get the Node number which has this atom |
762 |
> |
which_node = info_->getMolToProc(i); |
763 |
> |
|
764 |
> |
// let's let node 0 pass out constant values to all the processors |
765 |
> |
if (worldRank == masterNode) { |
766 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
767 |
> |
|
768 |
> |
if(mol == NULL) { |
769 |
> |
strcpy(painCave.errMsg, "Molecule not found on node 0!"); |
770 |
> |
haveError = 1; |
771 |
> |
simError(); |
772 |
> |
} |
773 |
> |
|
774 |
> |
for (integrableObject = mol->beginIntegrableObject(ii); |
775 |
> |
integrableObject != NULL; |
776 |
> |
integrableObject = mol->nextIntegrableObject(ii)){ |
777 |
> |
|
778 |
> |
integrableObject->setZangle( 0.0 ); |
779 |
> |
|
780 |
> |
} |
781 |
> |
|
782 |
> |
} else { |
783 |
> |
// I am MASTER OF THE UNIVERSE, but I don't own this molecule |
784 |
> |
|
785 |
> |
MPI_Recv(&nCurObj, 1, MPI_INT, which_node, |
786 |
> |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
787 |
> |
|
788 |
> |
for(j=0; j < nCurObj; j++){ |
789 |
> |
angleTransfer = 0.0; |
790 |
> |
MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, |
791 |
> |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); |
792 |
> |
|
793 |
> |
} |
794 |
> |
} |
795 |
> |
} |
796 |
> |
} else { |
797 |
> |
// I am SLAVE TO THE MASTER |
798 |
> |
for (i=0 ; i < info_->getNGlobalMolecules(); i++) { |
799 |
> |
int which_node = info_->getMolToProc(i); |
800 |
> |
|
801 |
> |
if (which_node == worldRank) { |
802 |
> |
|
803 |
> |
// BUT I OWN THIS MOLECULE!!! |
804 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
805 |
> |
|
806 |
> |
if(mol == NULL) { |
807 |
> |
sprintf(painCave.errMsg, |
808 |
> |
"RestReader Error: molecule not found on node %d\n", |
809 |
> |
worldRank); |
810 |
> |
painCave.isFatal = 1; |
811 |
> |
simError(); |
812 |
> |
} |
813 |
> |
|
814 |
> |
nCurObj = mol->getNIntegrableObjects(); |
815 |
> |
|
816 |
> |
MPI_Send(&nCurObj, 1, MPI_INT, 0, |
817 |
> |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
818 |
> |
|
819 |
> |
for (integrableObject = mol->beginIntegrableObject(ii); |
820 |
> |
integrableObject != NULL; |
821 |
> |
integrableObject = mol->nextIntegrableObject(ii)){ |
822 |
> |
|
823 |
|
MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, |
824 |
|
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); |
825 |
|
vecParticles[j]->setZangle(angleTransfer); |