| 60 |  |  | 
| 61 |  | #ifdef IS_MPI | 
| 62 |  | #include <mpi.h> | 
| 63 | – | #include "brains/mpiSimulation.hpp" | 
| 63 |  | #define TAKE_THIS_TAG_CHAR 0 | 
| 64 |  | #define TAKE_THIS_TAG_INT 1 | 
| 65 |  | #define TAKE_THIS_TAG_DOUBLE 2 | 
| 68 |  | namespace oopse { | 
| 69 |  |  | 
| 70 |  | RestReader::RestReader( SimInfo* info ) : info_(info){ | 
| 71 | < |  | 
| 71 | > |  | 
| 72 |  | idealName = "idealCrystal.in"; | 
| 73 |  |  | 
| 74 |  | isScanned = false; | 
| 92 |  | "File \"idealCrystal.in\" opened successfully for reading." ); | 
| 93 |  | MPIcheckPoint(); | 
| 94 |  | #endif | 
| 95 | + |  | 
| 96 |  | return; | 
| 97 |  | } | 
| 98 |  |  | 
| 220 |  | int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone | 
| 221 |  |  | 
| 222 |  | MPI_Status istatus; | 
| 223 | – | int localIndex; | 
| 223 |  | int nCurObj; | 
| 224 |  | int nitems; | 
| 225 | < |  | 
| 226 | < | nTotObjs = info_->getTotIntegrableObjects(); | 
| 225 | > | int haveError; | 
| 226 | > |  | 
| 227 | > | nTotObjs = info_->getNGlobalIntegrableObjects(); | 
| 228 |  | haveError = 0; | 
| 229 | < |  | 
| 229 | > |  | 
| 230 |  | if (worldRank == masterNode) { | 
| 231 |  | eof_test = fgets(read_buffer, sizeof(read_buffer), inIdealFile); | 
| 232 |  | if( eof_test == NULL ){ | 
| 259 |  | painCave.isFatal = 1; | 
| 260 |  | simError(); | 
| 261 |  | } | 
| 262 | < |  | 
| 262 | > |  | 
| 263 |  | for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 264 |  | int which_node = info_->getMolToProc(i); | 
| 265 |  |  | 
| 266 |  | if(which_node == masterNode){ | 
| 267 |  | //molecules belong to master node | 
| 268 |  |  | 
| 269 | < | localIndex = info_->getMoleculeByGlobalIndex(i); | 
| 269 | > | mol = info_->getMoleculeByGlobalIndex(i); | 
| 270 |  |  | 
| 271 | < | if(localIndex == NULL) { | 
| 272 | < | strcpy(painCave.errMsg, | 
| 273 | < | "RestReader Error: Molecule not found on node %d!", | 
| 274 | < | worldRank); | 
| 271 | > | if(mol == NULL) { | 
| 272 | > | sprintf(painCave.errMsg, | 
| 273 | > | "RestReader Error: Molecule not found on node %d!\n", | 
| 274 | > | worldRank); | 
| 275 |  | painCave.isFatal = 1; | 
| 276 |  | simError(); | 
| 277 |  | } | 
| 291 |  | painCave.isFatal = 1; | 
| 292 |  | simError(); | 
| 293 |  | } | 
| 294 | < |  | 
| 295 | < | parseIdealLine(read_buffer, integrableObjects[j]); | 
| 294 | > |  | 
| 295 | > | parseIdealLine(read_buffer, integrableObject); | 
| 296 | > |  | 
| 297 |  | } | 
| 298 |  | } else { | 
| 299 |  | //molecule belongs to slave nodes | 
| 314 |  | simError(); | 
| 315 |  | } | 
| 316 |  |  | 
| 317 | < | if(haveError) nodeZeroError(); | 
| 317 | < |  | 
| 318 | < | MPI_Send(read_buffer, BUFFERSIZE, MPI_CHAR, which_node, | 
| 317 | > | MPI_Send(read_buffer, BUFFERSIZE, MPI_CHAR, which_node, | 
| 318 |  | TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD); | 
| 319 |  | } | 
| 320 |  | } | 
| 327 |  | if(which_node == worldRank){ | 
| 328 |  | //molecule with global index i belongs to this processor | 
| 329 |  |  | 
| 330 | < | localIndex = info_->getMoleculeByGlobalIndex(i); | 
| 330 | > | mol = info_->getMoleculeByGlobalIndex(i); | 
| 331 |  |  | 
| 332 | < | if(localIndex == NULL) { | 
| 332 | > | if(mol == NULL) { | 
| 333 |  | sprintf(painCave.errMsg, | 
| 334 |  | "RestReader Error: molecule not found on node %d\n", | 
| 335 |  | worldRank); | 
| 337 |  | simError(); | 
| 338 |  | } | 
| 339 |  |  | 
| 340 | < | nCurObj = localIndex->getNIntegrableObjects(); | 
| 340 | > | nCurObj = mol->getNIntegrableObjects(); | 
| 341 |  |  | 
| 342 |  | MPI_Send(&nCurObj, 1, MPI_INT, masterNode, | 
| 343 |  | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 366 |  |  | 
| 367 |  | char *foo; // the pointer to the current string token | 
| 368 |  |  | 
| 369 | < | double pos[3];        // position place holders | 
| 370 | < | double q[4];          // the quaternions | 
| 371 | < | double RfromQ[3][3];  // the rotation matrix | 
| 372 | < | double normalize;     // to normalize the reference unit vector | 
| 373 | < | double uX, uY, uZ;    // reference unit vector place holders | 
| 374 | < | double uselessToken; | 
| 369 | > | RealType pos[3];        // position place holders | 
| 370 | > | RealType q[4];          // the quaternions | 
| 371 | > | RealType RfromQ[3][3];  // the rotation matrix | 
| 372 | > | RealType normalize;     // to normalize the reference unit vector | 
| 373 | > | RealType uX, uY, uZ;    // reference unit vector place holders | 
| 374 | > | RealType uselessToken; | 
| 375 |  | StringTokenizer tokenizer(readLine); | 
| 376 |  | int nTokens; | 
| 377 |  |  | 
| 378 |  | nTokens = tokenizer.countTokens(); | 
| 379 | < |  | 
| 379 | > |  | 
| 380 |  | if (nTokens < 14) { | 
| 381 |  | sprintf(painCave.errMsg, | 
| 382 |  | "RestReader Error: Not enough Tokens.\n"); | 
| 385 |  | } | 
| 386 |  |  | 
| 387 |  | std::string name = tokenizer.nextToken(); | 
| 388 | < |  | 
| 388 | > |  | 
| 389 |  | if (name != sd->getType()) { | 
| 390 |  |  | 
| 391 |  | sprintf(painCave.errMsg, | 
| 400 |  | pos[0] = tokenizer.nextTokenAsDouble(); | 
| 401 |  | pos[1] = tokenizer.nextTokenAsDouble(); | 
| 402 |  | pos[2] = tokenizer.nextTokenAsDouble(); | 
| 403 | < |  | 
| 403 | > |  | 
| 404 |  | // store the positions in the stuntdouble as generic data doubles | 
| 405 |  | DoubleGenericData* refPosX = new DoubleGenericData(); | 
| 406 |  | refPosX->setID("refPosX"); | 
| 407 |  | refPosX->setData(pos[0]); | 
| 408 |  | sd->addProperty(refPosX); | 
| 409 | < |  | 
| 409 | > |  | 
| 410 |  | DoubleGenericData* refPosY = new DoubleGenericData(); | 
| 411 |  | refPosY->setID("refPosY"); | 
| 412 |  | refPosY->setData(pos[1]); | 
| 416 |  | refPosZ->setID("refPosZ"); | 
| 417 |  | refPosZ->setData(pos[2]); | 
| 418 |  | sd->addProperty(refPosZ); | 
| 419 | < |  | 
| 419 | > |  | 
| 420 |  | // we don't need the velocities | 
| 421 |  | uselessToken = tokenizer.nextTokenAsDouble(); | 
| 422 |  | uselessToken = tokenizer.nextTokenAsDouble(); | 
| 492 |  | char *parseErr; | 
| 493 |  |  | 
| 494 |  | std::vector<StuntDouble*> vecParticles; | 
| 495 | < | std::vector<double> tempZangs; | 
| 495 | > | std::vector<RealType> tempZangs; | 
| 496 |  |  | 
| 497 |  | inAngFileName = info_->getRestFileName(); | 
| 498 |  |  | 
| 593 |  |  | 
| 594 |  | // first thing first, suspend fatalities. | 
| 595 |  | painCave.isEventLoop = 1; | 
| 596 | < |  | 
| 596 | > |  | 
| 597 | > | int masterNode = 0; | 
| 598 |  | int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone | 
| 599 | < | int haveError, index; | 
| 600 | < |  | 
| 601 | < | int *MolToProcMap = mpiSim->getMolToProcMap(); | 
| 602 | < | int localIndex; | 
| 599 | > | int haveError; | 
| 600 | > | int index; | 
| 601 | > |  | 
| 602 |  | int nCurObj; | 
| 603 | < | double angleTranfer; | 
| 603 | > | RealType angleTranfer; | 
| 604 |  |  | 
| 605 | < | nTotObjs = info_->getTotIntegrableObjects(); | 
| 605 | > | nTotObjs = info_->getNGlobalIntegrableObjects(); | 
| 606 |  | haveError = 0; | 
| 607 | < | if (worldRank == 0) { | 
| 607 | > |  | 
| 608 | > | if (worldRank == masterNode) { | 
| 609 |  |  | 
| 610 |  | eof_test = fgets(read_buffer, sizeof(read_buffer), inAngFile); | 
| 611 |  | if( eof_test == NULL ){ | 
| 636 |  | simError(); | 
| 637 |  | } | 
| 638 |  |  | 
| 639 | < | } | 
| 640 | < | // At this point, node 0 has a tempZangs vector completed, and | 
| 641 | < | // everyone else has nada | 
| 642 | < | index = 0; | 
| 643 | < |  | 
| 644 | < | for (i=0 ; i < mpiSim->getNMolGlobal(); i++) { | 
| 645 | < | // Get the Node number which has this atom | 
| 646 | < | which_node = MolToProcMap[i]; | 
| 639 | > | // At this point, node 0 has a tempZangs vector completed, and | 
| 640 | > | // everyone else has nada | 
| 641 | > | index = 0; | 
| 642 |  |  | 
| 643 | < | if (worldRank == 0) { | 
| 644 | < | if (which_node == 0) { | 
| 645 | < | localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 646 | < |  | 
| 647 | < | if(localIndex == -1) { | 
| 648 | < | strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 649 | < | haveError = 1; | 
| 643 | > | for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 644 | > | // Get the Node number which has this atom | 
| 645 | > | which_node = info_->getMolToProc(i); | 
| 646 | > |  | 
| 647 | > | if (worldRank == masterNode) { | 
| 648 | > | mol = info_->getMoleculeByGlobalIndex(i); | 
| 649 | > |  | 
| 650 | > | if(mol == NULL) { | 
| 651 | > | strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 652 | > | haveError = 1; | 
| 653 | > | simError(); | 
| 654 | > | } | 
| 655 | > |  | 
| 656 | > | for (integrableObject = mol->beginIntegrableObject(ii); | 
| 657 | > | integrableObject != NULL; | 
| 658 | > | integrableObject = mol->nextIntegrableObject(ii)){ | 
| 659 | > |  | 
| 660 | > | integrableObject->setZangle(tempZangs[index]); | 
| 661 | > | index++; | 
| 662 | > | } | 
| 663 | > |  | 
| 664 | > | } else { | 
| 665 | > | // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 666 | > |  | 
| 667 | > | MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 668 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 669 | > |  | 
| 670 | > | for(j=0; j < nCurObj; j++){ | 
| 671 | > | angleTransfer = tempZangs[index]; | 
| 672 | > | MPI_Send(&angleTransfer, 1, MPI_REALTYPE, which_node, | 
| 673 | > | TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); | 
| 674 | > | index++; | 
| 675 | > | } | 
| 676 | > |  | 
| 677 | > | } | 
| 678 | > | } | 
| 679 | > | } else { | 
| 680 | > | // I am SLAVE TO THE MASTER | 
| 681 | > | for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 682 | > | int which_node = info_->getMolToProc(i); | 
| 683 | > |  | 
| 684 | > | if (which_node == worldRank) { | 
| 685 | > |  | 
| 686 | > | // BUT I OWN THIS MOLECULE!!! | 
| 687 | > |  | 
| 688 | > | mol = info_->getMoleculeByGlobalIndex(i); | 
| 689 | > |  | 
| 690 | > | if(mol == NULL) { | 
| 691 | > | sprintf(painCave.errMsg, | 
| 692 | > | "RestReader Error: molecule not found on node %d\n", | 
| 693 | > | worldRank); | 
| 694 | > | painCave.isFatal = 1; | 
| 695 |  | simError(); | 
| 696 |  | } | 
| 697 | < |  | 
| 698 | < | vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); | 
| 699 | < | for(j = 0; j < vecParticles.size(); j++){ | 
| 700 | < | vecParticles[j]->setZangle(tempZangs[index]); | 
| 701 | < | index++; | 
| 702 | < | } | 
| 703 | < |  | 
| 704 | < | } else { | 
| 705 | < | // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 706 | < |  | 
| 707 | < | MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 708 | < | TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 709 | < |  | 
| 710 | < | for(j=0; j < nCurObj; j++){ | 
| 711 | < | angleTransfer = tempZangs[index]; | 
| 712 | < | MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, | 
| 673 | < | TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); | 
| 674 | < | index++; | 
| 675 | < | } | 
| 676 | < |  | 
| 677 | < | } | 
| 678 | < |  | 
| 679 | < | } else { | 
| 680 | < | // I am SLAVE TO THE MASTER | 
| 681 | < |  | 
| 682 | < | if (which_node == worldRank) { | 
| 683 | < |  | 
| 684 | < | // BUT I OWN THIS MOLECULE!!! | 
| 685 | < |  | 
| 686 | < | localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 687 | < | vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); | 
| 688 | < | nCurObj = vecParticles.size(); | 
| 689 | < |  | 
| 690 | < | MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 691 | < | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 692 | < |  | 
| 693 | < | for(j = 0; j < vecParticles.size(); j++){ | 
| 694 | < |  | 
| 695 | < | MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, | 
| 696 | < | TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); | 
| 697 | < | vecParticles[j]->setZangle(angleTransfer); | 
| 698 | < | } | 
| 699 | < | } | 
| 697 | > |  | 
| 698 | > | nCurObj = mol->getNIntegrableObjects(); | 
| 699 | > |  | 
| 700 | > | MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 701 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 702 | > |  | 
| 703 | > | for (integrableObject = mol->beginIntegrableObject(ii); | 
| 704 | > | integrableObject != NULL; | 
| 705 | > | integrableObject = mol->nextIntegrableObject(ii)){ | 
| 706 | > |  | 
| 707 | > | MPI_Recv(&angleTransfer, 1, MPI_REALTYPE, 0, | 
| 708 | > | TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); | 
| 709 | > |  | 
| 710 | > | integrableObject->setZangle(angleTransfer); | 
| 711 | > | } | 
| 712 | > | } | 
| 713 |  | } | 
| 714 | < | } | 
| 714 | > | } | 
| 715 |  | #endif | 
| 716 |  | } | 
| 717 |  |  | 
| 745 |  | // first thing first, suspend fatalities. | 
| 746 |  | painCave.isEventLoop = 1; | 
| 747 |  |  | 
| 748 | + | int masterNode = 0; | 
| 749 |  | int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone | 
| 750 | < | int haveError, index; | 
| 750 | > | int haveError; | 
| 751 |  | int which_node; | 
| 752 |  |  | 
| 753 |  | MPI_Status istatus; | 
| 754 | < | int *MolToProcMap = mpiSim->getMolToProcMap(); | 
| 741 | < | int localIndex; | 
| 754 | > |  | 
| 755 |  | int nCurObj; | 
| 756 | < | double angleTranfer; | 
| 756 | > | RealType angleTranfer; | 
| 757 |  |  | 
| 758 | < | nTotObjs = info_->getTotIntegrableObjects(); | 
| 758 | > | nTotObjs = info_->getNGlobalIntegrableObjects(); | 
| 759 |  | haveError = 0; | 
| 760 | < |  | 
| 761 | < | for (i=0 ; i < mpiSim->getNMolGlobal(); i++) { | 
| 762 | < | // Get the Node number which has this atom | 
| 763 | < | which_node = MolToProcMap[i]; | 
| 764 | < |  | 
| 765 | < | // let's let node 0 pass out constant values to all the processors | 
| 766 | < | if (worldRank == 0) { | 
| 767 | < | if (which_node == 0) { | 
| 768 | < | localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 769 | < |  | 
| 770 | < | if(localIndex == -1) { | 
| 771 | < | strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 772 | < | haveError = 1; | 
| 773 | < | simError(); | 
| 774 | < | } | 
| 775 | < |  | 
| 776 | < | vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); | 
| 777 | < | for(j = 0; j < vecParticles.size(); j++){ | 
| 778 | < | vecParticles[j]->setZangle( 0.0 ); | 
| 779 | < | } | 
| 780 | < |  | 
| 781 | < | } else { | 
| 782 | < | // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 783 | < |  | 
| 784 | < | MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 785 | < | TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 786 | < |  | 
| 787 | < | for(j=0; j < nCurObj; j++){ | 
| 788 | < | angleTransfer = 0.0; | 
| 789 | < | MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, | 
| 790 | < | TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); | 
| 791 | < | index++; | 
| 792 | < | } | 
| 793 | < | } | 
| 794 | < | } else { | 
| 795 | < | // I am SLAVE TO THE MASTER | 
| 796 | < |  | 
| 797 | < | if (which_node == worldRank) { | 
| 798 | < |  | 
| 799 | < | // BUT I OWN THIS MOLECULE!!! | 
| 800 | < |  | 
| 801 | < | localIndex = mpiSim->getGlobalToLocalMol(i); | 
| 802 | < | vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); | 
| 803 | < | nCurObj = vecParticles.size(); | 
| 804 | < |  | 
| 805 | < | MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 806 | < | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 807 | < |  | 
| 808 | < | for(j = 0; j < vecParticles.size(); j++){ | 
| 809 | < |  | 
| 810 | < | MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, | 
| 760 | > | if (worldRank == masterNode) { | 
| 761 | > |  | 
| 762 | > | for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 763 | > | // Get the Node number which has this atom | 
| 764 | > | which_node = info_->getMolToProc(i); | 
| 765 | > |  | 
| 766 | > | // let's let node 0 pass out constant values to all the processors | 
| 767 | > | if (worldRank == masterNode) { | 
| 768 | > | mol = info_->getMoleculeByGlobalIndex(i); | 
| 769 | > |  | 
| 770 | > | if(mol == NULL) { | 
| 771 | > | strcpy(painCave.errMsg, "Molecule not found on node 0!"); | 
| 772 | > | haveError = 1; | 
| 773 | > | simError(); | 
| 774 | > | } | 
| 775 | > |  | 
| 776 | > | for (integrableObject = mol->beginIntegrableObject(ii); | 
| 777 | > | integrableObject != NULL; | 
| 778 | > | integrableObject = mol->nextIntegrableObject(ii)){ | 
| 779 | > |  | 
| 780 | > | integrableObject->setZangle( 0.0 ); | 
| 781 | > |  | 
| 782 | > | } | 
| 783 | > |  | 
| 784 | > | } else { | 
| 785 | > | // I am MASTER OF THE UNIVERSE, but I don't own this molecule | 
| 786 | > |  | 
| 787 | > | MPI_Recv(&nCurObj, 1, MPI_INT, which_node, | 
| 788 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); | 
| 789 | > |  | 
| 790 | > | for(j=0; j < nCurObj; j++){ | 
| 791 | > | angleTransfer = 0.0; | 
| 792 | > | MPI_Send(&angleTransfer, 1, MPI_REALTYPE, which_node, | 
| 793 | > | TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); | 
| 794 | > |  | 
| 795 | > | } | 
| 796 | > | } | 
| 797 | > | } | 
| 798 | > | } else { | 
| 799 | > | // I am SLAVE TO THE MASTER | 
| 800 | > | for (i=0 ; i < info_->getNGlobalMolecules(); i++) { | 
| 801 | > | int which_node = info_->getMolToProc(i); | 
| 802 | > |  | 
| 803 | > | if (which_node == worldRank) { | 
| 804 | > |  | 
| 805 | > | // BUT I OWN THIS MOLECULE!!! | 
| 806 | > | mol = info_->getMoleculeByGlobalIndex(i); | 
| 807 | > |  | 
| 808 | > | if(mol == NULL) { | 
| 809 | > | sprintf(painCave.errMsg, | 
| 810 | > | "RestReader Error: molecule not found on node %d\n", | 
| 811 | > | worldRank); | 
| 812 | > | painCave.isFatal = 1; | 
| 813 | > | simError(); | 
| 814 | > | } | 
| 815 | > |  | 
| 816 | > | nCurObj = mol->getNIntegrableObjects(); | 
| 817 | > |  | 
| 818 | > | MPI_Send(&nCurObj, 1, MPI_INT, 0, | 
| 819 | > | TAKE_THIS_TAG_INT, MPI_COMM_WORLD); | 
| 820 | > |  | 
| 821 | > | for (integrableObject = mol->beginIntegrableObject(ii); | 
| 822 | > | integrableObject != NULL; | 
| 823 | > | integrableObject = mol->nextIntegrableObject(ii)){ | 
| 824 | > |  | 
| 825 | > | MPI_Recv(&angleTransfer, 1, MPI_REALTYPE, 0, | 
| 826 |  | TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); | 
| 827 |  | vecParticles[j]->setZangle(angleTransfer); | 
| 828 |  | } |