| 47 |  | using namespace std; | 
| 48 |  | namespace OpenMD { | 
| 49 |  |  | 
| 50 | + | ForceMatrixDecomposition::ForceMatrixDecomposition(SimInfo* info, InteractionManager* iMan) : ForceDecomposition(info, iMan) { | 
| 51 | + |  | 
| 52 | + | // In a parallel computation, row and colum scans must visit all | 
| 53 | + | // surrounding cells (not just the 14 upper triangular blocks that | 
| 54 | + | // are used when the processor can see all pairs) | 
| 55 | + | #ifdef IS_MPI | 
| 56 | + | cellOffsets_.push_back( Vector3i(-1, 0, 0) ); | 
| 57 | + | cellOffsets_.push_back( Vector3i(-1,-1, 0) ); | 
| 58 | + | cellOffsets_.push_back( Vector3i( 0,-1, 0) ); | 
| 59 | + | cellOffsets_.push_back( Vector3i( 1,-1, 0) ); | 
| 60 | + | cellOffsets_.push_back( Vector3i( 0, 0,-1) ); | 
| 61 | + | cellOffsets_.push_back( Vector3i(-1, 0, 1) ); | 
| 62 | + | cellOffsets_.push_back( Vector3i(-1,-1,-1) ); | 
| 63 | + | cellOffsets_.push_back( Vector3i( 0,-1,-1) ); | 
| 64 | + | cellOffsets_.push_back( Vector3i( 1,-1,-1) ); | 
| 65 | + | cellOffsets_.push_back( Vector3i( 1, 0,-1) ); | 
| 66 | + | cellOffsets_.push_back( Vector3i( 1, 1,-1) ); | 
| 67 | + | cellOffsets_.push_back( Vector3i( 0, 1,-1) ); | 
| 68 | + | cellOffsets_.push_back( Vector3i(-1, 1,-1) ); | 
| 69 | + | #endif | 
| 70 | + | } | 
| 71 | + |  | 
| 72 | + |  | 
| 73 |  | /** | 
| 74 |  | * distributeInitialData is essentially a copy of the older fortran | 
| 75 |  | * SimulationSetup | 
| 76 |  | */ | 
| 54 | – |  | 
| 77 |  | void ForceMatrixDecomposition::distributeInitialData() { | 
| 78 |  | snap_ = sman_->getCurrentSnapshot(); | 
| 79 |  | storageLayout_ = sman_->getStorageLayout(); | 
| 96 |  |  | 
| 97 |  | #ifdef IS_MPI | 
| 98 |  |  | 
| 99 | < | AtomCommIntRow = new Communicator<Row,int>(nLocal_); | 
| 100 | < | AtomCommRealRow = new Communicator<Row,RealType>(nLocal_); | 
| 79 | < | AtomCommVectorRow = new Communicator<Row,Vector3d>(nLocal_); | 
| 80 | < | AtomCommMatrixRow = new Communicator<Row,Mat3x3d>(nLocal_); | 
| 81 | < | AtomCommPotRow = new Communicator<Row,potVec>(nLocal_); | 
| 99 | > | MPI::Intracomm row = rowComm.getComm(); | 
| 100 | > | MPI::Intracomm col = colComm.getComm(); | 
| 101 |  |  | 
| 102 | < | AtomCommIntColumn = new Communicator<Column,int>(nLocal_); | 
| 103 | < | AtomCommRealColumn = new Communicator<Column,RealType>(nLocal_); | 
| 104 | < | AtomCommVectorColumn = new Communicator<Column,Vector3d>(nLocal_); | 
| 105 | < | AtomCommMatrixColumn = new Communicator<Column,Mat3x3d>(nLocal_); | 
| 106 | < | AtomCommPotColumn = new Communicator<Column,potVec>(nLocal_); | 
| 102 | > | AtomPlanIntRow = new Plan<int>(row, nLocal_); | 
| 103 | > | AtomPlanRealRow = new Plan<RealType>(row, nLocal_); | 
| 104 | > | AtomPlanVectorRow = new Plan<Vector3d>(row, nLocal_); | 
| 105 | > | AtomPlanMatrixRow = new Plan<Mat3x3d>(row, nLocal_); | 
| 106 | > | AtomPlanPotRow = new Plan<potVec>(row, nLocal_); | 
| 107 |  |  | 
| 108 | < | cgCommIntRow = new Communicator<Row,int>(nGroups_); | 
| 109 | < | cgCommVectorRow = new Communicator<Row,Vector3d>(nGroups_); | 
| 110 | < | cgCommIntColumn = new Communicator<Column,int>(nGroups_); | 
| 111 | < | cgCommVectorColumn = new Communicator<Column,Vector3d>(nGroups_); | 
| 108 | > | AtomPlanIntColumn = new Plan<int>(col, nLocal_); | 
| 109 | > | AtomPlanRealColumn = new Plan<RealType>(col, nLocal_); | 
| 110 | > | AtomPlanVectorColumn = new Plan<Vector3d>(col, nLocal_); | 
| 111 | > | AtomPlanMatrixColumn = new Plan<Mat3x3d>(col, nLocal_); | 
| 112 | > | AtomPlanPotColumn = new Plan<potVec>(col, nLocal_); | 
| 113 |  |  | 
| 114 | < | nAtomsInRow_ = AtomCommIntRow->getSize(); | 
| 115 | < | nAtomsInCol_ = AtomCommIntColumn->getSize(); | 
| 116 | < | nGroupsInRow_ = cgCommIntRow->getSize(); | 
| 117 | < | nGroupsInCol_ = cgCommIntColumn->getSize(); | 
| 114 | > | cgPlanIntRow = new Plan<int>(row, nGroups_); | 
| 115 | > | cgPlanVectorRow = new Plan<Vector3d>(row, nGroups_); | 
| 116 | > | cgPlanIntColumn = new Plan<int>(col, nGroups_); | 
| 117 | > | cgPlanVectorColumn = new Plan<Vector3d>(col, nGroups_); | 
| 118 |  |  | 
| 119 | + | nAtomsInRow_ = AtomPlanIntRow->getSize(); | 
| 120 | + | nAtomsInCol_ = AtomPlanIntColumn->getSize(); | 
| 121 | + | nGroupsInRow_ = cgPlanIntRow->getSize(); | 
| 122 | + | nGroupsInCol_ = cgPlanIntColumn->getSize(); | 
| 123 | + |  | 
| 124 |  | // Modify the data storage objects with the correct layouts and sizes: | 
| 125 |  | atomRowData.resize(nAtomsInRow_); | 
| 126 |  | atomRowData.setStorageLayout(storageLayout_); | 
| 134 |  | identsRow.resize(nAtomsInRow_); | 
| 135 |  | identsCol.resize(nAtomsInCol_); | 
| 136 |  |  | 
| 137 | < | AtomCommIntRow->gather(idents, identsRow); | 
| 138 | < | AtomCommIntColumn->gather(idents, identsCol); | 
| 137 | > | AtomPlanIntRow->gather(idents, identsRow); | 
| 138 | > | AtomPlanIntColumn->gather(idents, identsCol); | 
| 139 |  |  | 
| 140 |  | // allocate memory for the parallel objects | 
| 141 | + | atypesRow.resize(nAtomsInRow_); | 
| 142 | + | atypesCol.resize(nAtomsInCol_); | 
| 143 | + |  | 
| 144 | + | for (int i = 0; i < nAtomsInRow_; i++) | 
| 145 | + | atypesRow[i] = ff_->getAtomType(identsRow[i]); | 
| 146 | + | for (int i = 0; i < nAtomsInCol_; i++) | 
| 147 | + | atypesCol[i] = ff_->getAtomType(identsCol[i]); | 
| 148 | + |  | 
| 149 | + | pot_row.resize(nAtomsInRow_); | 
| 150 | + | pot_col.resize(nAtomsInCol_); | 
| 151 | + |  | 
| 152 |  | AtomRowToGlobal.resize(nAtomsInRow_); | 
| 153 |  | AtomColToGlobal.resize(nAtomsInCol_); | 
| 154 | + | AtomPlanIntRow->gather(AtomLocalToGlobal, AtomRowToGlobal); | 
| 155 | + | AtomPlanIntColumn->gather(AtomLocalToGlobal, AtomColToGlobal); | 
| 156 | + |  | 
| 157 | + | cerr << "Atoms in Local:\n"; | 
| 158 | + | for (int i = 0; i < AtomLocalToGlobal.size(); i++) { | 
| 159 | + | cerr << "i =\t" << i << "\t localAt =\t" << AtomLocalToGlobal[i] << "\n"; | 
| 160 | + | } | 
| 161 | + | cerr << "Atoms in Row:\n"; | 
| 162 | + | for (int i = 0; i < AtomRowToGlobal.size(); i++) { | 
| 163 | + | cerr << "i =\t" << i << "\t rowAt =\t" << AtomRowToGlobal[i] << "\n"; | 
| 164 | + | } | 
| 165 | + | cerr << "Atoms in Col:\n"; | 
| 166 | + | for (int i = 0; i < AtomColToGlobal.size(); i++) { | 
| 167 | + | cerr << "i =\t" << i << "\t colAt =\t" << AtomColToGlobal[i] << "\n"; | 
| 168 | + | } | 
| 169 | + |  | 
| 170 |  | cgRowToGlobal.resize(nGroupsInRow_); | 
| 171 |  | cgColToGlobal.resize(nGroupsInCol_); | 
| 172 | < | massFactorsRow.resize(nAtomsInRow_); | 
| 173 | < | massFactorsCol.resize(nAtomsInCol_); | 
| 122 | < | pot_row.resize(nAtomsInRow_); | 
| 123 | < | pot_col.resize(nAtomsInCol_); | 
| 172 | > | cgPlanIntRow->gather(cgLocalToGlobal, cgRowToGlobal); | 
| 173 | > | cgPlanIntColumn->gather(cgLocalToGlobal, cgColToGlobal); | 
| 174 |  |  | 
| 175 | < | AtomCommIntRow->gather(AtomLocalToGlobal, AtomRowToGlobal); | 
| 176 | < | AtomCommIntColumn->gather(AtomLocalToGlobal, AtomColToGlobal); | 
| 177 | < |  | 
| 178 | < | cgCommIntRow->gather(cgLocalToGlobal, cgRowToGlobal); | 
| 179 | < | cgCommIntColumn->gather(cgLocalToGlobal, cgColToGlobal); | 
| 175 | > | cerr << "Gruops in Local:\n"; | 
| 176 | > | for (int i = 0; i < cgLocalToGlobal.size(); i++) { | 
| 177 | > | cerr << "i =\t" << i << "\t localCG =\t" << cgLocalToGlobal[i] << "\n"; | 
| 178 | > | } | 
| 179 | > | cerr << "Groups in Row:\n"; | 
| 180 | > | for (int i = 0; i < cgRowToGlobal.size(); i++) { | 
| 181 | > | cerr << "i =\t" << i << "\t rowCG =\t" << cgRowToGlobal[i] << "\n"; | 
| 182 | > | } | 
| 183 | > | cerr << "Groups in Col:\n"; | 
| 184 | > | for (int i = 0; i < cgColToGlobal.size(); i++) { | 
| 185 | > | cerr << "i =\t" << i << "\t colCG =\t" << cgColToGlobal[i] << "\n"; | 
| 186 | > | } | 
| 187 |  |  | 
| 131 | – | AtomCommRealRow->gather(massFactors, massFactorsRow); | 
| 132 | – | AtomCommRealColumn->gather(massFactors, massFactorsCol); | 
| 188 |  |  | 
| 189 | + | massFactorsRow.resize(nAtomsInRow_); | 
| 190 | + | massFactorsCol.resize(nAtomsInCol_); | 
| 191 | + | AtomPlanRealRow->gather(massFactors, massFactorsRow); | 
| 192 | + | AtomPlanRealColumn->gather(massFactors, massFactorsCol); | 
| 193 | + |  | 
| 194 |  | groupListRow_.clear(); | 
| 195 |  | groupListRow_.resize(nGroupsInRow_); | 
| 196 |  | for (int i = 0; i < nGroupsInRow_; i++) { | 
| 247 |  |  | 
| 248 |  | #endif | 
| 249 |  |  | 
| 250 | + | // allocate memory for the parallel objects | 
| 251 | + | atypesLocal.resize(nLocal_); | 
| 252 | + |  | 
| 253 | + | for (int i = 0; i < nLocal_; i++) | 
| 254 | + | atypesLocal[i] = ff_->getAtomType(idents[i]); | 
| 255 | + |  | 
| 256 |  | groupList_.clear(); | 
| 257 |  | groupList_.resize(nGroups_); | 
| 258 |  | for (int i = 0; i < nGroups_; i++) { | 
| 305 |  | void ForceMatrixDecomposition::createGtypeCutoffMap() { | 
| 306 |  |  | 
| 307 |  | RealType tol = 1e-6; | 
| 308 | + | largestRcut_ = 0.0; | 
| 309 |  | RealType rc; | 
| 310 |  | int atid; | 
| 311 |  | set<AtomType*> atypes = info_->getSimulatedAtomTypes(); | 
| 312 | + |  | 
| 313 |  | map<int, RealType> atypeCutoff; | 
| 314 |  |  | 
| 315 |  | for (set<AtomType*>::iterator at = atypes.begin(); | 
| 317 |  | atid = (*at)->getIdent(); | 
| 318 |  | if (userChoseCutoff_) | 
| 319 |  | atypeCutoff[atid] = userCutoff_; | 
| 320 | < | else | 
| 320 | > | else | 
| 321 |  | atypeCutoff[atid] = interactionMan_->getSuggestedCutoffRadius(*at); | 
| 322 |  | } | 
| 323 | < |  | 
| 323 | > |  | 
| 324 |  | vector<RealType> gTypeCutoffs; | 
| 325 |  | // first we do a single loop over the cutoff groups to find the | 
| 326 |  | // largest cutoff for any atypes present in this group. | 
| 380 |  | vector<RealType> groupCutoff(nGroups_, 0.0); | 
| 381 |  | groupToGtype.resize(nGroups_); | 
| 382 |  | for (int cg1 = 0; cg1 < nGroups_; cg1++) { | 
| 315 | – |  | 
| 383 |  | groupCutoff[cg1] = 0.0; | 
| 384 |  | vector<int> atomList = getAtomsInGroupRow(cg1); | 
| 318 | – |  | 
| 385 |  | for (vector<int>::iterator ia = atomList.begin(); | 
| 386 |  | ia != atomList.end(); ++ia) { | 
| 387 |  | int atom1 = (*ia); | 
| 388 |  | atid = idents[atom1]; | 
| 389 | < | if (atypeCutoff[atid] > groupCutoff[cg1]) { | 
| 389 | > | if (atypeCutoff[atid] > groupCutoff[cg1]) | 
| 390 |  | groupCutoff[cg1] = atypeCutoff[atid]; | 
| 325 | – | } | 
| 391 |  | } | 
| 392 | < |  | 
| 392 | > |  | 
| 393 |  | bool gTypeFound = false; | 
| 394 |  | for (int gt = 0; gt < gTypeCutoffs.size(); gt++) { | 
| 395 |  | if (abs(groupCutoff[cg1] - gTypeCutoffs[gt]) < tol) { | 
| 397 |  | gTypeFound = true; | 
| 398 |  | } | 
| 399 |  | } | 
| 400 | < | if (!gTypeFound) { | 
| 400 | > | if (!gTypeFound) { | 
| 401 |  | gTypeCutoffs.push_back( groupCutoff[cg1] ); | 
| 402 |  | groupToGtype[cg1] = gTypeCutoffs.size() - 1; | 
| 403 |  | } | 
| 441 |  |  | 
| 442 |  | pair<int,int> key = make_pair(i,j); | 
| 443 |  | gTypeCutoffMap[key].first = thisRcut; | 
| 379 | – |  | 
| 444 |  | if (thisRcut > largestRcut_) largestRcut_ = thisRcut; | 
| 381 | – |  | 
| 445 |  | gTypeCutoffMap[key].second = thisRcut*thisRcut; | 
| 383 | – |  | 
| 446 |  | gTypeCutoffMap[key].third = pow(thisRcut + skinThickness_, 2); | 
| 385 | – |  | 
| 447 |  | // sanity check | 
| 448 |  |  | 
| 449 |  | if (userChoseCutoff_) { | 
| 569 |  | #ifdef IS_MPI | 
| 570 |  |  | 
| 571 |  | // gather up the atomic positions | 
| 572 | < | AtomCommVectorRow->gather(snap_->atomData.position, | 
| 572 | > | AtomPlanVectorRow->gather(snap_->atomData.position, | 
| 573 |  | atomRowData.position); | 
| 574 | < | AtomCommVectorColumn->gather(snap_->atomData.position, | 
| 574 | > | AtomPlanVectorColumn->gather(snap_->atomData.position, | 
| 575 |  | atomColData.position); | 
| 576 |  |  | 
| 577 |  | // gather up the cutoff group positions | 
| 578 | < | cgCommVectorRow->gather(snap_->cgData.position, | 
| 578 | > |  | 
| 579 | > | cerr  << "before gather\n"; | 
| 580 | > | for (int i = 0; i < snap_->cgData.position.size(); i++) { | 
| 581 | > | cerr << "cgpos = " << snap_->cgData.position[i] << "\n"; | 
| 582 | > | } | 
| 583 | > |  | 
| 584 | > | cgPlanVectorRow->gather(snap_->cgData.position, | 
| 585 |  | cgRowData.position); | 
| 586 | < | cgCommVectorColumn->gather(snap_->cgData.position, | 
| 586 | > |  | 
| 587 | > | cerr  << "after gather\n"; | 
| 588 | > | for (int i = 0; i < cgRowData.position.size(); i++) { | 
| 589 | > | cerr << "cgRpos = " << cgRowData.position[i] << "\n"; | 
| 590 | > | } | 
| 591 | > |  | 
| 592 | > | cgPlanVectorColumn->gather(snap_->cgData.position, | 
| 593 |  | cgColData.position); | 
| 594 | + | for (int i = 0; i < cgColData.position.size(); i++) { | 
| 595 | + | cerr << "cgCpos = " << cgColData.position[i] << "\n"; | 
| 596 | + | } | 
| 597 | + |  | 
| 598 |  |  | 
| 599 |  | // if needed, gather the atomic rotation matrices | 
| 600 |  | if (storageLayout_ & DataStorage::dslAmat) { | 
| 601 | < | AtomCommMatrixRow->gather(snap_->atomData.aMat, | 
| 601 | > | AtomPlanMatrixRow->gather(snap_->atomData.aMat, | 
| 602 |  | atomRowData.aMat); | 
| 603 | < | AtomCommMatrixColumn->gather(snap_->atomData.aMat, | 
| 603 | > | AtomPlanMatrixColumn->gather(snap_->atomData.aMat, | 
| 604 |  | atomColData.aMat); | 
| 605 |  | } | 
| 606 |  |  | 
| 607 |  | // if needed, gather the atomic eletrostatic frames | 
| 608 |  | if (storageLayout_ & DataStorage::dslElectroFrame) { | 
| 609 | < | AtomCommMatrixRow->gather(snap_->atomData.electroFrame, | 
| 609 | > | AtomPlanMatrixRow->gather(snap_->atomData.electroFrame, | 
| 610 |  | atomRowData.electroFrame); | 
| 611 | < | AtomCommMatrixColumn->gather(snap_->atomData.electroFrame, | 
| 611 | > | AtomPlanMatrixColumn->gather(snap_->atomData.electroFrame, | 
| 612 |  | atomColData.electroFrame); | 
| 613 |  | } | 
| 614 |  |  | 
| 625 |  |  | 
| 626 |  | if (storageLayout_ & DataStorage::dslDensity) { | 
| 627 |  |  | 
| 628 | < | AtomCommRealRow->scatter(atomRowData.density, | 
| 628 | > | AtomPlanRealRow->scatter(atomRowData.density, | 
| 629 |  | snap_->atomData.density); | 
| 630 |  |  | 
| 631 |  | int n = snap_->atomData.density.size(); | 
| 632 |  | vector<RealType> rho_tmp(n, 0.0); | 
| 633 | < | AtomCommRealColumn->scatter(atomColData.density, rho_tmp); | 
| 633 | > | AtomPlanRealColumn->scatter(atomColData.density, rho_tmp); | 
| 634 |  | for (int i = 0; i < n; i++) | 
| 635 |  | snap_->atomData.density[i] += rho_tmp[i]; | 
| 636 |  | } | 
| 646 |  | storageLayout_ = sman_->getStorageLayout(); | 
| 647 |  | #ifdef IS_MPI | 
| 648 |  | if (storageLayout_ & DataStorage::dslFunctional) { | 
| 649 | < | AtomCommRealRow->gather(snap_->atomData.functional, | 
| 649 | > | AtomPlanRealRow->gather(snap_->atomData.functional, | 
| 650 |  | atomRowData.functional); | 
| 651 | < | AtomCommRealColumn->gather(snap_->atomData.functional, | 
| 651 | > | AtomPlanRealColumn->gather(snap_->atomData.functional, | 
| 652 |  | atomColData.functional); | 
| 653 |  | } | 
| 654 |  |  | 
| 655 |  | if (storageLayout_ & DataStorage::dslFunctionalDerivative) { | 
| 656 | < | AtomCommRealRow->gather(snap_->atomData.functionalDerivative, | 
| 656 | > | AtomPlanRealRow->gather(snap_->atomData.functionalDerivative, | 
| 657 |  | atomRowData.functionalDerivative); | 
| 658 | < | AtomCommRealColumn->gather(snap_->atomData.functionalDerivative, | 
| 658 | > | AtomPlanRealColumn->gather(snap_->atomData.functionalDerivative, | 
| 659 |  | atomColData.functionalDerivative); | 
| 660 |  | } | 
| 661 |  | #endif | 
| 669 |  | int n = snap_->atomData.force.size(); | 
| 670 |  | vector<Vector3d> frc_tmp(n, V3Zero); | 
| 671 |  |  | 
| 672 | < | AtomCommVectorRow->scatter(atomRowData.force, frc_tmp); | 
| 672 | > | AtomPlanVectorRow->scatter(atomRowData.force, frc_tmp); | 
| 673 |  | for (int i = 0; i < n; i++) { | 
| 674 |  | snap_->atomData.force[i] += frc_tmp[i]; | 
| 675 |  | frc_tmp[i] = 0.0; | 
| 676 |  | } | 
| 677 |  |  | 
| 678 | < | AtomCommVectorColumn->scatter(atomColData.force, frc_tmp); | 
| 679 | < | for (int i = 0; i < n; i++) | 
| 678 | > | AtomPlanVectorColumn->scatter(atomColData.force, frc_tmp); | 
| 679 | > | for (int i = 0; i < n; i++) { | 
| 680 |  | snap_->atomData.force[i] += frc_tmp[i]; | 
| 681 | + | } | 
| 682 |  |  | 
| 683 |  | if (storageLayout_ & DataStorage::dslTorque) { | 
| 684 |  |  | 
| 685 |  | int nt = snap_->atomData.torque.size(); | 
| 686 |  | vector<Vector3d> trq_tmp(nt, V3Zero); | 
| 687 |  |  | 
| 688 | < | AtomCommVectorRow->scatter(atomRowData.torque, trq_tmp); | 
| 688 | > | AtomPlanVectorRow->scatter(atomRowData.torque, trq_tmp); | 
| 689 |  | for (int i = 0; i < nt; i++) { | 
| 690 |  | snap_->atomData.torque[i] += trq_tmp[i]; | 
| 691 |  | trq_tmp[i] = 0.0; | 
| 692 |  | } | 
| 693 |  |  | 
| 694 | < | AtomCommVectorColumn->scatter(atomColData.torque, trq_tmp); | 
| 694 | > | AtomPlanVectorColumn->scatter(atomColData.torque, trq_tmp); | 
| 695 |  | for (int i = 0; i < nt; i++) | 
| 696 |  | snap_->atomData.torque[i] += trq_tmp[i]; | 
| 697 |  | } | 
| 701 |  | int ns = snap_->atomData.skippedCharge.size(); | 
| 702 |  | vector<RealType> skch_tmp(ns, 0.0); | 
| 703 |  |  | 
| 704 | < | AtomCommRealRow->scatter(atomRowData.skippedCharge, skch_tmp); | 
| 704 | > | AtomPlanRealRow->scatter(atomRowData.skippedCharge, skch_tmp); | 
| 705 |  | for (int i = 0; i < ns; i++) { | 
| 706 |  | snap_->atomData.skippedCharge[i] += skch_tmp[i]; | 
| 707 |  | skch_tmp[i] = 0.0; | 
| 708 |  | } | 
| 709 |  |  | 
| 710 | < | AtomCommRealColumn->scatter(atomColData.skippedCharge, skch_tmp); | 
| 710 | > | AtomPlanRealColumn->scatter(atomColData.skippedCharge, skch_tmp); | 
| 711 |  | for (int i = 0; i < ns; i++) | 
| 712 |  | snap_->atomData.skippedCharge[i] += skch_tmp[i]; | 
| 713 |  | } | 
| 719 |  |  | 
| 720 |  | // scatter/gather pot_row into the members of my column | 
| 721 |  |  | 
| 722 | < | AtomCommPotRow->scatter(pot_row, pot_temp); | 
| 722 | > | AtomPlanPotRow->scatter(pot_row, pot_temp); | 
| 723 |  |  | 
| 724 |  | for (int ii = 0;  ii < pot_temp.size(); ii++ ) | 
| 725 |  | pairwisePot += pot_temp[ii]; | 
| 727 |  | fill(pot_temp.begin(), pot_temp.end(), | 
| 728 |  | Vector<RealType, N_INTERACTION_FAMILIES> (0.0)); | 
| 729 |  |  | 
| 730 | < | AtomCommPotColumn->scatter(pot_col, pot_temp); | 
| 730 | > | AtomPlanPotColumn->scatter(pot_col, pot_temp); | 
| 731 |  |  | 
| 732 |  | for (int ii = 0;  ii < pot_temp.size(); ii++ ) | 
| 733 |  | pairwisePot += pot_temp[ii]; | 
| 734 |  | #endif | 
| 735 |  |  | 
| 736 | + | cerr << "pairwisePot = " <<  pairwisePot << "\n"; | 
| 737 |  | } | 
| 738 |  |  | 
| 739 |  | int ForceMatrixDecomposition::getNAtomsInRow() { | 
| 768 |  |  | 
| 769 |  | #ifdef IS_MPI | 
| 770 |  | d = cgColData.position[cg2] - cgRowData.position[cg1]; | 
| 771 | + | cerr << "cg1 = " << cg1 << "\tcg1p = " << cgRowData.position[cg1] << "\n"; | 
| 772 | + | cerr << "cg2 = " << cg2 << "\tcg2p = " << cgColData.position[cg2] << "\n"; | 
| 773 |  | #else | 
| 774 |  | d = snap_->cgData.position[cg2] - snap_->cgData.position[cg1]; | 
| 775 | + | cerr << "cg1 = " << cg1 << "\tcg1p = " << snap_->cgData.position[cg1] << "\n"; | 
| 776 | + | cerr << "cg2 = " << cg2 << "\tcg2p = " << snap_->cgData.position[cg2] << "\n"; | 
| 777 |  | #endif | 
| 778 |  |  | 
| 779 |  | snap_->wrapVector(d); | 
| 848 |  | */ | 
| 849 |  | bool ForceMatrixDecomposition::skipAtomPair(int atom1, int atom2) { | 
| 850 |  | int unique_id_1, unique_id_2; | 
| 851 | + |  | 
| 852 |  |  | 
| 853 | + | cerr << "sap with atom1, atom2 =\t" << atom1 << "\t" << atom2 << "\n"; | 
| 854 |  | #ifdef IS_MPI | 
| 855 |  | // in MPI, we have to look up the unique IDs for each atom | 
| 856 |  | unique_id_1 = AtomRowToGlobal[atom1]; | 
| 857 |  | unique_id_2 = AtomColToGlobal[atom2]; | 
| 858 |  |  | 
| 859 | + | cerr << "sap with uid1, uid2 =\t" << unique_id_1 << "\t" << unique_id_2 << "\n"; | 
| 860 |  | // this situation should only arise in MPI simulations | 
| 861 |  | if (unique_id_1 == unique_id_2) return true; | 
| 862 |  |  | 
| 881 |  | */ | 
| 882 |  | bool ForceMatrixDecomposition::excludeAtomPair(int atom1, int atom2) { | 
| 883 |  | int unique_id_2; | 
| 798 | – |  | 
| 884 |  | #ifdef IS_MPI | 
| 885 |  | // in MPI, we have to look up the unique IDs for the row atom. | 
| 886 |  | unique_id_2 = AtomColToGlobal[atom2]; | 
| 921 |  | idat.excluded = excludeAtomPair(atom1, atom2); | 
| 922 |  |  | 
| 923 |  | #ifdef IS_MPI | 
| 924 | < |  | 
| 925 | < | idat.atypes = make_pair( ff_->getAtomType(identsRow[atom1]), | 
| 926 | < | ff_->getAtomType(identsCol[atom2]) ); | 
| 924 | > | idat.atypes = make_pair( atypesRow[atom1], atypesCol[atom2]); | 
| 925 | > | //idat.atypes = make_pair( ff_->getAtomType(identsRow[atom1]), | 
| 926 | > | //                         ff_->getAtomType(identsCol[atom2]) ); | 
| 927 |  |  | 
| 928 |  | if (storageLayout_ & DataStorage::dslAmat) { | 
| 929 |  | idat.A1 = &(atomRowData.aMat[atom1]); | 
| 967 |  |  | 
| 968 |  | #else | 
| 969 |  |  | 
| 970 | < | idat.atypes = make_pair( ff_->getAtomType(idents[atom1]), | 
| 971 | < | ff_->getAtomType(idents[atom2]) ); | 
| 970 | > | idat.atypes = make_pair( atypesLocal[atom1], atypesLocal[atom2]); | 
| 971 | > | //idat.atypes = make_pair( ff_->getAtomType(idents[atom1]), | 
| 972 | > | //                         ff_->getAtomType(idents[atom2]) ); | 
| 973 |  |  | 
| 974 |  | if (storageLayout_ & DataStorage::dslAmat) { | 
| 975 |  | idat.A1 = &(snap_->atomData.aMat[atom1]); | 
| 1107 |  | // add this cutoff group to the list of groups in this cell; | 
| 1108 |  | cellListRow_[cellIndex].push_back(i); | 
| 1109 |  | } | 
| 1024 | – |  | 
| 1110 |  | for (int i = 0; i < nGroupsInCol_; i++) { | 
| 1111 |  | rs = cgColData.position[i]; | 
| 1112 |  |  | 
| 1151 |  | whichCell.z() = nCells_.z() * scaled.z(); | 
| 1152 |  |  | 
| 1153 |  | // find single index of this cell: | 
| 1154 | < | cellIndex = Vlinear(whichCell, nCells_); | 
| 1154 | > | cellIndex = Vlinear(whichCell, nCells_); | 
| 1155 |  |  | 
| 1156 |  | // add this cutoff group to the list of groups in this cell; | 
| 1157 |  | cellList_[cellIndex].push_back(i); | 
| 1195 |  | for (vector<int>::iterator j2 = cellListCol_[m2].begin(); | 
| 1196 |  | j2 != cellListCol_[m2].end(); ++j2) { | 
| 1197 |  |  | 
| 1198 | < | // Always do this if we're in different cells or if | 
| 1199 | < | // we're in the same cell and the global index of the | 
| 1200 | < | // j2 cutoff group is less than the j1 cutoff group | 
| 1201 | < |  | 
| 1202 | < | if (m2 != m1 || cgColToGlobal[(*j2)] < cgRowToGlobal[(*j1)]) { | 
| 1203 | < | dr = cgColData.position[(*j2)] - cgRowData.position[(*j1)]; | 
| 1204 | < | snap_->wrapVector(dr); | 
| 1205 | < | cuts = getGroupCutoffs( (*j1), (*j2) ); | 
| 1121 | < | if (dr.lengthSquare() < cuts.third) { | 
| 1122 | < | neighborList.push_back(make_pair((*j1), (*j2))); | 
| 1123 | < | } | 
| 1124 | < | } | 
| 1198 | > | // In parallel, we need to visit *all* pairs of row & | 
| 1199 | > | // column indicies and will truncate later on. | 
| 1200 | > | dr = cgColData.position[(*j2)] - cgRowData.position[(*j1)]; | 
| 1201 | > | snap_->wrapVector(dr); | 
| 1202 | > | cuts = getGroupCutoffs( (*j1), (*j2) ); | 
| 1203 | > | if (dr.lengthSquare() < cuts.third) { | 
| 1204 | > | neighborList.push_back(make_pair((*j1), (*j2))); | 
| 1205 | > | } | 
| 1206 |  | } | 
| 1207 |  | } | 
| 1208 |  | #else |