--- trunk/OOPSE/libmdtools/mpiSimulation_module.F90 2003/12/17 20:13:33 882 +++ trunk/OOPSE/libmdtools/mpiSimulation_module.F90 2004/05/07 21:35:05 1150 @@ -7,7 +7,7 @@ !! !! @author Charles F. Vardeman II !! @author Matthew Meineke -!! @version $Id: mpiSimulation_module.F90,v 1.10 2003-12-17 20:13:33 chuckv Exp $, $Date: 2003-12-17 20:13:33 $, $Name: not supported by cvs2svn $, $Revision: 1.10 $ +!! @version $Id: mpiSimulation_module.F90,v 1.12 2004-05-07 21:35:04 gezelter Exp $, $Date: 2004-05-07 21:35:04 $, $Name: not supported by cvs2svn $, $Revision: 1.12 $ module mpiSimulation use definitions @@ -25,6 +25,8 @@ module mpiSimulation public :: replanSimParallel public :: getNcol public :: getNrow + public :: getNcolGroup + public :: getNrowGroup public :: isMPISimSet public :: printComponentPlan public :: getMyNode @@ -60,9 +62,9 @@ module mpiSimulation #ifdef PROFILE public :: printCommTime - - real(kind = dp ), save :: commTime = 0.0_dp - real(kind = dp ) :: commTimeInitial,commTimeFinal + public :: getCommTime + real,save :: commTime = 0.0 + real :: commTimeInitial,commTimeFinal #endif !! Include mpiComponentPlan type. mpiComponentPlan is a @@ -71,7 +73,6 @@ module mpiSimulation #include "mpiComponentPlan.h" - !! Tags used during force loop for parallel simulation integer, public, allocatable, dimension(:) :: tagLocal integer, public, allocatable, dimension(:) :: tagRow @@ -103,6 +104,10 @@ module mpiSimulation type (gs_plan), public, save :: plan_col3d type (gs_plan), public, save :: plan_row_Rotation type (gs_plan), public, save :: plan_col_Rotation + type (gs_plan), public, save :: plan_row_Group + type (gs_plan), public, save :: plan_col_Group + type (gs_plan), public, save :: plan_row_Group_3d + type (gs_plan), public, save :: plan_col_Group_3d type (mpiComponentPlan), pointer :: simComponentPlan @@ -147,8 +152,6 @@ contains write(*,*) 'mpiSim_mod thinks node', thisComponentPlan%myNode, ' has tags(1) = ', tags(1) - - status = 0 if (componentPlanSet) then return @@ -181,15 +184,22 @@ contains mpiSim,mpiSim%rowComm,plan_row3d) call plan_gather_scatter(9,mpiSim%myNlocal,& mpiSim,mpiSim%rowComm,plan_row_Rotation) + call plan_gather_scatter(1,mpiSim%myNgroup,& + mpiSim,mpiSim%rowComm,plan_row_Group) + call plan_gather_scatter(nDim,mpiSim%myNgroup,& + mpiSim,mpiSim%rowComm,plan_row_Group_3d) + call plan_gather_scatter(1,mpiSim%myNlocal,& mpiSim,mpiSim%columnComm,plan_col) call plan_gather_scatter(nDim,mpiSim%myNlocal,& mpiSim,mpiSim%columnComm,plan_col3d) - call plan_gather_scatter(9,mpiSim%myNlocal,& + call plan_gather_scatter(9,mpiSim%myNlocal,& mpiSim,mpiSim%columnComm,plan_col_Rotation) - - - + call plan_gather_scatter(1,mpiSim%myNgroup,& + mpiSim,mpiSim%columnComm,plan_col_Group) + call plan_gather_scatter(nDim,mpiSim%myNgroup,& + mpiSim,mpiSim%columnComm,plan_col_Group_3d) + ! Initialize tags call setTags(tags,localStatus) if (localStatus /= 0) then @@ -220,9 +230,14 @@ contains call unplan_gather_scatter(plan_row) call unplan_gather_scatter(plan_row3d) call unplan_gather_scatter(plan_row_Rotation) + call unplan_gather_scatter(plan_row_Group) + call unplan_gather_scatter(plan_row_Group_3d) + call unplan_gather_scatter(plan_col) call unplan_gather_scatter(plan_col3d) call unplan_gather_scatter(plan_col_Rotation) + call unplan_gather_scatter(plan_col_Group) + call unplan_gather_scatter(plan_col_Group_3d) !! initialize gather and scatter plans used in this simulation call plan_gather_scatter(1,thisComponentPlan%myNlocal,& @@ -231,14 +246,21 @@ contains thisComponentPlan,thisComponentPlan%rowComm,plan_row3d) call plan_gather_scatter(9,thisComponentPlan%myNlocal,& thisComponentPlan,thisComponentPlan%rowComm,plan_row_Rotation) + call plan_gather_scatter(1,thisComponentPlan%myNgroup,& + thisComponentPlan,thisComponentPlan%rowComm,plan_row_Group) + call plan_gather_scatter(nDim,thisComponentPlan%myNgroup,& + thisComponentPlan,thisComponentPlan%rowComm,plan_row_Group_3d) + call plan_gather_scatter(1,thisComponentPlan%myNlocal,& thisComponentPlan,thisComponentPlan%columnComm,plan_col) call plan_gather_scatter(nDim,thisComponentPlan%myNlocal,& - thisComponentPlan,thisComponentPlan%rowComm,plan_col3d) + thisComponentPlan,thisComponentPlan%columnComm,plan_col3d) call plan_gather_scatter(9,thisComponentPlan%myNlocal,& - thisComponentPlan,thisComponentPlan%rowComm,plan_col_Rotation) - - + thisComponentPlan,thisComponentPlan%columnComm,plan_col_Rotation) + call plan_gather_scatter(1,thisComponentPlan%myNgroup,& + thisComponentPlan,thisComponentPlan%columnComm,plan_col_Group) + call plan_gather_scatter(nDim,thisComponentPlan%myNgroup,& + thisComponentPlan,thisComponentPlan%columnComm,plan_col_Group_3d) end subroutine replanSimParallel @@ -512,13 +534,13 @@ contains if (present(status)) status = 0 noffset = this_plan%displs(this_plan%myPlanRank) #ifdef PROFILE - commTimeInitial = mpi_wtime() + call cpu_time(commTimeInitial) #endif call mpi_allgatherv(sbuffer,this_plan%gsPlanSize, mpi_double_precision, & rbuffer,this_plan%counts,this_plan%displs,mpi_double_precision, & this_plan%myPlanComm, mpi_err) #ifdef PROFILE - commTimeFinal = mpi_wtime() + call cpu_time(commTimeFinal) commTime = commTime + commTimeFinal - commTimeInitial #endif @@ -542,7 +564,7 @@ contains ! noffset = this_plan%displs(this_plan%me) #ifdef PROFILE - commTimeInitial = mpi_wtime() + call cpu_time(commTimeInitial) #endif call mpi_allgatherv(sbuffer,this_plan%gsPlanSize, mpi_double_precision, & @@ -550,7 +572,7 @@ contains this_plan%myPlanComm, mpi_err) #ifdef PROFILE - commTimeFinal = mpi_wtime() + call cpu_time(commTimeFinal) commTime = commTime + commTimeFinal - commTimeInitial #endif @@ -571,12 +593,12 @@ contains if (present(status)) status = 0 #ifdef PROFILE - commTimeInitial = mpi_wtime() + call cpu_time(commTimeInitial) #endif call mpi_reduce_scatter(sbuffer,rbuffer, this_plan%counts, & mpi_double_precision, MPI_SUM, this_plan%myPlanComm, mpi_err) #ifdef PROFILE - commTimeFinal = mpi_wtime() + call cpu_time(commTimeFinal) commTime = commTime + commTimeFinal - commTimeInitial #endif @@ -596,13 +618,13 @@ contains if (present(status)) status = 0 #ifdef PROFILE - commTimeInitial = mpi_wtime() + call cpu_time(commTimeInitial) #endif call mpi_reduce_scatter(sbuffer,rbuffer, this_plan%counts, & mpi_double_precision, MPI_SUM, this_plan%myPlanComm, mpi_err) #ifdef PROFILE - commTimeFinal = mpi_wtime() + call cpu_time(commTimeFinal) commTime = commTime + commTimeFinal - commTimeInitial #endif @@ -697,6 +719,18 @@ contains nrow = thisplan%gsComponentPlan%nComponentsRow end function getNrow + function getNcolGroup(thisplan) result(ncol_group) + type (gs_plan), intent(in) :: thisplan + integer :: ncol_group + ncol_group = thisplan%gsComponentPlan%nGroupColumn + end function getNcolGroup + + function getNrowGroup(thisplan) result(nrow_group) + type (gs_plan), intent(in) :: thisplan + integer :: nrow_group + nrow_group = thisplan%gsComponentPlan%nGroupRow + end function getNrowGroup + function isMPISimSet() result(isthisSimSet) logical :: isthisSimSet if (isSimSet) then @@ -749,10 +783,14 @@ contains #ifdef PROFILE subroutine printCommTime() - write(*,*) "MPI communication time is: ", commTime - end subroutine printCommTime + + function getCommTime() result(comm_time) + real :: comm_time + comm_time = commTime + end function getCommTime + #endif #endif // is_mpi