--- trunk/OOPSE/libmdtools/mdProfile.cpp	2003/12/18 21:47:51	884
+++ trunk/OOPSE/libmdtools/mdProfile.cpp	2003/12/19 20:36:35	891
@@ -1,10 +1,23 @@
 #include <sys/time.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
 
+#ifdef IS_MPI
+#include <mpi.h>
+
+#include "mpiSimulation.hpp"
+#endif //is_mpi
+
+#include "config.h"
+#include "simError.h"
 #include "mdProfile.hpp"
 
 namespace mdProfileSpace {
 
   class ProfileString{
+  public:
     char myName[MAX_PROFILE_NAMELENGTH];
   };
 
@@ -18,10 +31,17 @@ namespace mdProfileSpace {
 #ifdef IS_MPI
   double globalTime[N_PROFILES];
 #endif //is_mpi
+
   
 }
 
+extern "C"{
+  
+  void F90_FUNC(gettimes, GETTIMES)(double* forceTime, 
+				    double* commTime);
+}
 
+
 using namespace mdProfileSpace;
 
 
@@ -38,9 +58,9 @@ void initProfile( void ){
 #endif //is_mpi
   }
 
-  strncpy( theNames[pro1].myName, "unamed1", MAX_PROFILE_NAMELENGTH );
-  strncpy( theNames[pro2].myName, "unamed2", MAX_PROFILE_NAMELENGTH );
-  strncpy( theNames[pro3].myName, "unamed3", MAX_PROFILE_NAMELENGTH );
+  strncpy( theNames[pro1].myName, "BASS Parse", MAX_PROFILE_NAMELENGTH );
+  strncpy( theNames[pro2].myName, "SimSetup->CreateSim", MAX_PROFILE_NAMELENGTH );
+  strncpy( theNames[pro3].myName, "Integrate the Sim", MAX_PROFILE_NAMELENGTH );
   strncpy( theNames[pro4].myName, "unamed4", MAX_PROFILE_NAMELENGTH );
   strncpy( theNames[pro5].myName, "unamed5", MAX_PROFILE_NAMELENGTH );
   strncpy( theNames[pro6].myName, "unamed6", MAX_PROFILE_NAMELENGTH );
@@ -62,10 +82,10 @@ void endProfile( proNames theProfile ){
   gettimeofday( &endTime[theProfile], &tz );
 
   startVal = (double)startTime[theProfile].tv_sec 
-    + (double)startTime[theProfile].tv_usec / 1000.0;
+    + (double)startTime[theProfile].tv_usec / 1000000.0;
 
   endVal = (double)endTime[theProfile].tv_sec 
-    + (double)endTime[theProfile].tv_usec / 1000.0;
+    + (double)endTime[theProfile].tv_usec / 1000000.0;
   
   accumTime[theProfile] += endVal - startVal;
 }
@@ -74,16 +94,33 @@ void writeProfiles( void ){
 void writeProfiles( void ){
  
   int i;
-  double totTime;
+  double totalTime;
   double percentTime[N_PROFILES];
-  int days, hours, minutes, secs, usecs;
+  int days, hours, minutes, secs, msecs;
   double donkey;
   
+  double forceTime, commTime;
+  
+#ifdef IS_MPI
+  int j;
+
+  MPI_Status istatus;    
+
+  double nodeTime, nodeForceTime, nodeCommTime;
+  double nodeAccum[N_PROFILES];
+  double nodePercent[N_PROFILES];
+
+  double globalTime, globalForceTime, globalCommTime;
+  double globalAccum[N_PROFILES];
+  double globalPercent[N_PROFILES];
+#endif // is_mpi
+
+
 #ifndef IS_MPI // single processor version 
 
-  totTime = 0.0;
+  totalTime = 0.0;
   for(i=0;i<N_PROFILES;i++)
-    totTime += accumTime[i];
+    totalTime += accumTime[i];
 
   for(i=0;i<N_PROFILES;i++)
     percentTime[i] = accumTime[i] / totalTime;
@@ -111,20 +148,245 @@ void writeProfiles( void ){
   donkey -= minutes * 60;
 
   secs = (int)donkey;
-  usecs = (int)( (donkey - secs) * 1000 );
+  msecs = (int)( (donkey - secs) * 1000 );
 
+  F90_FUNC(gettimes, GETTIMES)(&forceTime, &commTime);
+
   fprintf( stdout,
 	   "----------------------------------------------------------------------------\n"
-	   "  Total Time = %03d:%02d:%02d:%02d.%03d ( %G sec )\n",
+	   "  Total Time = %03d:%02d:%02d:%02d.%03d ( %G sec )\n"
+	   "\n"
+	   "  From Fortran: forceTime = %G secs; communicationTime = %G secs.\n",
 	   days,
 	   hours,
 	   minutes,
 	   secs,
-	   usecs,
-	   totalTime );
+	   msecs,
+	   totalTime,
+	   forceTime,
+	   commTime);
 
 #else // the parrallel version
+
+  if( worldRank == 0 ){
+    
+    double *nodeTots = new double[mpiSim->getNumberProcessors()];
+    double *nodePercentTots = new double[mpiSim->getNumberProcessors()];
+    
+    totalTime = 0.0;
+    for(i=0;i<N_PROFILES;i++)
+      totalTime += accumTime[i];
+    
+    for(i=0;i<N_PROFILES;i++)
+      percentTime[i] = accumTime[i] / totalTime;
+    
+    fprintf(stdout,
+	    "\n"
+	    "----------------------------------------------------------------------------\n"
+	    "  Output from Node %d:   \n"
+	    "\n"
+	    "  Time Spent      Percent Time                        Name\n"
+	    "--------------  ----------------   -----------------------------------------\n",
+	    worldRank);
+    
+    for(i=0;i<N_PROFILES;i++){
+      fprintf(stdout,
+	      " %12G    %14G     %40s\n",
+	      accumTime[i],
+	      percentTime[i],
+	      theNames[i].myName );
+    }
+    
+    days = (int)floor( totalTime / 86400 );
+    donkey = totalTime - 86400 * days;
+    
+    hours = (int)floor( donkey / 3600 );
+    donkey -= hours * 3600;
+    
+    minutes = (int)floor( donkey / 60 );
+    donkey -= minutes * 60;
+    
+    secs = (int)donkey;
+    msecs = (int)( (donkey - secs) * 1000 );
+    
+    F90_FUNC(gettimes, GETTIMES)(&forceTime, &commTime);
+
+    fprintf( stdout,
+	     "----------------------------------------------------------------------------\n"
+	     "  Total Time = %03d:%02d:%02d:%02d.%03d ( %G sec )\n"
+	     "\n"
+	     "  From Fortran: forceTime = %G secs; communicationTime = %G secs.\n",
+	     days,
+	     hours,
+	     minutes,
+	     secs,
+	     msecs,
+	     totalTime,
+	     forceTime,
+	     commTime);
+    
+    // now the rest of the nodes
+    
+    nodeTots[0] = totalTime;
+    
+    globalTime = totalTime;
+    globalForceTime = forceTime;
+    globalCommTime = commTime;
+    for(i=0;i<N_PROFILES;i++)
+      globalAccum[i] = accumTime[i];
+    
+    
+    for(j=1;j<mpiSim->getNumberProcessors();j++){
+      
+      nodeTime = 0.0;
+     
+      MPI_Recv(nodeAccum, N_PROFILES, MPI_DOUBLE, j,
+	       1, MPI_COMM_WORLD, &istatus );
+
+      MPI_Recv(&nodeForceTime, 1, MPI_DOUBLE, j,
+	       1, MPI_COMM_WORLD, &istatus );
+      MPI_Recv(&nodeCommTime, 1, MPI_DOUBLE, j,
+	       1, MPI_COMM_WORLD, &istatus );
 
+      for(i=0;i<N_PROFILES;i++){
+	nodeTime += nodeAccum[i];
+      }
+      
+      for(i=0;i<N_PROFILES;i++)
+	nodePercent[i] = nodeAccum[i] / nodeTime;
+      
+      fprintf(stdout,
+	      "\n"
+	      "----------------------------------------------------------------------------\n"
+	      "  Output from Node %d:   \n"
+	      "\n"
+	      "  Time Spent      Percent Time                        Name\n"
+	      "--------------  ----------------   -----------------------------------------\n",
+	      j);
+      
+      for(i=0;i<N_PROFILES;i++){
+	fprintf(stdout,
+		" %12G    %14G     %40s\n",
+		nodeAccum[i],
+		nodePercent[i],
+		theNames[i].myName );
+      }
+      
+      days = (int)floor( nodeTime / 86400 );
+      donkey = nodeTime - 86400 * days;
+      
+      hours = (int)floor( donkey / 3600 );
+      donkey -= hours * 3600;
+      
+      minutes = (int)floor( donkey / 60 );
+      donkey -= minutes * 60;
+      
+      secs = (int)donkey;
+      msecs = (int)( (donkey - secs) * 1000 );
+      
+      fprintf( stdout,
+	       "----------------------------------------------------------------------------\n"
+	       "  Total Time = %03d:%02d:%02d:%02d.%03d ( %G sec )\n"
+	       "\n"
+	       "  From Fortran: forceTime = %G secs; communicationTime = %G secs.\n",
+	       days,
+	       hours,
+	       minutes,
+	       secs,
+	       msecs,
+	       nodeTime,
+	       nodeForceTime,
+	       nodeCommTime);
+      
+      for(i=0;i<N_PROFILES;i++)
+	globalAccum[i] += nodeAccum[i];
+      
+      globalTime += nodeTime;
+      globalForceTime += nodeForceTime;
+      globalCommTime += nodeCommTime;
+      nodeTots[j] = nodeTime;
+    }
+
+    // print out the totals
+    
+    for(j=0;j<mpiSim->getNumberProcessors();j++)
+      nodePercentTots[j] = nodeTots[j] / globalTime;
+    
+    for(i=0;i<N_PROFILES;i++)
+      globalPercent[i] = globalAccum[i] / globalTime;
+
+    fprintf(stdout,
+	    "\n"
+	    "----------------------------------------------------------------------------\n"
+	    "  Total Across Nodes\n"
+	    "\n"
+	    "  Time Spent      Percent Time                        Name\n"
+	    "--------------  ----------------   -----------------------------------------\n",
+	    j);
+    
+    for(i=0;i<N_PROFILES;i++){
+      fprintf(stdout,
+	      " %12G    %14G     %40s\n",
+	      globalAccum[i],
+	      globalPercent[i],
+	      theNames[i].myName );
+    }
+    fprintf(stdout,
+	    "\n"
+	    "\n" );
+    
+    for(j=0;j<mpiSim->getNumberProcessors();j++){
+      
+      fprintf(stdout,
+	      " %12G    %14G     node %d\n",
+	      nodeTots[j],
+	      nodePercentTots[j],
+	      j );
+    }
+    
+    days = (int)floor( globalTime / 86400 );
+    donkey = nodeTime - 86400 * days;
+    
+
+    hours = (int)floor( donkey / 3600 );
+    donkey -= hours * 3600;
+    
+    minutes = (int)floor( donkey / 60 );
+    donkey -= minutes * 60;
+    
+    secs = (int)donkey;
+    msecs = (int)( (donkey - secs) * 1000 );
+    
+    fprintf( stdout,
+	     "----------------------------------------------------------------------------\n"
+	     "  Total Time = %03d:%02d:%02d:%02d.%03d ( %G sec )\n"
+	     "\n"
+	     "  From Fortran: forceTime = %G secs; communicationTime = %G secs.\n",
+	     days,
+	     hours,
+	     minutes,
+	     secs,
+	     msecs,
+	     globalTime,
+	     globalForceTime,
+	     globalCommTime);
+  }
+
+  else{
+
+    for(j=1;j<mpiSim->getNumberProcessors();j++){
+      
+      if( worldRank == j ){
+	
+	F90_FUNC(gettimes, GETTIMES)(&forceTime, &commTime);
+
+	MPI_Send( accumTime, N_PROFILES, MPI_DOUBLE, 0, 1, MPI_COMM_WORLD );
+	MPI_Send( &forceTime, 1, MPI_DOUBLE, 0, 1, MPI_COMM_WORLD );
+	MPI_Send( &commTime, 1, MPI_DOUBLE, 0, 1, MPI_COMM_WORLD );
+      }
+    }
+  }
+    
 #endif // is_mpi