src/parallel/Communicator.hpp

/**
 * @file Communicator.hpp
 * @author Charles Vardeman <cvardema.at.nd.edu>
 * @date 08/18/2010
 * @version 1.0
 *
 * @section LICENSE
 * Copyright (c) 2010 The University of Notre Dame. All Rights Reserved.
 *
 * The University of Notre Dame grants you ("Licensee") a
 * non-exclusive, royalty free, license to use, modify and
 * redistribute this software in source and binary code form, provided
 * that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the
 *    distribution.
 *
 * This software is provided "AS IS," without a warranty of any
 * kind. All express or implied conditions, representations and
 * warranties, including any implied warranty of merchantability,
 * fitness for a particular purpose or non-infringement, are hereby
 * excluded.  The University of Notre Dame and its licensors shall not
 * be liable for any damages suffered by licensee as a result of
 * using, modifying or distributing the software or its
 * derivatives. In no event will the University of Notre Dame or its
 * licensors be liable for any lost revenue, profit or data, or for
 * direct, indirect, special, consequential, incidental or punitive
 * damages, however caused and regardless of the theory of liability,
 * arising out of the use of or inability to use software, even if the
 * University of Notre Dame has been advised of the possibility of
 * such damages.
 *
 * SUPPORT OPEN SCIENCE!  If you use OpenMD or its source code in your
 * research, please cite the appropriate papers when you publish your
 * work.  Good starting points are:
 *                                                                      
 * [1]  Meineke, et al., J. Comp. Chem. 26, 252-271 (2005).             
 * [2]  Fennell & Gezelter, J. Chem. Phys. 124, 234104 (2006).          
 * [3]  Sun, Lin & Gezelter, J. Chem. Phys. 128, 234107 (2008).          
 * [4]  Kuang & Gezelter,  J. Chem. Phys. 133, 164101 (2010).
 * [5]  Vardeman, Stocker & Gezelter, J. Chem. Theory Comput. 7, 834 (2011).
 */

#ifndef PARALLEL_COMMUNICATOR_HPP
#define PARALLEL_COMMUNICATOR_HPP

#include <config.h>
#include <mpi.h>
#include "math/SquareMatrix3.hpp"

using namespace std;
namespace OpenMD{
  
#ifdef IS_MPI

  enum communicatorType {
    Global = 0,
    Row = 1,
    Column = 2
  };
    
  template<class T>
  class MPITraits {
  public:
    static MPI_Datatype Type();
    static int Length() { return 1; };
  };
  
  template<> inline MPI_Datatype MPITraits<int>::Type() { return MPI_INT; }
  template<> inline MPI_Datatype MPITraits<RealType>::Type() { return MPI_REALTYPE; }

  template<class T, unsigned int Dim>
  class MPITraits< Vector<T, Dim> > {
  public:
    static MPI_Datatype Type() { return MPITraits<T>::Type(); }
    static int Length() {return Dim;}
  };

  template<class T>
  class MPITraits< Vector3<T> > {
  public:
    static MPI_Datatype Type() { return MPITraits<T>::Type(); }
    static int Length() {return 3;}
  };

  template<class T, unsigned int R, unsigned int C>
  class MPITraits< RectMatrix<T, R, C> > {
  public:
    static MPI_Datatype Type() { return MPITraits<T>::Type(); }
    static int Length() {return R * C;}
  };

  template<class T>
  class MPITraits< SquareMatrix3<T> > {
  public:
    static MPI_Datatype Type() { return MPITraits<T>::Type(); }
    static int Length() {return 9;}
  };
  
  
  template<communicatorType D>
  class Communicator { 
  public: 
    
    Communicator<D>() {
      
      int nProc;
      int myRank;

      MPI_Comm_size( MPI_COMM_WORLD, &nProc);
      MPI_Comm_rank( MPI_COMM_WORLD, &myRank);
      
      int nColumnsMax = (int) sqrt(RealType(nProc));

      int nColumns;
      for (int i = 1; i < nColumnsMax + 1; i++) {
        if (nProc % i == 0) nColumns = i;        
      }
        
      // int nRows = nProc / nColumns;
      rowIndex_ = myRank / nColumns;      
      columnIndex_ = myRank % nColumns;

      switch(D) {
      case Row :
        MPI_Comm_split(MPI_COMM_WORLD, rowIndex_, 0, &myComm);
        //  myComm = MPI::COMM_WORLD.Split(rowIndex_, 0);
        break;
      case Column:
        MPI_Comm_split(MPI_COMM_WORLD, columnIndex_, 0, &myComm);
        //myComm = MPI::COMM_WORLD.Split(columnIndex_, 0);
        break;
      case Global:
        MPI_Comm_split(MPI_COMM_WORLD, myRank, 0, &myComm);
        //myComm = MPI::COMM_WORLD.Split(myRank, 0);
      }

    }
    
    MPI_Comm getComm() { return myComm; }
    
  private:
    int rowIndex_;
    int columnIndex_;
    MPI_Comm myComm;
  };
  

  template<typename T>
  class Plan {
  public:
    
    Plan<T>(MPI_Comm comm, int nObjects) : myComm(comm) {

      int nCommProcs;
      MPI_Comm_size( myComm, &nCommProcs );

      //int nCommProcs = myComm.Get_size();
      
      counts.resize(nCommProcs, 0);
      displacements.resize(nCommProcs, 0);
      
      planSize_ = MPITraits<T>::Length() * nObjects; 
      
      MPI_Allgather(&planSize_, 1, MPI_INT, &counts[0], 1, MPI_INT, myComm);
      //myComm.Allgather(&planSize_, 1, MPI_INT, &counts[0], 1, MPI_INT);
      
      displacements[0] = 0;
      for (int i = 1; i < nCommProcs; i++) {
        displacements[i] = displacements[i-1] + counts[i-1];
      }

      size_ = 0;
      for (int i = 0; i < nCommProcs; i++) {
        size_ += counts[i];
      }
    }

    
    void gather(vector<T>& v1, vector<T>& v2) {
      
      // an assert would be helpful here to make sure the vectors are the
      // correct geometry
      
      MPI_Allgatherv(&v1[0], 
                     planSize_, 
                     MPITraits<T>::Type(), 
                     &v2[0], 
                     &counts[0], 
                     &displacements[0], 
                     MPITraits<T>::Type(),
                     myComm);
      // myComm.Allgatherv(&v1[0], 
      //                   planSize_, 
      //                   MPITraits<T>::Type(), 
      //                   &v2[0], 
      //                   &counts[0], 
      //                   &displacements[0], 
      //                   MPITraits<T>::Type());      
    }       
    
    void scatter(vector<T>& v1, vector<T>& v2) {
      // an assert would be helpful here to make sure the vectors are the
      // correct geometry
            
      MPI_Reduce_scatter(&v1[0], &v2[0], &counts[0], 
                         MPITraits<T>::Type(), MPI_SUM, myComm);

      // myComm.Reduce_scatter(&v1[0], &v2[0], &counts[0], 
      //                       MPITraits<T>::Type(), MPI::SUM);
    }
    
    int getSize() {
      return size_;
    }
    
  private:
    int planSize_;     ///< how many are on local proc
    int size_;
    vector<int> counts;
    vector<int> displacements;
    MPI_Comm myComm;
  }; 

#endif
}
#endif


Revision:	1969
Committed:	Wed Feb 26 14:14:50 2014 UTC (11 years, 8 months ago) by gezelter
File size:	7018 byte(s)
Log Message:	Fixes to deal with deprecation of MPI C++ bindings. We've reverted back to the C calls.
#	Content
1	/**
2	* @file Communicator.hpp
3	* @author Charles Vardeman <cvardema.at.nd.edu>
4	* @date 08/18/2010
5	* @version 1.0
6	*
7	* @section LICENSE
8	* Copyright (c) 2010 The University of Notre Dame. All Rights Reserved.
9	*
10	* The University of Notre Dame grants you ("Licensee") a
11	* non-exclusive, royalty free, license to use, modify and
12	* redistribute this software in source and binary code form, provided
13	* that the following conditions are met:
14	*
15	* 1. Redistributions of source code must retain the above copyright
16	* notice, this list of conditions and the following disclaimer.
17	*
18	* 2. Redistributions in binary form must reproduce the above copyright
19	* notice, this list of conditions and the following disclaimer in the
20	* documentation and/or other materials provided with the
21	* distribution.
22	*
23	* This software is provided "AS IS," without a warranty of any
24	* kind. All express or implied conditions, representations and
25	* warranties, including any implied warranty of merchantability,
26	* fitness for a particular purpose or non-infringement, are hereby
27	* excluded. The University of Notre Dame and its licensors shall not
28	* be liable for any damages suffered by licensee as a result of
29	* using, modifying or distributing the software or its
30	* derivatives. In no event will the University of Notre Dame or its
31	* licensors be liable for any lost revenue, profit or data, or for
32	* direct, indirect, special, consequential, incidental or punitive
33	* damages, however caused and regardless of the theory of liability,
34	* arising out of the use of or inability to use software, even if the
35	* University of Notre Dame has been advised of the possibility of
36	* such damages.
37	*
38	* SUPPORT OPEN SCIENCE! If you use OpenMD or its source code in your
39	* research, please cite the appropriate papers when you publish your
40	* work. Good starting points are:
41	*
42	* [1] Meineke, et al., J. Comp. Chem. 26, 252-271 (2005).
43	* [2] Fennell & Gezelter, J. Chem. Phys. 124, 234104 (2006).
44	* [3] Sun, Lin & Gezelter, J. Chem. Phys. 128, 234107 (2008).
45	* [4] Kuang & Gezelter, J. Chem. Phys. 133, 164101 (2010).
46	* [5] Vardeman, Stocker & Gezelter, J. Chem. Theory Comput. 7, 834 (2011).
47	*/
48
49	#ifndef PARALLEL_COMMUNICATOR_HPP
50	#define PARALLEL_COMMUNICATOR_HPP
51
52	#include <config.h>
53	#include <mpi.h>
54	#include "math/SquareMatrix3.hpp"
55
56	using namespace std;
57	namespace OpenMD{
58
59	#ifdef IS_MPI
60
61	enum communicatorType {
62	Global = 0,
63	Row = 1,
64	Column = 2
65	};
66
67	template<class T>
68	class MPITraits {
69	public:
70	static MPI_Datatype Type();
71	static int Length() { return 1; };
72	};
73
74	template<> inline MPI_Datatype MPITraits<int>::Type() { return MPI_INT; }
75	template<> inline MPI_Datatype MPITraits<RealType>::Type() { return MPI_REALTYPE; }
76
77	template<class T, unsigned int Dim>
78	class MPITraits< Vector<T, Dim> > {
79	public:
80	static MPI_Datatype Type() { return MPITraits<T>::Type(); }
81	static int Length() {return Dim;}
82	};
83
84	template<class T>
85	class MPITraits< Vector3<T> > {
86	public:
87	static MPI_Datatype Type() { return MPITraits<T>::Type(); }
88	static int Length() {return 3;}
89	};
90
91	template<class T, unsigned int R, unsigned int C>
92	class MPITraits< RectMatrix<T, R, C> > {
93	public:
94	static MPI_Datatype Type() { return MPITraits<T>::Type(); }
95	static int Length() {return R * C;}
96	};
97
98	template<class T>
99	class MPITraits< SquareMatrix3<T> > {
100	public:
101	static MPI_Datatype Type() { return MPITraits<T>::Type(); }
102	static int Length() {return 9;}
103	};
104
105
106	template<communicatorType D>
107	class Communicator {
108	public:
109
110	Communicator<D>() {
111
112	int nProc;
113	int myRank;
114
115	MPI_Comm_size( MPI_COMM_WORLD, &nProc);
116	MPI_Comm_rank( MPI_COMM_WORLD, &myRank);
117
118	int nColumnsMax = (int) sqrt(RealType(nProc));
119
120	int nColumns;
121	for (int i = 1; i < nColumnsMax + 1; i++) {
122	if (nProc % i == 0) nColumns = i;
123	}
124
125	// int nRows = nProc / nColumns;
126	rowIndex_ = myRank / nColumns;
127	columnIndex_ = myRank % nColumns;
128
129	switch(D) {
130	case Row :
131	MPI_Comm_split(MPI_COMM_WORLD, rowIndex_, 0, &myComm);
132	// myComm = MPI::COMM_WORLD.Split(rowIndex_, 0);
133	break;
134	case Column:
135	MPI_Comm_split(MPI_COMM_WORLD, columnIndex_, 0, &myComm);
136	//myComm = MPI::COMM_WORLD.Split(columnIndex_, 0);
137	break;
138	case Global:
139	MPI_Comm_split(MPI_COMM_WORLD, myRank, 0, &myComm);
140	//myComm = MPI::COMM_WORLD.Split(myRank, 0);
141	}
142
143	}
144
145	MPI_Comm getComm() { return myComm; }
146
147	private:
148	int rowIndex_;
149	int columnIndex_;
150	MPI_Comm myComm;
151	};
152
153
154	template<typename T>
155	class Plan {
156	public:
157
158	Plan<T>(MPI_Comm comm, int nObjects) : myComm(comm) {
159
160	int nCommProcs;
161	MPI_Comm_size( myComm, &nCommProcs );
162
163	//int nCommProcs = myComm.Get_size();
164
165	counts.resize(nCommProcs, 0);
166	displacements.resize(nCommProcs, 0);
167
168	planSize_ = MPITraits<T>::Length() * nObjects;
169
170	MPI_Allgather(&planSize_, 1, MPI_INT, &counts[0], 1, MPI_INT, myComm);
171	//myComm.Allgather(&planSize_, 1, MPI_INT, &counts[0], 1, MPI_INT);
172
173	displacements[0] = 0;
174	for (int i = 1; i < nCommProcs; i++) {
175	displacements[i] = displacements[i-1] + counts[i-1];
176	}
177
178	size_ = 0;
179	for (int i = 0; i < nCommProcs; i++) {
180	size_ += counts[i];
181	}
182	}
183
184
185	void gather(vector<T>& v1, vector<T>& v2) {
186
187	// an assert would be helpful here to make sure the vectors are the
188	// correct geometry
189
190	MPI_Allgatherv(&v1[0],
191	planSize_,
192	MPITraits<T>::Type(),
193	&v2[0],
194	&counts[0],
195	&displacements[0],
196	MPITraits<T>::Type(),
197	myComm);
198	// myComm.Allgatherv(&v1[0],
199	// planSize_,
200	// MPITraits<T>::Type(),
201	// &v2[0],
202	// &counts[0],
203	// &displacements[0],
204	// MPITraits<T>::Type());
205	}
206
207	void scatter(vector<T>& v1, vector<T>& v2) {
208	// an assert would be helpful here to make sure the vectors are the
209	// correct geometry
210
211	MPI_Reduce_scatter(&v1[0], &v2[0], &counts[0],
212	MPITraits<T>::Type(), MPI_SUM, myComm);
213
214	// myComm.Reduce_scatter(&v1[0], &v2[0], &counts[0],
215	// MPITraits<T>::Type(), MPI::SUM);
216	}
217
218	int getSize() {
219	return size_;
220	}
221
222	private:
223	int planSize_; ///< how many are on local proc
224	int size_;
225	vector<int> counts;
226	vector<int> displacements;
227	MPI_Comm myComm;
228	};
229
230	#endif
231	}
232	#endif
233
234