src/parallel/Communicator.hpp

/**
 * @file Communicator.hpp
 * @author Charles Vardeman <cvardema.at.nd.edu>
 * @date 08/18/2010
 * @time 11:56am
 * @version 1.0
 *
 * @section LICENSE
 * Copyright (c) 2010 The University of Notre Dame. All Rights Reserved.
 *
 * The University of Notre Dame grants you ("Licensee") a
 * non-exclusive, royalty free, license to use, modify and
 * redistribute this software in source and binary code form, provided
 * that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the
 *    distribution.
 *
 * This software is provided "AS IS," without a warranty of any
 * kind. All express or implied conditions, representations and
 * warranties, including any implied warranty of merchantability,
 * fitness for a particular purpose or non-infringement, are hereby
 * excluded.  The University of Notre Dame and its licensors shall not
 * be liable for any damages suffered by licensee as a result of
 * using, modifying or distributing the software or its
 * derivatives. In no event will the University of Notre Dame or its
 * licensors be liable for any lost revenue, profit or data, or for
 * direct, indirect, special, consequential, incidental or punitive
 * damages, however caused and regardless of the theory of liability,
 * arising out of the use of or inability to use software, even if the
 * University of Notre Dame has been advised of the possibility of
 * such damages.
 *
 * SUPPORT OPEN SCIENCE!  If you use OpenMD or its source code in your
 * research, please cite the appropriate papers when you publish your
 * work.  Good starting points are:
 *                                                                      
 * [1]  Meineke, et al., J. Comp. Chem. 26, 252-271 (2005).             
 * [2]  Fennell & Gezelter, J. Chem. Phys. 124, 234104 (2006).          
 * [3]  Sun, Lin & Gezelter, J. Chem. Phys. 128, 24107 (2008).          
 * [4]  Kuang & Gezelter,  J. Chem. Phys. 133, 164101 (2010).
 * [5]  Vardeman, Stocker & Gezelter, J. Chem. Theory Comput. 7, 834 (2011).
 */

#ifndef PARALLEL_COMMUNICATOR_HPP
#define PARALLEL_COMMUNICATOR_HPP

#include <config.h>
#include <mpi.h>
#include "math/SquareMatrix3.hpp"

using namespace std;
namespace OpenMD{
  
#ifdef IS_MPI

  enum communicatorType {
    Global = 0,
    Row = 1,
    Column = 2
  };
    
  template<class T>
  class MPITraits {
  public:
    static MPI::Datatype Type();
    static int Length() { return 1; };
  };
  
  template<> inline MPI::Datatype MPITraits<int>::Type() { return MPI::INT; }
  template<> inline MPI::Datatype MPITraits<RealType>::Type() { return MPI::REALTYPE; }

  template<class T, unsigned int Dim>
  class MPITraits< Vector<T, Dim> > {
  public:
    static MPI::Datatype Type() { return MPITraits<T>::Type(); }
    static int Length() {return Dim;}
  };

  template<class T>
  class MPITraits< Vector3<T> > {
  public:
    static MPI::Datatype Type() { return MPITraits<T>::Type(); }
    static int Length() {return 3;}
  };

  template<class T, unsigned int Row, unsigned int Col>
  class MPITraits< RectMatrix<T, Row, Col> > {
  public:
    static MPI::Datatype Type() { return MPITraits<T>::Type(); }
    static int Length() {return Row * Col;}
  };

  template<class T>
  class MPITraits< SquareMatrix3<T> > {
  public:
    static MPI::Datatype Type() { return MPITraits<T>::Type(); }
    static int Length() {return 9;}
  };
  
  
  template<communicatorType D>
  class Communicator { 
  public: 
    
    Communicator<D>() {
      
      int nProc = MPI::COMM_WORLD.Get_size();
      int myRank = MPI::COMM_WORLD.Get_rank();
      
      int nColumnsMax = (int) sqrt(RealType(nProc));

      int nColumns;
      for (int i = 1; i < nColumnsMax + 1; i++) {
        if (nProc % i == 0) nColumns = i;        
      }
        
      rowIndex_ = myRank / nColumns;      
      columnIndex_ = myRank % nColumns;

      switch(D) {
      case Row :
        myComm = MPI::COMM_WORLD.Split(rowIndex_, 0);
        break;
      case Column:
        myComm = MPI::COMM_WORLD.Split(columnIndex_, 0);
        break;
      case Global:
        myComm = MPI::COMM_WORLD.Split(myRank, 0);
      }

    }
    
    MPI::Intracomm getComm() { return myComm; }
    
  private:
    int rowIndex_;
    int columnIndex_;
    MPI::Intracomm myComm;
  };
  

  template<typename T>
  class Plan {
  public:
    
    Plan<T>(MPI::Intracomm comm, int nObjects) {
      myComm = comm;
      int nCommProcs = myComm.Get_size();
      
      counts.resize(nCommProcs, 0);
      displacements.resize(nCommProcs, 0);
      
      planSize_ = MPITraits<T>::Length() * nObjects; 
      
      myComm.Allgather(&planSize_, 1, MPI::INT, &counts[0], 1, MPI::INT);
      
      displacements[0] = 0;
      for (int i = 1; i < nCommProcs; i++) {
        displacements[i] = displacements[i-1] + counts[i-1];
      }

      size_ = 0;
      for (int i = 0; i < nCommProcs; i++) {
        size_ += counts[i];
      }
    }

    
    void gather(vector<T>& v1, vector<T>& v2) {
      
      // an assert would be helpful here to make sure the vectors are the
      // correct geometry
      
      myComm.Allgatherv(&v1[0], 
                        planSize_, 
                        MPITraits<T>::Type(), 
                        &v2[0], 
                        &counts[0], 
                        &displacements[0], 
                        MPITraits<T>::Type());      
    }       
    
    void scatter(vector<T>& v1, vector<T>& v2) {
      // an assert would be helpful here to make sure the vectors are the
      // correct geometry
            
      myComm.Reduce_scatter(&v1[0], &v2[0], &counts[0], 
                            MPITraits<T>::Type(), MPI::SUM);
    }
    
    int getSize() {
      return size_;
    }
    
  private:
    int planSize_;     ///< how many are on local proc
    int size_;
    vector<int> counts;
    vector<int> displacements;
    MPI::Intracomm myComm;
  }; 

#endif
}
#endif


Revision:	1796
Committed:	Mon Sep 10 18:38:44 2012 UTC (13 years, 1 month ago) by gezelter
File size:	6234 byte(s)
Log Message:	Updating MPI calls to C++, fixing a DumpWriter bug, cleaning cruft
#	Content
1	/**
2	* @file Communicator.hpp
3	* @author Charles Vardeman <cvardema.at.nd.edu>
4	* @date 08/18/2010
5	* @time 11:56am
6	* @version 1.0
7	*
8	* @section LICENSE
9	* Copyright (c) 2010 The University of Notre Dame. All Rights Reserved.
10	*
11	* The University of Notre Dame grants you ("Licensee") a
12	* non-exclusive, royalty free, license to use, modify and
13	* redistribute this software in source and binary code form, provided
14	* that the following conditions are met:
15	*
16	* 1. Redistributions of source code must retain the above copyright
17	* notice, this list of conditions and the following disclaimer.
18	*
19	* 2. Redistributions in binary form must reproduce the above copyright
20	* notice, this list of conditions and the following disclaimer in the
21	* documentation and/or other materials provided with the
22	* distribution.
23	*
24	* This software is provided "AS IS," without a warranty of any
25	* kind. All express or implied conditions, representations and
26	* warranties, including any implied warranty of merchantability,
27	* fitness for a particular purpose or non-infringement, are hereby
28	* excluded. The University of Notre Dame and its licensors shall not
29	* be liable for any damages suffered by licensee as a result of
30	* using, modifying or distributing the software or its
31	* derivatives. In no event will the University of Notre Dame or its
32	* licensors be liable for any lost revenue, profit or data, or for
33	* direct, indirect, special, consequential, incidental or punitive
34	* damages, however caused and regardless of the theory of liability,
35	* arising out of the use of or inability to use software, even if the
36	* University of Notre Dame has been advised of the possibility of
37	* such damages.
38	*
39	* SUPPORT OPEN SCIENCE! If you use OpenMD or its source code in your
40	* research, please cite the appropriate papers when you publish your
41	* work. Good starting points are:
42	*
43	* [1] Meineke, et al., J. Comp. Chem. 26, 252-271 (2005).
44	* [2] Fennell & Gezelter, J. Chem. Phys. 124, 234104 (2006).
45	* [3] Sun, Lin & Gezelter, J. Chem. Phys. 128, 24107 (2008).
46	* [4] Kuang & Gezelter, J. Chem. Phys. 133, 164101 (2010).
47	* [5] Vardeman, Stocker & Gezelter, J. Chem. Theory Comput. 7, 834 (2011).
48	*/
49
50	#ifndef PARALLEL_COMMUNICATOR_HPP
51	#define PARALLEL_COMMUNICATOR_HPP
52
53	#include <config.h>
54	#include <mpi.h>
55	#include "math/SquareMatrix3.hpp"
56
57	using namespace std;
58	namespace OpenMD{
59
60	#ifdef IS_MPI
61
62	enum communicatorType {
63	Global = 0,
64	Row = 1,
65	Column = 2
66	};
67
68	template<class T>
69	class MPITraits {
70	public:
71	static MPI::Datatype Type();
72	static int Length() { return 1; };
73	};
74
75	template<> inline MPI::Datatype MPITraits<int>::Type() { return MPI::INT; }
76	template<> inline MPI::Datatype MPITraits<RealType>::Type() { return MPI::REALTYPE; }
77
78	template<class T, unsigned int Dim>
79	class MPITraits< Vector<T, Dim> > {
80	public:
81	static MPI::Datatype Type() { return MPITraits<T>::Type(); }
82	static int Length() {return Dim;}
83	};
84
85	template<class T>
86	class MPITraits< Vector3<T> > {
87	public:
88	static MPI::Datatype Type() { return MPITraits<T>::Type(); }
89	static int Length() {return 3;}
90	};
91
92	template<class T, unsigned int Row, unsigned int Col>
93	class MPITraits< RectMatrix<T, Row, Col> > {
94	public:
95	static MPI::Datatype Type() { return MPITraits<T>::Type(); }
96	static int Length() {return Row * Col;}
97	};
98
99	template<class T>
100	class MPITraits< SquareMatrix3<T> > {
101	public:
102	static MPI::Datatype Type() { return MPITraits<T>::Type(); }
103	static int Length() {return 9;}
104	};
105
106
107	template<communicatorType D>
108	class Communicator {
109	public:
110
111	Communicator<D>() {
112
113	int nProc = MPI::COMM_WORLD.Get_size();
114	int myRank = MPI::COMM_WORLD.Get_rank();
115
116	int nColumnsMax = (int) sqrt(RealType(nProc));
117
118	int nColumns;
119	for (int i = 1; i < nColumnsMax + 1; i++) {
120	if (nProc % i == 0) nColumns = i;
121	}
122
123	rowIndex_ = myRank / nColumns;
124	columnIndex_ = myRank % nColumns;
125
126	switch(D) {
127	case Row :
128	myComm = MPI::COMM_WORLD.Split(rowIndex_, 0);
129	break;
130	case Column:
131	myComm = MPI::COMM_WORLD.Split(columnIndex_, 0);
132	break;
133	case Global:
134	myComm = MPI::COMM_WORLD.Split(myRank, 0);
135	}
136
137	}
138
139	MPI::Intracomm getComm() { return myComm; }
140
141	private:
142	int rowIndex_;
143	int columnIndex_;
144	MPI::Intracomm myComm;
145	};
146
147
148	template<typename T>
149	class Plan {
150	public:
151
152	Plan<T>(MPI::Intracomm comm, int nObjects) {
153	myComm = comm;
154	int nCommProcs = myComm.Get_size();
155
156	counts.resize(nCommProcs, 0);
157	displacements.resize(nCommProcs, 0);
158
159	planSize_ = MPITraits<T>::Length() * nObjects;
160
161	myComm.Allgather(&planSize_, 1, MPI::INT, &counts[0], 1, MPI::INT);
162
163	displacements[0] = 0;
164	for (int i = 1; i < nCommProcs; i++) {
165	displacements[i] = displacements[i-1] + counts[i-1];
166	}
167
168	size_ = 0;
169	for (int i = 0; i < nCommProcs; i++) {
170	size_ += counts[i];
171	}
172	}
173
174
175	void gather(vector<T>& v1, vector<T>& v2) {
176
177	// an assert would be helpful here to make sure the vectors are the
178	// correct geometry
179
180	myComm.Allgatherv(&v1[0],
181	planSize_,
182	MPITraits<T>::Type(),
183	&v2[0],
184	&counts[0],
185	&displacements[0],
186	MPITraits<T>::Type());
187	}
188
189	void scatter(vector<T>& v1, vector<T>& v2) {
190	// an assert would be helpful here to make sure the vectors are the
191	// correct geometry
192
193	myComm.Reduce_scatter(&v1[0], &v2[0], &counts[0],
194	MPITraits<T>::Type(), MPI::SUM);
195	}
196
197	int getSize() {
198	return size_;
199	}
200
201	private:
202	int planSize_; ///< how many are on local proc
203	int size_;
204	vector<int> counts;
205	vector<int> displacements;
206	MPI::Intracomm myComm;
207	};
208
209	#endif
210	}
211	#endif
212
213