21#include "rheolef/config.h"
22#ifdef _RHEOLEF_HAVE_MPI
24#include "rheolef/hack_array.h"
31template<
class T,
class A>
41template<
class T,
class A>
43 :
base(ownership,param,alloc),
51template <
class T,
class A>
55 base::resize (ownership, param);
59 _receive.waits.clear();
60 _receive.data.clear();
61 _receive_max_size = 0;
66template <
class T,
class A>
70 if (ownership().is_owned (dis_i)) {
71 size_type i = dis_i - ownership().first_index();
75 assert_macro (dis_i < ownership().dis_size(),
"index "<<dis_i
76 <<
" is out of range [0:" << ownership().dis_size() <<
"[");
78 size_type dis_iraw = base::_value_size*dis_i + 1;
79 typename generic_value_type::const_iterator iter = val._data_begin();
80 for (
size_type loc_iraw = 0, loc_nraw = val._data_size(); loc_iraw < loc_nraw; loc_iraw++, iter++) {
81 _stash.insert (std::pair<const size_type,raw_type>(dis_iraw + loc_iraw, *iter));
84template <
class T,
class A>
92 raw_base::ownership(),
98template <
class T,
class A>
107 raw_base::begin() - raw_base::ownership().first_index(),
108 details::generic_set_op(),
114 _receive.waits.clear();
115 _receive.data.clear();
116 _receive_max_size = 0;
125template <
class T,
class A>
126template <
class Set,
class Map>
135 std::vector<size_type> raw_ext_idx (base::_data_size*ext_idx_set.size());
136 typename Set::const_iterator iter = ext_idx_set.begin();
137 for (
size_type i = 0; i < ext_idx_set.size(); i++, iter++) {
139 for (
size_type l = 0; l < base::_data_size; l++) {
140 size_type raw_i = base::_data_size*i + l;
141 size_type raw_idx = base::_value_size*idx + l + (base::_value_size - base::_data_size);
142 raw_ext_idx [raw_i] = raw_idx;
146 std::vector<size_type> raw_id (raw_ext_idx.size());
147 for (
size_type i = 0; i < raw_id.size(); i++) raw_id[i] = i;
150 size_type raw_dis_size = base::_value_size*ownership().dis_size();
151 size_type raw_size = base::_value_size*ownership().size();
152 distributor raw_ownership (raw_dis_size, ownership().comm(), raw_size);
156 raw_ext_idx.begin().operator->(),
158 raw_id.begin().operator->(),
160 raw_ownership.
begin().operator->(),
162 raw_ownership.
comm(),
167 std::vector<raw_type> raw_buffer (raw_ext_idx.size());
171 typename raw_base::const_iterator raw_begin = raw_base::begin();
174 raw_buffer.begin().operator->(),
177 details::generic_set_op(),
179 raw_ownership.
comm());
184 raw_buffer.begin().operator->(),
187 details::generic_set_op(),
189 raw_ownership.
comm());
192 iter = ext_idx_set.begin();
193 for (
size_type i = 0; i < ext_idx_set.size(); i++, iter++) {
196 typename automatic_value_type::iterator
p = value._data_begin();
197 for (
size_type l = 0; l < base::_data_size; l++,
p++) {
198 size_type raw_i = base::_data_size*i + l;
199 *
p = raw_buffer[raw_i];
201 ext_idx_map.insert (std::make_pair (idx, value));
204template <
class T,
class A>
208 if (ownership().is_owned(dis_i)) {
209 size_type i = dis_i - ownership().first_index();
210 return operator[](i);
212 typename scatter_map_type::const_iterator iter = _ext_x.find (dis_i);
213 check_macro (iter != _ext_x.end(),
"unexpected external index="<<dis_i);
214 return (*iter).second;
216template <
class T,
class A>
220 std::set<size_type> ext_i;
221 for (
typename scatter_map_type::const_iterator
222 iter = _ext_x.begin(),
223 last = _ext_x.end(); iter != last; ++iter) {
224 ext_i.insert ((*iter).first);
226 get_dis_entry (ext_i, _ext_x);
231template <
class T,
class A>
232template <
class PutFunction>
237 std::ostream& os = ops.
os();
241 mpi::reduce(comm(), size(), max_size, mpi::maximum<size_type>(), 0);
244 if (ownership().process() == io_proc) {
245 base::put_values (ops, put_element);
247 std::vector<raw_type> raw_values (max_size*base::_data_size, std::numeric_limits<raw_type>::max());
248 for (
size_type iproc = 0; iproc < ownership().n_process(); iproc++) {
249 if (iproc == io_proc)
continue;
250 size_type loc_sz_i = ownership().size(iproc);
251 if (loc_sz_i == 0)
continue;
252 mpi::status status = comm().recv(iproc, tag, raw_values.begin().operator->(), raw_values.size());
253 boost::optional<int> n_data_opt = status.count<
raw_type>();
256 check_macro (n_data == loc_sz_i*base::_data_size,
"unexpected receive message size");
257 typename T::automatic_type tmp (base::_parameter);
258 for (
size_type i = 0; i < loc_sz_i; i++) {
259 typename T::iterator
p = tmp._data_begin();
260 for (
size_type iloc = 0; iloc < base::_data_size; iloc++,
p++) {
261 *
p = raw_values [i*base::_data_size + iloc];
263 put_element (os, tmp);
270 std::vector<raw_type> raw_values (size()*base::_data_size, std::numeric_limits<raw_type>::max());
271 for (
size_type i = 0, n = size(); i < n; i++) {
272 for (
size_type j = 0; j < base::_data_size; j++) {
273 raw_values [i*base::_data_size + j] = raw_base::operator[] (i*base::_value_size + j+(base::_value_size - base::_data_size));
276 comm().send(io_proc, tag, raw_values.begin().operator->(), raw_values.size());
281template <
class T,
class A>
287template <
class T,
class A>
288template <
class GetFunction>
293 std::istream& is = ips.
is();
294 size_type my_proc = ownership().process();
297 for (
size_type iproc = 0; iproc < ownership().n_process(); iproc++) {
298 size_max = std::max (size_max, ownership().size(iproc));
300 distributor io_ownership (size_max, comm(), (my_proc == io_proc) ? size_max : 0);
302 if (my_proc == io_proc) {
305 if (ownership().n_process() > 1) {
308 std::vector<raw_type> raw_values (size_max*base::_data_size, std::numeric_limits<raw_type>::max());
312 for (
size_type jproc = 0; jproc < ownership().n_process(); jproc++) {
313 if (jproc == io_proc)
continue;
315 size_type loc_sz_j = ownership().size(jproc);
316 if (loc_sz_j == 0)
continue;
319 for (
size_type i = 0, n = loc_sz_j; i < n; i++) {
320 for (
size_type j = 0; j < base::_data_size; j++) {
321 raw_values [i*base::_data_size + j] = data_proc_j.raw_base::operator[] (i*base::_value_size + j+(base::_value_size - base::_data_size));
324 comm().send (jproc, tag, raw_values.begin().operator->(), loc_sz_j*base::_data_size);
329 std::vector<raw_type> raw_values (size()*base::_data_size, std::numeric_limits<raw_type>::max());
330 comm().recv (io_proc, tag, raw_values.begin().operator->(), size()*base::_data_size);
331 for (
size_type i = 0, n = size(); i < n; i++) {
332 for (
size_type j = 0; j < base::_data_size; j++) {
333 raw_base::operator[] (i*base::_value_size + j+(base::_value_size - base::_data_size))
334 = raw_values [i*base::_data_size + j];
341template <
class T,
class A>
347template <
class T,
class A>
348template <
class PutFunction,
class Permutation>
352 const Permutation& perm,
353 PutFunction put_element)
const
356 assert_macro (perm.size() == size(),
"permutation size does not match");
359 distributor io_ownership (dis_size(), comm(), (my_proc == io_proc) ? dis_size() : 0);
361 for (
size_type i = 0, n = size(); i < n; i++) {
362 perm_x.dis_entry (perm[i]) = operator[](i);
364 perm_x.dis_entry_assembly();
365 perm_x.hack_array_seq_rep<T,A>::put_values (ops, put_element);
371template <
class T,
class A>
384 vector<size_type> send_local_elt_size (nproc, 0);
386 for (
size_type ie = 0; ie < partition.size(); ie++, iter_part++) {
387 send_local_elt_size [*iter_part]++;
389 vector<size_type> recv_local_elt_size (nproc, 0);
390 all_to_all (comm, send_local_elt_size, recv_local_elt_size);
391 vector<size_type> recv_local_elt_start (nproc+1);
392 recv_local_elt_start [0] = 0;
393 for (
size_type iproc = 0; iproc < nproc; iproc++) {
394 recv_local_elt_start [iproc+1] = recv_local_elt_start [iproc] + recv_local_elt_size[iproc];
396 vector<size_type> send_local_elt_start (nproc);
397 all_to_all (comm, recv_local_elt_start.begin().operator->(), send_local_elt_start.begin().operator->());
398 size_type new_local_n_elt = recv_local_elt_start [nproc];
402 distributor new_elt_ownership (global_n_elt, comm, new_local_n_elt);
403 new_array.resize (new_elt_ownership, base::_parameter);
404 old_numbering.resize (new_elt_ownership, numeric_limits<size_type>::max());
405 new_numbering.resize (ownership(), numeric_limits<size_type>::max());
406 iter_part = partition.begin();
409 for (
size_type ie = 0, ne = partition.size(); ie < ne; ie++, iter_part++, iter_elt++, iter_new_num_elt++) {
412 size_type new_global_ie = new_elt_ownership[iproc] + send_local_elt_start[iproc];
413 new_array.dis_entry (new_global_ie) = x;
414 *iter_new_num_elt = new_global_ie;
415 size_type old_global_ie = ownership()[my_proc] + ie;
416 old_numbering.dis_entry (new_global_ie) = old_global_ie;
417 send_local_elt_start[iproc]++;
419 new_array.dis_entry_assembly();
420 old_numbering.template dis_entry_assembly<typename details::default_set_op_traits<size_type>::type>();
field::size_type size_type
see the communicator page for the full documentation
see the distributor page for the full documentation
size_type dis_size() const
global and local sizes
static tag_type get_new_tag()
returns a new tag
const communicator_type & comm() const
void dis_entry_assembly_end()
const_reference dis_at(size_type dis_i) const
odiststream & put_values(odiststream &ops) const
void repartition(const disarray_rep< size_type, distributed, A2 > &partition, hack_array_mpi_rep< T, A > &new_array, disarray_rep< size_type, distributed, A2 > &old_numbering, disarray_rep< size_type, distributed, A2 > &new_numbering) const
void dis_entry_assembly_begin()
odiststream & permuted_put_values(odiststream &ops, const Permutation &perm, PutFunction put_element) const
base::size_type size_type
base::const_reference const_reference
hack_array_mpi_rep(const A &alloc=A())
void resize(const distributor &ownership, const parameter_type ¶m)
base::const_iterator const_iterator
base::parameter_type parameter_type
base::automatic_value_type automatic_value_type
void update_dis_entries() const
void append_dis_entry(const Set &ext_idx_set, Map &ext_idx_map) const
get values from ext_idx_set, that are managed by another proc new version: instead of sending automat...
base::generic_value_type generic_value_type
void set_dis_entry(size_type dis_i, const generic_value_type &val)
idiststream & get_values(idiststream &ips)
hack_array_iterator< generic_value_type, generic_value_type &, generic_value_type *, raw_type, raw_type * > iterator
idiststream: see the diststream page for the full documentation
odiststream: see the diststream page for the full documentation
static size_type io_proc()
#define assert_macro(ok_condition, message)
check_macro(expr1.have_homogeneous_space(Xh1), "dual(expr1,expr2); expr1 should have homogeneous space. HINT: use dual(interpolate(Xh, expr1),expr2)")
This file is part of Rheolef.
Size mpi_assembly_end(Message &receive, Message &send, Size receive_max_size, Container x)
void mpi_scatter_end(InputIterator x, OutputIterator y, Message &from, Message &to, SetOp op, Tag tag, Comm comm)
void mpi_scatter_begin(InputIterator x, OutputIterator y, Message &from, Message &to, SetOp op, Tag tag, Comm comm)
bool load_chunk(std::istream &s, RandomIterator iter, RandomIterator last)
disarray_store< OutputRandomIterator, SetOp, Size, IsContainer > disarray_make_store(OutputRandomIterator x, SetOp op, Size, IsContainer)
void mpi_scatter_init(Size nidx, SizeRandomIterator1 idx, Size nidy, SizeRandomIterator2 idy, Size idy_maxval, SizeRandomIterator3 ownership, Tag tag, const distributor::communicator_type &comm, Message &from, Message &to)
apply_iterator< Iterator, Operator > make_apply_iterator(Iterator i, Operator op)
Stash::size_type mpi_assembly_begin(const Stash &stash, InputIterator first_stash_idx, InputIterator last_stash_idx, const distributor &ownership, Message &receive, Message &send)
disarray element input helper
disarray element output helper