Pane_communicator.h

Go to the documentation of this file.

/* *******************************************************************
 * Rocstar Simulation Suite                                          *
 * Copyright@2015, Illinois Rocstar LLC. All rights reserved.        *
 *                                                                   *
 * Illinois Rocstar LLC                                              *
 * Champaign, IL                                                     *
 * www.illinoisrocstar.com                                           *
 * sales@illinoisrocstar.com                                         *
 *                                                                   *
 * License: See LICENSE file in top level of distribution package or *
 * http://opensource.org/licenses/NCSA                               *
 *********************************************************************/
/* *******************************************************************
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,   *
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES   *
 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND          *
 * NONINFRINGEMENT.  IN NO EVENT SHALL THE CONTRIBUTORS OR           *
 * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER       *
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,   *
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE    *
 * USE OR OTHER DEALINGS WITH THE SOFTWARE.                          *
 *********************************************************************/
// $Id: Pane_communicator.h,v 1.20 2009/08/27 14:04:49 mtcampbe Exp $

#ifndef _PANE_COMMUNICATOR_H
#define _PANE_COMMUNICATOR_H

#include "mapbasic.h"
#include "roccom_devel.h"
#include "commpi.h"
#include "roccom_assertion.h"

MAP_BEGIN_NAMESPACE

class Pane_communicator {
protected:
  struct Pane_comm_buffers {
    // Default constructor
    Pane_comm_buffers() : rank(-1), tag(-1), index(-1) {}

    int                   rank;   // rank for communicating process
    int                   tag;    // tag for MPI message
    int                   index;  // starting index of information 
                                  // between the two panes in the 
                                  // pconn of the local pane
    std::vector< char>    outbuf; // buffer for outgoing messages
    std::vector< char>    inbuf;  // buffer for incoming messages
  };
public:
  // Note: One can use the RNS and GNR for sending and receiving data for
  // boundary edges with a custmized pconn.
  enum Buff_type {
    RNS,         // Real nodes to send
    RCS,         // Real cells to send.
    SHARED_NODE, // Shared nodes to send and receive.
    GNR,         // Ghost nodes to receive.
    GCR          // Ghost cells to receive.
  };

  explicit Pane_communicator( COM::Window *w, MPI_Comm c=MPI_COMM_WORLD);
  
  void init( void** ptrs, COM_Type type, int ncomp, 
             const int *sizes=NULL, const int *strds=NULL);

  void init( COM::Attribute *att, const COM::Attribute *my_pconn = NULL);

  MPI_Comm mpi_comm() const { return _comm; }

  std::vector<COM::Pane*> &panes() { return _panes; }

  int owner_rank( const int pane_id) const 
  { return _appl_window->owner_rank( pane_id); }

  int total_npanes() const { return _total_npanes; }

  int lpaneid( const int pane_id) const {
    COM_assertion( _total_npanes>0); 
    return _lpaneid_map.find(pane_id)->second;
  }

  void begin_update_shared_nodes(std::vector<std::vector<bool> > *involved=NULL){
    begin_update(SHARED_NODE,involved);
  }

  void begin_update_ghost_nodes(){
    begin_update(RNS);
    begin_update(GNR);
  }

  void begin_update_ghost_cells(){
    begin_update(RCS);
    begin_update(GCR);
  }

  void end_update_shared_nodes() {
    end_update();
  }

  void end_update_ghost_nodes(){
    update_ghost_values();
    end_update();
  }

  void end_update_ghost_cells(){
    update_ghost_values();
    end_update();
  }

  void reduce_on_shared_nodes( MPI_Op);

  void reduce_maxabs_on_shared_nodes();

  void reduce_minabs_on_shared_nodes();

  void reduce_diff_on_shared_nodes();

  void update_ghost_values();

  void reduce_average_on_shared_nodes();

protected:
  void init_pane_comm_buffers(std::vector< Pane_comm_buffers>& pcb,
                              const int* ptr, int& index, const int n_items, const int lpid);

  void end_update();

  void begin_update(const Buff_type btype,
                    std::vector<std::vector<bool> > *involved=NULL);

  int                           _my_pconn_id;

  COM::Window                  *_appl_window;
  const MPI_Comm                _comm;
  std::vector<COM::Pane*>       _panes;
  int                           _total_npanes;
  std::map<int,int>             _lpaneid_map;
  int                              _type, _ncomp, _ncomp_bytes;
  std::vector< void*>              _ptrs;
  std::vector< int>                _sizes;
  std::vector< int>                _strds;
  //  communicating panes for each local pane.
  std::vector< std::vector< Pane_comm_buffers> > _shr_buffs;
  std::vector< std::vector< Pane_comm_buffers> > _rns_buffs;
  std::vector< std::vector< Pane_comm_buffers> > _gnr_buffs;
  std::vector< std::vector< Pane_comm_buffers> > _rcs_buffs;
  std::vector< std::vector< Pane_comm_buffers> > _gcr_buffs;
  std::vector<MPI_Request>         _reqs_send, _reqs_recv;
  std::vector<std::pair<int,int> > _reqs_indices;

private:
  // Disable the following operators
  Pane_communicator( const Pane_communicator &);
  Pane_communicator &operator=( const Pane_communicator&); 
};

MAP_END_NAMESPACE

#endif  /* _PANE_COMMUNICATOR_H */