/**
 * @file xt_exchanger_mix_isend_irecv.c
 *
 * @copyright Copyright  (C)  2016 Jörg Behrens <behrens@dkrz.de>
 *                                 Moritz Hanke <hanke@dkrz.de>
 *                                 Thomas Jahns <jahns@dkrz.de>
 *
 * @author Jörg Behrens <behrens@dkrz.de>
 *         Moritz Hanke <hanke@dkrz.de>
 *         Thomas Jahns <jahns@dkrz.de>
 */
/*
 * Keywords:
 * Maintainer: Jörg Behrens <behrens@dkrz.de>
 *             Moritz Hanke <hanke@dkrz.de>
 *             Thomas Jahns <jahns@dkrz.de>
 * URL: https://dkrz-sw.gitlab-pages.dkrz.de/yaxt/
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are  permitted provided that the following conditions are
 * met:
 *
 * Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 *
 * Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 *
 * Neither the name of the DKRZ GmbH nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <assert.h>
#include <mpi.h>
#ifdef _OPENMP
#include <omp.h>
#endif

#include "core/core.h"
#include "core/ppm_xfuncs.h"
#include "xt_config_internal.h"
#include "xt/xt_mpi.h"
#include "xt/xt_request_msgs.h"
#include "xt_request_msgs_internal.h"
#include "xt_mpi_internal.h"
#include "xt_exchanger.h"
#include "xt_exchanger_internal.h"
#include "xt_exchanger_mix_isend_irecv.h"

/* unfortunately GCC 11 to 13 cannot handle the literal constants used for
 * MPI_STATUSES_IGNORE by MPICH */
#if __GNUC__ >= 11 && __GNUC__ <= 13 && defined MPICH
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wstringop-overread"
#pragma GCC diagnostic ignored "-Wstringop-overflow"
#endif

static Xt_exchanger
xt_exchanger_mix_isend_irecv_copy(Xt_exchanger exchanger,
                                  MPI_Comm newComm, int new_tag_offset);
static void xt_exchanger_mix_isend_irecv_delete(Xt_exchanger exchanger);
static void
xt_exchanger_mix_isend_irecv_s_exchange(Xt_exchanger exchanger,
                                        const void *src_data,
                                        void *dst_data);
#ifdef _OPENMP
static void
xt_exchanger_mix_isend_irecv_s_exchange_omp(Xt_exchanger exchanger,
                                            const void *src_data,
                                            void *dst_data);
#endif
static void
xt_exchanger_mix_isend_irecv_a_exchange(
  Xt_exchanger exchanger, const void * src_data, void * dst_data,
  Xt_request *request);
#ifdef _OPENMP
static void
xt_exchanger_mix_isend_irecv_a_exchange_omp(
  Xt_exchanger exchanger, const void * src_data, void * dst_data,
  Xt_request *request);
#endif
static Xt_exchanger_thread_share
xt_exchanger_mix_isend_irecv_create_thread_share(Xt_exchanger exchanger,
                                                 Xt_config config);

static void
xt_exchanger_mix_isend_irecv_destroy_thread_share(
  Xt_exchanger exchanger, Xt_exchanger_thread_share thread_share);

static int
xt_exchanger_mix_isend_irecv_get_num_msg(Xt_exchanger exchanger,
                                         int direction);
static int
xt_exchanger_mix_isend_irecv_get_msg_ranks(Xt_exchanger exchanger,
                                           enum xt_msg_direction direction,
                                           int *restrict *ranks);

static MPI_Datatype
xt_exchanger_mix_isend_irecv_get_MPI_Datatype(Xt_exchanger exchanger,
                                              int rank,
                                              enum xt_msg_direction direction,
                                              bool do_dup);

static MPI_Aint
xt_exchanger_mix_isend_irecv_get_displacement(Xt_exchanger exchanger,
                                              int rank,
                                              enum xt_msg_direction direction);

static int
xt_exchanger_mix_isend_irecv_get_count(Xt_exchanger exchanger,
                                       int rank,
                                       enum xt_msg_direction direction);

static Xt_exchanger_iter
xt_exchanger_mix_isend_irecv_get_iterator(Xt_exchanger exchanger, Xt_config config);

const struct xt_exchanger_vtable
xt_exchanger_mix_isend_irecv_vtable = {
  .copy = xt_exchanger_mix_isend_irecv_copy,
  .delete = xt_exchanger_mix_isend_irecv_delete,
  .s_exchange = xt_exchanger_mix_isend_irecv_s_exchange,
  .a_exchange = xt_exchanger_mix_isend_irecv_a_exchange,
  .get_num_msg = xt_exchanger_mix_isend_irecv_get_num_msg,
  .get_msg_ranks = xt_exchanger_mix_isend_irecv_get_msg_ranks,
  .get_MPI_Datatype = xt_exchanger_mix_isend_irecv_get_MPI_Datatype,
  .get_displacement = xt_exchanger_mix_isend_irecv_get_displacement,
  .get_count = xt_exchanger_mix_isend_irecv_get_count,
  .create_thread_share = xt_exchanger_mix_isend_irecv_create_thread_share,
  .destroy_thread_share = xt_exchanger_mix_isend_irecv_destroy_thread_share,
  .get_iterator = xt_exchanger_mix_isend_irecv_get_iterator,
}
#ifdef _OPENMP
  ,
xt_exchanger_mix_isend_irecv_auto_omp_vtable = {
  .copy = xt_exchanger_mix_isend_irecv_copy,
  .delete = xt_exchanger_mix_isend_irecv_delete,
  .s_exchange = xt_exchanger_mix_isend_irecv_s_exchange_omp,
  .a_exchange = xt_exchanger_mix_isend_irecv_a_exchange_omp,
  .get_num_msg = xt_exchanger_mix_isend_irecv_get_num_msg,
  .get_msg_ranks = xt_exchanger_mix_isend_irecv_get_msg_ranks,
  .get_MPI_Datatype = xt_exchanger_mix_isend_irecv_get_MPI_Datatype,
  .get_displacement = xt_exchanger_mix_isend_irecv_get_displacement,
  .get_count = xt_exchanger_mix_isend_irecv_get_count,
  .create_thread_share = xt_exchanger_mix_isend_irecv_create_thread_share,
  .destroy_thread_share = xt_exchanger_mix_isend_irecv_destroy_thread_share,
  .get_iterator = xt_exchanger_mix_isend_irecv_get_iterator,
}
#endif
;

typedef struct Xt_exchanger_mix_isend_irecv_ * Xt_exchanger_mix_isend_irecv;

#define MSG_DIR(msg) ((msg).rank < 0)

struct Xt_exchanger_mix_isend_irecv_ {

  const struct xt_exchanger_vtable * vtable;

  int n, tag_offset;
  MPI_Comm comm;
  enum Xt_exch_send_comm_mode send_mode;
  struct Xt_msg_param msgs[];
};

static Xt_exchanger_mix_isend_irecv
xt_exchanger_mix_isend_irecv_alloc(size_t nmsg,
                                   Xt_config config)
{
  Xt_exchanger_mix_isend_irecv exchanger;
  size_t header_size = sizeof (*exchanger),
    body_size = sizeof (struct Xt_msg_param) * nmsg;
  exchanger = xmalloc(header_size + body_size);
  exchanger->n = (int)nmsg;
  exchanger->send_mode = XT_CONFIG_GET_EXCH_SEND_COMM_MODE(config);
#ifdef _OPENMP
  int mthread_mode = xt_config_get_redist_mthread_mode(config);
  if (mthread_mode == XT_MT_OPENMP)
    exchanger->vtable = &xt_exchanger_mix_isend_irecv_auto_omp_vtable;
  else
#endif
    exchanger->vtable = &xt_exchanger_mix_isend_irecv_vtable;
  return exchanger;
}

static inline int
adjusted_rank(int r, int comm_rank, int comm_size)
{
  int r_ = r & INT_MAX;
  return r_ + (r_ <= comm_rank ? comm_size : 0);
}

#define XT_SORTFUNC_DECL static
#define SORT_TYPE struct Xt_msg_param
#define SORT_TYPE_SUFFIX msg_param_mix
#define SORT_TYPE_CMP_LT(u,v,i,j)                       \
  (adjusted_rank((u).rank, comm_rank, comm_size)        \
   < adjusted_rank((v).rank, comm_rank, comm_size))
#define SORT_TYPE_CMP_LE(u,v,i,j)                       \
  (adjusted_rank((u).rank, comm_rank, comm_size)        \
   <= adjusted_rank((v).rank, comm_rank, comm_size))
#define SORT_TYPE_CMP_EQ(u,v,i,j) \
  (((u).rank & INT_MAX) == ((v).rank & INT_MAX))
#define XT_SORT_EXTRA_ARGS_DECL , int comm_rank, int comm_size
#define XT_SORT_EXTRA_ARGS_PASS , comm_rank, comm_size
#define XT_SORT_VECSWAP_EXTRA_ARGS_DECL
#define XT_SORT_VECSWAP_EXTRA_ARGS_PASS

#include "xt_quicksort_base.h"


Xt_exchanger
xt_exchanger_mix_isend_irecv_new(int nsend, int nrecv,
                                 const struct Xt_msg_param *send_msgs,
                                 const struct Xt_msg_param *recv_msgs,
                                 MPI_Comm comm, int tag_offset,
                                 Xt_config config)
{
  /** note: tag_offset + xt_mpi_tag_exchange_msg must not
   *        be used on @a comm by any other part of the program during the
   *        lifetime of the created exchanger object
   */

  assert((nsend >= 0) & (nrecv >= 0));
  size_t nmsg = (size_t)nsend + (size_t)nrecv;
  Xt_exchanger_mix_isend_irecv exchanger
    = xt_exchanger_mix_isend_irecv_alloc(nmsg, config);
  exchanger->comm = comm;
  exchanger->tag_offset = tag_offset;
  struct Xt_msg_param *restrict msgs = exchanger->msgs;
  bool dt_dup = !(config->flags & exch_no_dt_dup);
  xt_msg_params_copy((size_t)nsend, send_msgs, msgs, comm, dt_dup);
  xt_msg_params_copy((size_t)nrecv, recv_msgs, msgs+nsend, comm, dt_dup);
  for (size_t i = 0; i < (size_t)nrecv; ++i)
    msgs[i + (size_t)nsend].rank |= ~INT_MAX;

  {
    int comm_size, comm_rank, is_inter;
    xt_mpi_call(MPI_Comm_rank(comm, &comm_rank), comm);
    xt_mpi_call(MPI_Comm_test_inter(comm, &is_inter), comm);
    int (*get_comm_size)(MPI_Comm, int *)
      = is_inter ? MPI_Comm_remote_size : MPI_Comm_size;
    xt_mpi_call(get_comm_size(comm, &comm_size), comm);
    xt_quicksort_msg_param_mix(msgs, nmsg, comm_rank, comm_size);
  }

  for (size_t i = 1; i < nmsg; ++i) {

    if ((msgs[i-1].rank & INT_MAX) == (msgs[i].rank & INT_MAX)
        && MSG_DIR(msgs[i]) == SEND) {

      struct Xt_msg_param temp = msgs[i-1];
      msgs[i-1] = msgs[i];
      msgs[i] = temp;
      i++;
    }
  }

  return (Xt_exchanger)exchanger;
}

static Xt_exchanger
xt_exchanger_mix_isend_irecv_copy(Xt_exchanger exchanger,
                                  MPI_Comm new_comm, int new_tag_offset)
{
  Xt_exchanger_mix_isend_irecv exchanger_msr =
    (Xt_exchanger_mix_isend_irecv)exchanger;
  size_t nmsg = (size_t)exchanger_msr->n;
  struct Xt_config_ config = xt_default_config;
  XT_CONFIG_SET_EXCH_SEND_COMM_MODE(&config, exchanger_msr->send_mode);
  /* fixme: needs to use custom config */
  Xt_exchanger_mix_isend_irecv exchanger_copy
    = xt_exchanger_mix_isend_irecv_alloc(nmsg, &config);
  exchanger_copy->comm = new_comm;
  exchanger_copy->tag_offset = new_tag_offset;
  exchanger_copy->vtable = exchanger_msr->vtable;
  xt_msg_params_copy(nmsg, exchanger_msr->msgs,
                     exchanger_copy->msgs, new_comm, true);
  return (Xt_exchanger)exchanger_copy;
}

enum { max_on_stack_req = 16 };

static void
xt_exchanger_mix_isend_irecv_delete(Xt_exchanger exchanger)
{
  Xt_exchanger_mix_isend_irecv exchanger_msr =
    (Xt_exchanger_mix_isend_irecv)exchanger;
  xt_msg_params_destruct((size_t)exchanger_msr->n, exchanger_msr->msgs,
                         exchanger_msr->comm);
  free(exchanger_msr);
}

static inline void
msgs_to_reqs(size_t nmsg,
             const struct Xt_msg_param *restrict msgs,
             const void *src_data, void *dst_data,
             MPI_Request *requests,
             Xt_exchanger_mix_isend_irecv exchanger)
{
  typedef int (*ifp)(void *buf, int count, MPI_Datatype datatype, int dest,
                     int tag, MPI_Comm comm, MPI_Request *request);
#if XT_ENABLE_ALTERNATIVE_EXCHANGER_SEND_MODES
  ifp a_send = (ifp)xt_send_func_tab[0][exchanger->send_mode];
#else
#  define a_send MPI_Isend
#endif
  for (size_t i = 0; i < nmsg; ++i) {
    ifp op = MSG_DIR(msgs[i]) == SEND ? (ifp)a_send : (ifp)MPI_Irecv;
    unsigned char *data = MSG_DIR(msgs[i]) == SEND
      ? (void *)(intptr_t)src_data : dst_data;
    xt_mpi_call(op(data + msgs[i].displacement, msgs[i].count, msgs[i].datatype,
                   msgs[i].rank & INT_MAX,
                   exchanger->tag_offset + xt_mpi_tag_exchange_msg,
                   exchanger->comm, requests+i), exchanger->comm);
  }
#if XT_ENABLE_ALTERNATIVE_EXCHANGER_SEND_MODES
#  undef a_send
#endif
}


static void
xt_exchanger_mix_isend_irecv_s_exchange(Xt_exchanger exchanger,
                                        const void * src_data,
                                        void *dst_data)
{
  Xt_exchanger_mix_isend_irecv exchanger_msr =
    (Xt_exchanger_mix_isend_irecv)exchanger;

  if (exchanger_msr->n > 0) {
    size_t nmsg = (size_t)exchanger_msr->n;
    MPI_Request req_buf[max_on_stack_req];
    MPI_Request *requests
      = nmsg <= max_on_stack_req
      ? req_buf : xmalloc(nmsg * sizeof (*requests));
    msgs_to_reqs((size_t)nmsg, exchanger_msr->msgs,
                 src_data, dst_data, requests,
                 exchanger_msr);
    xt_mpi_call(MPI_Waitall((int)nmsg, requests, MPI_STATUSES_IGNORE),
                exchanger_msr->comm);
    if (requests != req_buf)
      free(requests);
  }
}

static void xt_exchanger_mix_isend_irecv_a_exchange(
  Xt_exchanger exchanger, const void *src_data, void *dst_data,
  Xt_request *request) {

  Xt_exchanger_mix_isend_irecv exchanger_msr =
    (Xt_exchanger_mix_isend_irecv)exchanger;

  if (exchanger_msr->n > 0) {
    size_t nmsg = (size_t)exchanger_msr->n;
    struct Xt_config_ conf = xt_default_config;
    xt_config_set_redist_mthread_mode(&conf, XT_MT_NONE);
    Xt_request requests
      = xt_request_msgs_alloc((int)nmsg, exchanger_msr->comm, &conf);
    MPI_Request *requests_
      = xt_request_msgs_get_req_ptr(requests);
    msgs_to_reqs(nmsg, exchanger_msr->msgs,
                 src_data, dst_data, requests_,
                 exchanger_msr);
    *request = requests;
  } else
    *request = XT_REQUEST_NULL;
}

#ifdef _OPENMP
static void
xt_exchanger_mix_isend_irecv_a_exchange_mt(Xt_exchanger exchanger,
                                           const void * src_data,
                                           void * dst_data,
                                           Xt_exchanger_thread_share shared_req)
{
  MPI_Request *requests
    = xt_request_msgs_get_req_ptr((Xt_request)shared_req);
  Xt_exchanger_mix_isend_irecv exchanger_msr =
    (Xt_exchanger_mix_isend_irecv)exchanger;
  size_t num_threads = (size_t)omp_get_num_threads(),
    tid = (size_t)omp_get_thread_num();
  size_t nmsg = (size_t)exchanger_msr->n,
    start = (nmsg * tid) / num_threads,
    nmsg_ = (nmsg * (tid+1)) / num_threads - start;
  msgs_to_reqs(nmsg_, exchanger_msr->msgs+start,
               src_data, dst_data, requests+start,
               exchanger_msr);
}

static size_t
xt_exchanger_mix_isend_irecv_get_num_omp_threads(Xt_exchanger exchanger)
{
  size_t num_threads = (size_t)omp_get_max_threads();
  size_t num_msg = (size_t)(((Xt_exchanger_mix_isend_irecv)exchanger)->n);
  return num_msg >= num_threads ? num_threads : num_msg;
}

static void
xt_exchanger_mix_isend_irecv_a_exchange_omp(Xt_exchanger exchanger,
                                            const void *src_data,
                                            void *dst_data,
                                            Xt_request *request)
{
  size_t num_threads_limit
    = xt_exchanger_mix_isend_irecv_get_num_omp_threads(exchanger);
  if (num_threads_limit) {
    struct Xt_config_ conf = xt_default_config;
    xt_config_set_redist_mthread_mode(&conf, XT_MT_OPENMP);
    Xt_exchanger_thread_share shared_req
      = xt_exchanger_mix_isend_irecv_create_thread_share(exchanger, &conf);
#pragma omp parallel num_threads(num_threads_limit)
    xt_exchanger_mix_isend_irecv_a_exchange_mt(exchanger, src_data, dst_data,
                                               shared_req);
    *request = (Xt_request)shared_req;
  } else
    *request = XT_REQUEST_NULL;
}

static void
xt_exchanger_mix_isend_irecv_s_exchange_mt(Xt_exchanger exchanger,
                                           const void *src_data,
                                           void *dst_data,
                                           Xt_exchanger_thread_share shared_req)
{
  MPI_Request *requests
    = xt_request_msgs_get_req_ptr((Xt_request)shared_req);
  Xt_exchanger_mix_isend_irecv exchanger_msr =
    (Xt_exchanger_mix_isend_irecv)exchanger;
  size_t num_threads = (size_t)omp_get_num_threads(),
    tid = (size_t)omp_get_thread_num();
  size_t nmsg = (size_t)exchanger_msr->n,
    start = (nmsg * tid) / num_threads,
    nmsg_ = (nmsg * (tid+1)) / num_threads - start;
  msgs_to_reqs(nmsg_, exchanger_msr->msgs+start,
               src_data, dst_data, requests+start,
               exchanger_msr);
  xt_mpi_call(MPI_Waitall((int)nmsg_, requests+start,
                          MPI_STATUSES_IGNORE), exchanger_msr->comm);
}

static void
xt_exchanger_mix_isend_irecv_s_exchange_omp(Xt_exchanger exchanger,
                                            const void *src_data,
                                            void *dst_data)
{
  size_t num_threads_limit
    = xt_exchanger_mix_isend_irecv_get_num_omp_threads(exchanger);
  if (num_threads_limit) {
    struct Xt_config_ conf = xt_default_config;
    xt_config_set_redist_mthread_mode(&conf, XT_MT_OPENMP);
    Xt_exchanger_thread_share shared_req
      = xt_exchanger_mix_isend_irecv_create_thread_share(exchanger, &conf);
#pragma omp parallel num_threads(num_threads_limit)
    xt_exchanger_mix_isend_irecv_s_exchange_mt(exchanger, src_data, dst_data,
                                               shared_req);
    free(shared_req);
  }
}
#endif

static Xt_exchanger_thread_share
xt_exchanger_mix_isend_irecv_create_thread_share(Xt_exchanger exchanger,
                                                 Xt_config config)
{
  Xt_exchanger_mix_isend_irecv exchanger_msr =
    (Xt_exchanger_mix_isend_irecv)exchanger;
  return (Xt_exchanger_thread_share)xt_request_msgs_alloc(exchanger_msr->n,
                                                          exchanger_msr->comm,
                                                          config);
}

static void
xt_exchanger_mix_isend_irecv_destroy_thread_share(
  Xt_exchanger exchanger, Xt_exchanger_thread_share thread_share)
{
  (void)exchanger;
  /* assert that thread_share contains no in-flight requests */
  free(thread_share);
}

static int
xt_exchanger_mix_isend_irecv_get_num_msg(Xt_exchanger exchanger,
                                         int direction)
{
  Xt_exchanger_mix_isend_irecv exchanger_msr =
    (Xt_exchanger_mix_isend_irecv)exchanger;
  size_t nmsg = 0, nmsg_all = (size_t)exchanger_msr->n;
  assert(direction >= 0 && direction <= 2);
  switch (direction) {
  case xt_direction_send:
  case xt_direction_recv:
    {
      const struct Xt_msg_param *restrict msgs = exchanger_msr->msgs;
      for (size_t i = 0; i < nmsg_all; ++i)
        nmsg += MSG_DIR(msgs[i]) == direction;
    }
    break;
  case 2:
    nmsg = nmsg_all;
    break;
  }
  return (int)nmsg;
}

static int
xt_exchanger_mix_isend_irecv_get_msg_ranks(Xt_exchanger exchanger,
                                           enum xt_msg_direction direction,
                                           int *restrict *ranks)
{
  Xt_exchanger_mix_isend_irecv exchanger_msr =
    (Xt_exchanger_mix_isend_irecv)exchanger;
  size_t nmsg = 0, nmsg_all = (size_t)exchanger_msr->n;
  const struct Xt_msg_param *restrict msgs = exchanger_msr->msgs;
  for (size_t i = 0; i < nmsg_all; ++i)
    nmsg += MSG_DIR(msgs[i]) == direction;
  int *restrict ranks_ = *ranks;
  if (!ranks_)
    ranks_ = *ranks = xmalloc(nmsg * sizeof (*ranks_));
  for (size_t i = 0, j = (size_t)-1; i < nmsg_all; ++i)
    if (MSG_DIR(msgs[i]) == direction)
      ranks_[++j] = msgs[i].rank & INT_MAX;
  return (int)nmsg;
}

static MPI_Datatype
xt_exchanger_mix_isend_irecv_get_MPI_Datatype(Xt_exchanger exchanger,
                                              int rank,
                                              enum xt_msg_direction direction,
                                              bool do_dup)
{
  Xt_exchanger_mix_isend_irecv exchanger_msr =
    (Xt_exchanger_mix_isend_irecv)exchanger;
  size_t nmsg = (size_t)exchanger_msr->n;
  struct Xt_msg_param *restrict msgs = exchanger_msr->msgs;
  MPI_Datatype datatype_copy = MPI_DATATYPE_NULL;
  if (direction == xt_direction_recv) rank |= ~INT_MAX;
  for (size_t i = 0; i < nmsg; ++i)
    if (msgs[i].rank == rank) {
      MPI_Datatype dt = msgs[i].datatype;
      if (do_dup && !xt_mpi_ddt_is_named(dt))
        xt_mpi_call(MPI_Type_dup(dt, &datatype_copy), exchanger_msr->comm);
      else
        datatype_copy = dt;
      break;
    }
  return datatype_copy;
}

static MPI_Aint
xt_exchanger_mix_isend_irecv_get_displacement(Xt_exchanger exchanger,
                                              int rank,
                                              enum xt_msg_direction direction)
{
  Xt_exchanger_mix_isend_irecv exchanger_msr =
    (Xt_exchanger_mix_isend_irecv)exchanger;
  size_t nmsg = (size_t)exchanger_msr->n;
  struct Xt_msg_param *restrict msgs = exchanger_msr->msgs;
  MPI_Aint displacement = -1;
  if (direction == xt_direction_recv) rank |= ~INT_MAX;
  for (size_t i = 0; i < nmsg; ++i)
    if (msgs[i].rank == rank) {
      displacement = msgs[i].displacement;
      break;
    }
  return displacement;
}

static int
xt_exchanger_mix_isend_irecv_get_count(Xt_exchanger exchanger,
                                       int rank,
                                       enum xt_msg_direction direction)
{
  Xt_exchanger_mix_isend_irecv exchanger_msr =
    (Xt_exchanger_mix_isend_irecv)exchanger;
  size_t nmsg = (size_t)exchanger_msr->n;
  struct Xt_msg_param *restrict msgs = exchanger_msr->msgs;
  int count = -1;
  if (direction == xt_direction_recv) rank |= ~INT_MAX;
  for (size_t i = 0; i < nmsg; ++i)
    if (msgs[i].rank == rank) {
      count = msgs[i].count;
      break;
    }
  return count;
}

typedef struct Xt_exchanger_msr_iter_ *Xt_exchanger_msr_iter;

struct Xt_exchanger_msr_iter_ {
  const struct xt_exchanger_iter_vtable *vtable;
  Xt_exchanger_mix_isend_irecv exchanger;
  unsigned msg_ofs;
  uint32_t config_flags;
};

static int
xt_exchanger_mix_isend_irecv_iter_next(Xt_exchanger_iter);
static void
xt_exchanger_mix_isend_irecv_iter_delete(Xt_exchanger_iter);
static MPI_Datatype
xt_exchanger_mix_isend_irecv_iter_get_MPI_Datatype(Xt_exchanger_iter iter);
static int
xt_exchanger_mix_isend_irecv_iter_get_rank(Xt_exchanger_iter iter);
static int
xt_exchanger_mix_isend_irecv_iter_get_count(Xt_exchanger_iter iter);
static int
xt_exchanger_mix_isend_irecv_iter_get_direction(Xt_exchanger_iter iter);
static MPI_Aint
xt_exchanger_mix_isend_irecv_iter_get_displacement(Xt_exchanger_iter iter);

static const struct xt_exchanger_iter_vtable
xt_exchanger_mix_isend_irecv_iterator_vtable = {
  .next = xt_exchanger_mix_isend_irecv_iter_next,
  .delete = xt_exchanger_mix_isend_irecv_iter_delete,
  .get_MPI_Datatype = xt_exchanger_mix_isend_irecv_iter_get_MPI_Datatype,
  .get_rank = xt_exchanger_mix_isend_irecv_iter_get_rank,
  .get_count = xt_exchanger_mix_isend_irecv_iter_get_count,
  .get_direction = xt_exchanger_mix_isend_irecv_iter_get_direction,
  .get_displacement = xt_exchanger_mix_isend_irecv_iter_get_displacement,
};


static Xt_exchanger_iter
xt_exchanger_mix_isend_irecv_get_iterator(Xt_exchanger exchanger,
                                          Xt_config config)
{
  Xt_exchanger_mix_isend_irecv exchanger_msr
    = (Xt_exchanger_mix_isend_irecv)exchanger;
  Xt_exchanger_msr_iter iter_msr;
  if (exchanger_msr->n) {
    iter_msr = xmalloc(sizeof (*iter_msr));
    iter_msr->vtable = &xt_exchanger_mix_isend_irecv_iterator_vtable;
    iter_msr->exchanger = exchanger_msr;
    iter_msr->msg_ofs = 0;
    iter_msr->config_flags = config->flags;
  } else
    iter_msr = NULL;
  return (Xt_exchanger_iter)iter_msr;
}

static void
xt_exchanger_mix_isend_irecv_iter_delete(Xt_exchanger_iter iter)
{
  free(iter);
}

static int
xt_exchanger_mix_isend_irecv_iter_next(Xt_exchanger_iter iter)
{
  Xt_exchanger_msr_iter iter_msr = (Xt_exchanger_msr_iter)iter;
  unsigned msg_ofs = iter_msr->msg_ofs,
    nmsg = (unsigned)iter_msr->exchanger->n;
  if (msg_ofs+1 <= nmsg) {
    iter_msr->msg_ofs = ++msg_ofs;
    return msg_ofs < nmsg;
  } else
    return 0;
}

static MPI_Datatype
xt_exchanger_mix_isend_irecv_iter_get_MPI_Datatype(Xt_exchanger_iter iter)
{
  Xt_exchanger_msr_iter iter_msr = (Xt_exchanger_msr_iter)iter;
  const struct Xt_msg_param *msgs = iter_msr->exchanger->msgs;
  unsigned msg_ofs = iter_msr->msg_ofs,
    nmsg = (unsigned)iter_msr->exchanger->n;
  MPI_Datatype datatype_copy = MPI_DATATYPE_NULL;
  if (msg_ofs < nmsg) {
    MPI_Datatype dt = msgs[msg_ofs].datatype;
    if (iter_msr->config_flags & exch_no_dt_dup
        || xt_mpi_ddt_is_named(dt))
      datatype_copy = dt;
    else
      xt_mpi_call(MPI_Type_dup(dt, &datatype_copy), iter_msr->exchanger->comm);
  }
  return datatype_copy;
}

static int
xt_exchanger_mix_isend_irecv_iter_get_rank(Xt_exchanger_iter iter)
{
  Xt_exchanger_msr_iter iter_msr = (Xt_exchanger_msr_iter)iter;
  const struct Xt_msg_param *msgs = iter_msr->exchanger->msgs;
  unsigned msg_ofs = iter_msr->msg_ofs,
    nmsg = (unsigned)iter_msr->exchanger->n;
  return msg_ofs < nmsg ? (msgs[msg_ofs].rank & INT_MAX) : -1;
}

static int
xt_exchanger_mix_isend_irecv_iter_get_count(Xt_exchanger_iter iter)
{
  Xt_exchanger_msr_iter iter_msr = (Xt_exchanger_msr_iter)iter;
  const struct Xt_msg_param *msgs = iter_msr->exchanger->msgs;
  unsigned msg_ofs = iter_msr->msg_ofs,
    nmsg = (unsigned)iter_msr->exchanger->n;
  return msg_ofs < nmsg ? msgs[msg_ofs].count : -1;
}

static int
xt_exchanger_mix_isend_irecv_iter_get_direction(Xt_exchanger_iter iter)
{
  Xt_exchanger_msr_iter iter_msr = (Xt_exchanger_msr_iter)iter;
  const struct Xt_msg_param *msgs = iter_msr->exchanger->msgs;
  unsigned msg_ofs = iter_msr->msg_ofs,
    nmsg = (unsigned)iter_msr->exchanger->n;
  if (msg_ofs < nmsg)
    return MSG_DIR(msgs[msg_ofs]);
  else
    return -1;
}

static MPI_Aint
xt_exchanger_mix_isend_irecv_iter_get_displacement(Xt_exchanger_iter iter)
{
  Xt_exchanger_msr_iter iter_msr = (Xt_exchanger_msr_iter)iter;
  const struct Xt_msg_param *msgs = iter_msr->exchanger->msgs;
  unsigned msg_ofs = iter_msr->msg_ofs,
    nmsg = (unsigned)iter_msr->exchanger->n;
  return msg_ofs < nmsg ? msgs[msg_ofs].displacement : -1;
}


/*
 * Local Variables:
 * c-basic-offset: 2
 * coding: utf-8
 * indent-tabs-mode: nil
 * show-trailing-whitespace: t
 * require-trailing-newline: t
 * End:
 */
