source: XIOS/dev/dev_trunk_omp/extern/src_ep_dev2/ep_lib.cpp @ 1776

#ifdef _usingEP
#include "ep_lib.hpp"
#include <mpi.h>
#include "ep_declaration.hpp"
#include <iostream>
#include <fstream>
#include "ep_mpi.hpp"

using namespace std;

std::list< ep_lib::MPI_Request* > * EP_PendingRequests = 0;
#pragma omp threadprivate(EP_PendingRequests)


std::map<std::pair<int, int>, MPI_Group* > * tag_group_map = 0;

std::map<int, std::pair<ep_lib::MPI_Comm*, std::pair<int, int> > > * tag_comm_map = 0;


MPI_Group MPI_GROUP_WORLD;

namespace ep_lib
{ 

  int tag_combine(int real_tag, int src, int dest)
  {
    int a = real_tag << 16;
    int b = src << 8;
    int c = dest;

    return a+b+c;
  }

  int get_ep_rank(MPI_Comm comm, int ep_rank_loc, int mpi_rank)
  {    
    int ep_rank = -1;
    for(std::map<int, std::pair<int, int> >::iterator it = comm->ep_rank_map->begin(); it != comm->ep_rank_map->end(); it++)
    {
      if(   ( it->second.first  == ep_rank_loc )
         && ( it->second.second == mpi_rank ) )
      {
        ep_rank = it->first;
      }
    }

    if(comm->is_intercomm)
    {
      for(std::map<int, int>::iterator it = comm->inter_rank_map->begin(); it!=comm->inter_rank_map->end(); it++)
      {
        if(it->second == ep_rank)
        {
          ep_rank = it->first;
        }
      }
    }
    if(ep_rank >= 0) return ep_rank;
    else
    {
      printf("rank not find, intercomm = %d, ep_rank_loc = %d, mpi_rank = %d\n", comm->is_intercomm, ep_rank_loc, mpi_rank);
      return MPI_Abort(comm, 0);
    }
  }

  int get_ep_rank2(MPI_Comm comm, int ep_rank_loc, int mpi_rank)
  {    
    for(std::map<int, std::pair<int, int> >::iterator it = comm->ep_rank_map->begin(); it != comm->ep_rank_map->end(); it++)
    {
      if(   ( it->second.first  == ep_rank_loc )
         && ( it->second.second == mpi_rank ) )
      {
        return it->first;
      }
    }
    printf("rank not find for EP_intracomm\n");
    return MPI_Abort(comm, 0);
  }
  

  int MPI_Get_count(const MPI_Status *status, MPI_Datatype datatype, int *count)
  {
    return ::MPI_Get_count(to_mpi_status_ptr(*status), to_mpi_type(datatype), count);
  }

  double MPI_Wtime()
  {
    return ::MPI_Wtime();
  }

  int MPI_Comm_test_inter(MPI_Comm comm, int *flag)
  {
    if(comm->is_ep) return *flag = comm->is_intercomm;
    else return ::MPI_Comm_test_inter(to_mpi_comm(comm->mpi_comm), flag);
  }

  void check_sum_send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, int type)
  {

    int src_rank;
    int int_count;
    ::MPI_Aint datasize, intsize, charsize, lb;
    
    ::MPI_Type_get_extent(*(static_cast< ::MPI_Datatype*>(datatype)), &lb, &datasize);
    ::MPI_Type_get_extent(*(static_cast< ::MPI_Datatype*>(MPI_CHAR)), &lb, &intsize);

    int_count = count * datasize / intsize ;

    char *buffer = static_cast< char* >(const_cast< void*> (buf));
    
    unsigned long sum = 0;
    for(int i = 0; i<int_count; i++)
      sum += *(buffer+i); 


    MPI_Comm_rank(comm, &src_rank);
    
    ofstream myfile;
    myfile.open ("send_log.txt", ios::app);
    if (myfile.is_open())
    {
      myfile << "type = " << type << " src = "<< src_rank<< " dest = "<< dest <<" tag = "<< tag << "  count = "<< count << " sum = "<< sum << "\n";
      myfile.close();  
    }
    else printf("Unable to open file\n");

  }


  void check_sum_recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, int type)
  {

    int dest_rank;
    int int_count;
    ::MPI_Aint datasize, intsize, charsize, lb;
    
    ::MPI_Type_get_extent(*(static_cast< ::MPI_Datatype*>(datatype)), &lb, &datasize);
    ::MPI_Type_get_extent(*(static_cast< ::MPI_Datatype*>(MPI_CHAR)), &lb, &intsize);

    int_count = count * datasize / intsize ;

    char *buffer = static_cast< char* >(buf);
    
    unsigned long sum = 0;
    for(int i = 0; i<int_count; i++)
      sum += *(buffer+i); 


    MPI_Comm_rank(comm, &dest_rank);
    
    ofstream myfile;
    myfile.open ("recv_log.txt", ios::app);
    if (myfile.is_open())
    {
      myfile << "type = " << type << " src = "<< src << " dest = "<< dest_rank <<" tag = "<< tag << "  count = "<< count << " sum = "<< sum << "\n";
      myfile.close();  
    }
    else printf("Unable to open file\n");


  }


  bool valid_type(MPI_Datatype datatype)
  {
    if(   datatype == MPI_INT 
       || datatype == MPI_FLOAT
       || datatype == MPI_DOUBLE 
       || datatype == MPI_CHAR 
       || datatype == MPI_LONG 
       || datatype == MPI_UNSIGNED_LONG
       || datatype == MPI_LONG_LONG)
    {
      return true;
    }

    return false;
  }


  bool valid_op(MPI_Op op)
  {
    if(   op == MPI_MAX 
       || op == MPI_MIN
       || op == MPI_SUM
       || op == MPI_LOR)
    {
      return true;
    }

    return false;
  }


}


MPI_Datatype to_mpi_type(ep_lib::MPI_Datatype type)
{
  return *static_cast< MPI_Datatype* >(type);
}

MPI_Op to_mpi_op(ep_lib::MPI_Op op)
{
  return *static_cast< MPI_Op* >(op);
}

MPI_Comm to_mpi_comm(void* comm)
{
  return *(static_cast< MPI_Comm* >(comm));
} 

MPI_Comm* to_mpi_comm_ptr(void* comm)
{
  return static_cast< MPI_Comm* >(comm);
}  

MPI_Message to_mpi_message(void* message)
{
  return *(static_cast< MPI_Message* >(message));
}

MPI_Message* to_mpi_message_ptr(ep_lib::MPI_Message message)
{
  return static_cast< MPI_Message* >(message->mpi_message);
}

MPI_Info to_mpi_info(ep_lib::MPI_Info info)
{
  return *(static_cast< MPI_Info* >(info->mpi_info));
}

MPI_Win to_mpi_win(void* win)
{
  return *(static_cast< MPI_Win* >(win));
}

MPI_Aint to_mpi_aint(ep_lib::MPI_Aint aint)
{
  return *(static_cast< MPI_Aint* >(aint.mpi_aint));
}

MPI_Status* to_mpi_status_ptr(ep_lib::MPI_Status status)
{
  return static_cast< MPI_Status* >(status.mpi_status);
}

MPI_Request* to_mpi_request_ptr(ep_lib::MPI_Request request)
{
  return static_cast< MPI_Request* >(request->mpi_request);
}

#endif