mpiUtil.h

/*  _________________________________________________________________________
 *
 *  UTILIB: A utility library for developing portable C++ codes.
 *  Copyright (c) 2001, Sandia National Laboratories.
 *  This software is distributed under the GNU Lesser General Public License.
 *  For more information, see the README file in the top UTILIB directory.
 *  _________________________________________________________________________
 */

//
// mpiUtil.h
//
// Jonathan Eckstein
//
#ifdef __GNUC__
#pragma interface
#endif

#ifndef __mpiUtil_H
#define __mpiUtil_H

#include "utilib_dll.h"
#include "errmsg.h"
#include "_generic.h"
#include "logEvent.h"

#ifdef USING_MPI
#include "mpi.h"

//
// To compile code for logging of all messages.
//
#ifdef EVENT_LOGGING_PRESENT

#include "paramTable.h"

#define LOG_SEND(a,b,c,d)   if (messageLog()) logSend(a,b,c,d)
#define LOG_RECV(s)         if (messageLog()) logRecv(s)

#else

#define LOG_SEND(a,b,c,d)               // Nothing
#define LOG_RECV(s)                     // Nothing

#endif


class UTILIB_API uMPI 
{
public:

  static MPI_Comm comm;
  static int rank;
  static int size;
  static int ioProc;
  static int iDoIO;

  static int errorCode;

#ifdef EVENT_LOGGING_PRESENT

  CLASS_PARAMETER(messageLog);

  static void logSend(int dest,int tag,int count,MPI_Datatype t);
  static void logRecv(MPI_Status* status);

#endif
  
  static int running()
    {
      int running;
      MPI_Initialized(&running);
      return(running);
    };

  static int packSlop(int m)
    {
      return 2*m + 32;
    };

  static int sizeOf(MPI_Datatype t);

  static void init(int* argcP,char*** argv, MPI_Comm comm_=MPI_COMM_WORLD);
  
  static void done();

  static void barrier()
    {
      errorCode = MPI_Barrier(comm);
      if (errorCode)
        Warning(errmsg("MPI_Barrier failed, code %d",errorCode));
    }


  static void reduce(void* sendbuf,void* recvbuf,int count,
                     MPI_Datatype datatype,MPI_Op op,int root)
    {
      errorCode = MPI_Reduce(sendbuf,recvbuf,count,datatype,op,root,
                             comm);
      if (errorCode)
        Warning(errmsg("MPI_Reduce failed, code %d",errorCode));
    };


  static int sum(int value,int root = ioProc)
    {
      int result = 0;
      errorCode = MPI_Reduce(&value,&result,1,MPI_INT,MPI_SUM,root,
                             comm);
      if (errorCode)
        Warning(errmsg("MPI_Reduce failed, code %d",errorCode));
      return result;
    };


  static double sum(double value,int root = ioProc)
    {
      double result = 0;
      errorCode = MPI_Reduce(&value,&result,1,MPI_DOUBLE,MPI_SUM,root,
                             comm);
      if (errorCode)
        Warning(errmsg("MPI_Reduce failed, code %d",errorCode));
      return result;
    };


  static int max(int value,int root = ioProc)
    {
      int result = 0;
      errorCode = MPI_Reduce(&value,&result,1,MPI_INT,MPI_MAX,root,
                             comm);
      if (errorCode)
        Warning(errmsg("MPI_Reduce failed, code %d",errorCode));
      return result;
    };


  static double max(double value,int root = ioProc)
    {
      double result = 0;
      errorCode = MPI_Reduce(&value,&result,1,MPI_DOUBLE,MPI_MAX,root,
                             comm);
      if (errorCode)
        Warning(errmsg("MPI_Reduce failed, code %d",errorCode));
      return result;
    };


  static void broadcast(void* buffer,int count,MPI_Datatype datatype,int root)
    {
      errorCode = MPI_Bcast(buffer,count,datatype,root,comm);
      if (errorCode)
        Warning(errmsg("MPI_Broadcast failed, code %d",errorCode));
    };


  static void reduceCast(void* sendbuf,void* recvbuf,int count,
                         MPI_Datatype datatype,MPI_Op op)
    {
      reduce(sendbuf,recvbuf,count,datatype,op,0);
      broadcast(recvbuf,count,datatype,0);
    };
      

  static void isend(void* buffer,int count,MPI_Datatype datatype,int dest,
                    int tag,MPI_Request* request)
    {
      LOG_SEND(dest,tag,count,datatype);
      errorCode = MPI_Isend(buffer,count,datatype,dest,tag,comm,
                            request);
      if (errorCode)
        Warning(errmsg("MPI_Isend failed, code %d",errorCode));
    };


  static void isend(void* buffer,int count,MPI_Datatype datatype,int dest,
                    int tag)
    {
      MPI_Request request;
      isend(buffer,count,datatype,dest,tag,&request);
      requestFree(&request);
    };


  static void send(void* buffer,int count,MPI_Datatype datatype,int dest,
                   int tag)
    {
      LOG_SEND(dest,tag,count,datatype);
      errorCode = MPI_Send(buffer,count,datatype,dest,tag,comm);
      if (errorCode)
        Warning(errmsg("MPI_Send failed, code %d",errorCode));
    }
      

  static void requestFree(MPI_Request* request)
    {
      errorCode = MPI_Request_free(request);
      if (errorCode)
        Warning(errmsg("MPI_Request_free failed, code %d",errorCode));
    };


  static void issend(void* buffer,int count,MPI_Datatype datatype,int dest,
                     int tag,MPI_Request* request)
    {
      LOG_SEND(dest,tag,count,datatype);
      errorCode = MPI_Issend(buffer,count,datatype,dest,tag,comm,
                             request);
      if (errorCode)
        Warning(errmsg("MPI_Issend failed, code %d",errorCode));
    };


  static void irecv(void* buffer,int count,MPI_Datatype datatype,int source,
                    int tag,MPI_Request* request)
    {
      errorCode = MPI_Irecv(buffer,count,datatype,source,tag,comm,
                            request);
      if (errorCode)
        Warning(errmsg("MPI_Irecv failed, code %d",errorCode));
    };


  static void recv(void* buffer,int count,MPI_Datatype datatype,int source,
                   int tag,MPI_Status* status)
    {
      errorCode = MPI_Recv(buffer,count,datatype,source,tag,comm,
                            status);
      if (errorCode)
        Warning(errmsg("MPI_Recv failed, code %d",errorCode));
      LOG_RECV(status);
    };


  static int getCount(MPI_Status* status,MPI_Datatype datatype)
    {
      int count;
      errorCode = MPI_Get_count(status,datatype,&count);
      if (errorCode)
        Warning(errmsg("MPI_Get_count failed, code %d",errorCode));
      return count;
    };
  

  static void test(MPI_Request* request,int* flag,MPI_Status* status)
    {
      errorCode = MPI_Test(request,flag,status);
      if (errorCode)
        Warning(errmsg("MPI_Test failed, code %d",errorCode));
#ifdef EVENT_LOGGING_PRESENT
      if (*flag)
        LOG_RECV(status);
#endif
    };

  static int test(MPI_Request* request,MPI_Status* status)
    {
      int flag;
      test(request,&flag,status);
      return flag;
    };

  static int test(MPI_Request* request)
    {
      MPI_Status status;
      return test(request,&status);
    };

  static int testsome(int incount, MPI_Request* request_array,
                int& outcount, int* array_of_indeces, MPI_Status* status_array)
    {
      errorCode = MPI_Testsome(incount,request_array,&outcount,
                                array_of_indeces,status_array);
      if (errorCode != MPI_SUCCESS)
        Warning(errmsg("MPI_Testsome failed, code %d",errorCode));
#ifdef EVENT_LOGGING_PRESENT
      for(int i=0; i<outcount; i++)
        LOG_RECV(status_array+i);
#endif
      return (outcount > 0);
    };

  static void cancel(MPI_Request* request)
    {
#ifndef COUGAR
      errorCode = MPI_Cancel(request);
      if (errorCode)
        Warning(errmsg("MPI_Cancel failed, code %d",errorCode));
#else
      Warning("uMPI::cancel - MPI_Cancel is not yet implemented");
#endif
    };


  static void wait(MPI_Request* request,MPI_Status* status)
    {
      errorCode = MPI_Wait(request,status);
      if (errorCode)
        Warning(errmsg("MPI_Wait failed, code %d",errorCode));
      LOG_RECV(status);
    };


  static void killSendRequest(MPI_Request* request);
  static void killRecvRequest(MPI_Request* request);

};


#else


//  If no MPI, don't define anything except running(), which returns FALSE.

class uMPI 
{
public:
  static int running() { return FALSE; };
};
 

#endif

#endif