/* globus_mp_mpi.h.  Generated automatically by configure.  */
/*
 * globus_mp_mpi.h
 *
 * MPI (Message Passing Interface) protocol configuration
 *
 * rcsid = "$Header: /home/globdev/CVS/globus-packages/mp/source/globus_mp_mpi.h.in,v 1.2 2001/12/21 17:24:49 link Exp $"
 */



#ifndef GLOBUS_INCLUDE_GLOBUS_MP_MPI
#define GLOBUS_INCLUDE_GLOBUS_MP_MPI

#ifndef EXTERN_C_BEGIN
#    ifdef __cplusplus
#        define EXTERN_C_BEGIN extern "C" {
#        define EXTERN_C_END }
#    else
#        define EXTERN_C_BEGIN
#        define EXTERN_C_END
#    endif
#endif

EXTERN_C_BEGIN

/* #undef GLOBUS_MP_HAS_MPI_PROTO */

#ifdef GLOBUS_MP_HAS_MPI_PROTO

/* nexus uses HAVE_MPI_PROTO */
#define HAVE_MPI_PROTO 1
#define GLOBUS_USING_THIS_MP_PROTO  1

#include "globus_common.h"
#include "mpi.h"


#define GLOBUS_NEXUS_PROTO_TYPE_MP NEXUS_PROTO_TYPE_MPI

#define GLOBUS_NEXUS_MP_PROTOCOL_INFO      globus_i_nexus_pr_mpi_info
#define GLOBUS_NEXUS_MP_PROTOCOL_TYPE      globus_i_nexus_pr_mpi_type
#define GLOBUS_NEXUS_MP_PROTOCOL_COUNT     globus_i_nexus_pr_mpi_count
#define GLOBUS_NEXUS_MP_PROTOCOL_SP_MATCH  globus_i_nexus_pr_mpi_sp_match


/*
 * Module activation declaration
 */
extern globus_module_descriptor_t	globus_i_mp_mpi_module;

#define GLOBUS_MP_MPI_MODULE (&globus_i_mp_mpi_module)


/*
 * GLOBUS_MP_PROTO_IS_THREAD_SAFE
 *
 * It is advantageous if an MP module be thread safe, since this
 * allows you to simply create a handler thread which blocks on a
 * receive, and automatically wakes up and handles the message when
 * one arrives.
 *
 * In order for an MP module to be considered "thread safe", the
 * following must be true:
 *   1) One thread can be doing a blocking receive, while other threads
 *	are simultaneously doing sends.
 *   2) A thread can send a message to its own node, which will
 *	be received by the blocking receive in the handler thread.
 *
 * If these conditions are true, then #define GLOBUS_MP_PROTO_IS_THREAD_SAFE
 */
/*
#define GLOBUS_MP_PROTO_IS_THREAD_SAFE
*/

/*
 * globus_mp_destination_t
 *
 * This encapsulates the information about a node to which we
 * may send messages.
 */
typedef int globus_mp_destination_t;

/*
 * globus_mp_send_status_t
 * globus_mp_receive_status_t
 */
typedef MPI_Request globus_mp_send_status_t;
typedef MPI_Request globus_mp_receive_status_t;

/*
 * communicators (globus_mp_communicator_t)
 */
typedef MPI_Comm globus_mp_communicator_t;
#define GLOBUS_MP_MPI_MSG_TYPE		42

/*
 * GLOBUS_MP_DEFAULT_STORAGE_SIZE
 *
 * This is the default buffer size that will be used on sends and
 * receives to avoid extraneous mallocs and locks.
 */
#define GLOBUS_MP_DEFAULT_STORAGE_SIZE 32736

/*
 * GLOBUS_MP_BIG_MESSAGE_MIN_SIZE
 *
 * Any message >= to this size will be sent using a two message
 * protocol.  A first, small message is sent with the size,
 * followed by a second message with the actual data.
 */
#define GLOBUS_MP_BIG_MESSAGE_MIN_SIZE 1500


/*
 * GLOBUS_MP_INITIALIZE()
 *
 * Do any initialization required before calling any other
 * GLOBUS_MP_ functions.
 */
#define GLOBUS_MP_INITIALIZE()					\
{								\
    globus_module_activate(GLOBUS_MP_MPI_MODULE);		\
}


/*
 * GLOBUS_MP_INIT_NODE_INFO(globus_mp_destination_t Mynode, int N_nodes)
 *
 * Set Mynode to the globus_mp_destionation_t info for my node.
 * Set N_nodes the the number of nodes.
 */
#define GLOBUS_MP_INIT_NODE_INFO(My_node, N_nodes)	\
{							\
     MPI_Comm_rank(MPI_COMM_WORLD, &(My_node));		\
     MPI_Comm_size(MPI_COMM_WORLD, &(N_nodes));		\
}


/*
 * GLOBUS_MP_NODE_SHUTDOWN()
 *
 * If a terminating process needs to do something to map itself out
 * of the set of processes, this macro should do it.
 */
#define GLOBUS_MP_NODE_SHUTDOWN()			\
{							\
    globus_module_deactivate(GLOBUS_MP_MPI_MODULE);	\
}


/*
 * GLOBUS_MP_ABORT()
 *
 * Special statements to abort all processes.
 */
#define GLOBUS_MP_ABORT()			\
{						\
    MPI_Abort(MPI_COMM_WORLD,1);		\
}


/*
 * GLOBUS_MP_COMMUNICATOR_ALLOC(globus_mp_communicator_t CPP_Comm)
 *
 * Create a new communicator, setting CPP_Comm with its indentifier.
 */
#define GLOBUS_MP_COMMUNICATOR_ALLOC(CPP_Comm)			\
{								\
    MPI_Comm_dup(MPI_COMM_WORLD, &(CPP_Comm));			\
}


/*
 * GLOBUS_MP_COMMUNICATOR_FREE(globus_mp_communicator_t CPP_Comm)
 *
 * Free resources used by the specified communicator.
 */
#define GLOBUS_MP_COMMUNICATOR_FREE(CPP_Comm)	\
{						\
    MPI_Comm_free(&(CPP_Comm));			\
}

/*
 * GLOBUS_MP_IS_MASTER_NODE(globus_mp_destination_t dest, globus_bool_t result)
 */
#define GLOBUS_MP_IS_MASTER_NODE(dest, result)	\
{						\
    if ((dest) == 0)				\
	result = GLOBUS_TRUE;			\
    else					\
	result = GLOBUS_FALSE;			\
}

/*
 * GLOBUS_MP_SET_DESTINATION(globus_mp_destination_t dest, int node_num)
 */
#define GLOBUS_MP_SET_DESTINATION(dest, node_num)	\
    ((dest) = (node_num))

/*
 * GLOBUS_MP_POST_RECEIVE(globus_mp_communicator_t CPP_Comm,
 *		          Func,
 *		          globus__byte_t *Buf,
 *		          unsigned long Buf_size,
 *		          globus_mp_receive_status_t Status,
 *                        int Error)
 *
 * Post a receive into Buf with a maximum size of Buf_size.
 * GLOBUS_MP_RECEIVE_STATUS() and GLOBUS_MP_RECEIVE_WAIT() should be used
 * with Status to determine completion of the receive.
 */
#define GLOBUS_MP_POST_RECEIVE(CPP_Comm, Func, Buf, Buf_size, Status, Error) \
{									     \
    if (MPI_Irecv((char *) (Buf),					     \
		  (int) (Buf_size),					     \
		  MPI_BYTE,						     \
		  MPI_ANY_SOURCE,					     \
		  GLOBUS_MP_MPI_MSG_TYPE,				     \
		  (CPP_Comm),						     \
		  &(Status)) == MPI_SUCCESS)				     \
    {									     \
	  (Error) = GLOBUS_SUCCESS;					     \
    }									     \
    else								     \
    {									     \
	  (Error) = GLOBUS_FAILURE;					     \
    }									     \
}


/*
 * GLOBUS_MP_RECEIVE_STATUS(Func,
 *                          globus_mp_receive_status_t Status,
 *                          nexus_bool_t Done,
 *                          int Error)
 *
 * See if the receive described by Status has completed.  If so, then
 * set Done=GLOBUS_TRUE. else set Done=GLOBUS_FALSE.
 */
#define GLOBUS_MP_RECEIVE_STATUS(Func, Status, Done, Error)	\
{								\
    int mpi_done;						\
    MPI_Status mpi_status;					\
    if ((MPI_Test(&(Status),					\
		  &mpi_done,					\
		  &mpi_status)) == MPI_SUCCESS)			\
    {								\
	(Done) = (mpi_done ? GLOBUS_TRUE : GLOBUS_FALSE);	\
        (Error) = GLOBUS_SUCCESS;				\
    }								\
    else							\
    {								\
        (Error) = GLOBUS_FAILURE;				\
    }								\
}


/*
 * GLOBUS_MP_RECEIVE_WAIT(Func,
 *                   	  globus_mp_receive_status_t Status,
 *		     	  int CPP_nbytes_ptr,
 *                   	  nexus_bool_t Done,
 *                        int Error)
 *
 * Wait for the receive described by Status to complete.
 * If it completes successfully, then
 * set Done=GLOBUS_TRUE. else set Done=GLOBUS_FALSE.
 */
#define GLOBUS_MP_RECEIVE_WAIT(Func, Status, CPP_nbytes_ptr, Done, Error) \
{									  \
    MPI_Status				 CPP_mpi_status;		  \
									  \
    if (MPI_Wait(&(Status), &CPP_mpi_status) == MPI_SUCCESS)		  \
    {									  \
        int *				CPPl_nbytes_ptr;		  \
									  \
        CPPl_nbytes_ptr = (CPP_nbytes_ptr);				  \
        if (CPPl_nbytes_ptr != NULL)					  \
	{								  \
	    MPI_Get_count(&CPP_mpi_status, MPI_BYTE, CPPl_nbytes_ptr);	  \
	}								  \
									  \
	(Done) = GLOBUS_TRUE;						  \
        (Error) = GLOBUS_SUCCESS;					  \
    }									  \
    else								  \
    {									  \
        (Error) = GLOBUS_FAILURE;					  \
    }									  \
}

/*
 * GLOBUS_MP_RECEIVE_CANCEL(globus_mp_receive_status_t Status,
 *                          int Error)
 *
 * Cancel a previously posted receive.
 */
#define GLOBUS_MP_RECEIVE_CANCEL(Status, Error)	\
{						\
    if (MPI_Cancel(&Status) == MPI_SUCCESS)	\
    {						\
        (Error) = GLOBUS_SUCCESS;		\
    }						\
    else					\
    {						\
        (Error) = GLOBUS_FAILURE;		\
    }						\
}


/*
 * GLOBUS_MP_SEND(globus_mp_communicator_t CPP_Comm,
 *                globus_mp_destination_t Dest,
 *                nexus_byte_t *Buf,
 *                unsigned long Buf_size,
 *                globus_mp_send_status_t Status,
 *                int Error)
 *
 * Send Buf, of size Buf_size, to Dest.
 * The completion of the send will be checked by calling
 * GLOBUS_MP_SEND_STATUS() using the Status argument.
 */
#define GLOBUS_MP_SEND(CPP_Comm, Dest, Buf, Buf_size, Status, Error)	\
{ 									\
    if (MPI_Isend((char *) (Buf), 					\
		  (int) (Buf_size), 					\
		  MPI_BYTE, 						\
		  (Dest), 						\
		  GLOBUS_MP_MPI_MSG_TYPE,				\
		  (CPP_Comm), 						\
		  &(Status)) == MPI_SUCCESS) 				\
    {									\
        (Error) = GLOBUS_SUCCESS;					\
    }									\
    else								\
    {									\
        (Error) = GLOBUS_FAILURE;					\
    }									\
}


/*
 * GLOBUS_MP_SEND_STATUS(globus_mp_send_status_t Status,
 *                       nexus_bool_t Done,
 *                       int Error)
 *
 * See if the send described by Status has completed.
 * If so, set Done=GLOBUS_TRUE, else set Done=GLOBUS_FALSE.
 */
#define GLOBUS_MP_SEND_STATUS(Status, Done, Error)			 \
{									 \
    int mpi_done;							 \
    MPI_Status mpi_status;						 \
    if (MPI_Test(&(Status),						 \
		 &mpi_done,						 \
		 &mpi_status) == MPI_SUCCESS)				 \
    {									 \
	(Done) = (mpi_done ? GLOBUS_TRUE : GLOBUS_FALSE);		 \
        (Error) = GLOBUS_SUCCESS;					 \
    }									 \
    else								 \
    {									 \
        (Error) = GLOBUS_FAILURE;					 \
    }									 \
}


/*
 * GLOBUS_MP_COMPARE_DESTINATIONS(globus_mp_destination_t D1,
 *                                globus_mp_destination_t D2,
 *			          globus_bool_t Result)
 *
 * Set Result==GLOBUS_TRUE if globus_mp_destination_t's D1 and D2 are the same.
 */
#define GLOBUS_MP_COMPARE_DESTINATIONS(D1, D2, Result) \
    Result = (((D1) == (D2)) ? GLOBUS_TRUE : GLOBUS_FALSE)


/*
 * GLOBUS_MP_COPY_DESTINATION(globus_mp_destination_t To,
 *                            globus_mp_destination_t From)
 *
 * Copy globus_mp_destination_t 'From' to 'To'.
 */
#define GLOBUS_MP_COPY_DESTINATION(To, From) \
    (To) = (From)


/*
 * GLOBUS_MP_FREE_DESTINATION(globus_mp_destination_t Dest)
 *
 * Free up any memory in 'Dest' which was malloced.
 */
#define GLOBUS_MP_FREE_DESTINATION(Dest)


/*
 * GLOBUS_MP_HASH_DESTINATION(globus_mp_destination_t Dest, int Hash)
 *
 * Hash the destination information into a value
 * between 0 and PROTO_TABLE_SIZE.
 */
#define GLOBUS_MP_HASH_DESTINATION(Dest, Hash) \
    (Hash) = ((Dest) % PROTO_TABLE_SIZE)


/*
 * GLOBUS_MP_INIT_DESTINATION(globus_mp_destination_t Dest)
 *
 * Initialize the destination.
 */
#define GLOBUS_MP_INIT_DESTINATION(Dest)


/*
 * GLOBUS_MP_GET_MY_MI_PROTO_SIZE(int Size)
 *
 * Fillin Size with the number of bytes I need to store my_node
 * into the mi_proto byte array.
 */
#define GLOBUS_MP_GET_MY_MI_PROTO_SIZE(Size) (Size) = sizeof(int)


/*
 * GLOBUS_MP_GET_MY_MI_PROTO(nexus_byte_t *Array)
 *
 * Fillin my_node into the mi_proto byte array
 */
#define GLOBUS_MP_GET_MY_MI_PROTO(Array) \
{ \
    memcpy((Array), &my_node, sizeof(int)); \
}


/*
 * GLOBUS_MP_CONSTRUCT_FROM_MI_PROTO(globus_mp_destination_t Dest,
 *			             nexus_mi_proto_t *Mi_proto,
 *			             nexus_byte_t *   Array)
 *
 * Copy the needed elements from Array to Dest.
 */
#define GLOBUS_MP_CONSTRUCT_FROM_MI_PROTO(Dest, Mi_proto, Array) \
{ \
    memcpy(&(Dest), (Array), sizeof(int)); \
}


#else /* NOT HAVE_MPI_PROTO */

#define GLOBUS_NEXUS_PROTO_TYPE_MP NEXUS_PROTO_TYPE_MPI

#define GLOBUS_NEXUS_MP_PROTOCOL_INFO      globus_i_nexus_pr_mpi_info
#define GLOBUS_NEXUS_MP_PROTOCOL_TYPE      globus_i_nexus_pr_mpi_type
#define GLOBUS_NEXUS_MP_PROTOCOL_COUNT     globus_i_nexus_pr_mpi_count
#define GLOBUS_NEXUS_MP_PROTOCOL_SP_MATCH  globus_i_nexus_pr_mpi_sp_match


#endif /* HAVE_MPI_PROTO */

EXTERN_C_END

#endif /* GLOBUS_INCLUDE_GLOBUS_MP_MPI */
