StarPU Handbook
|
Initialisation | |
#define | STARPU_USE_MPI |
int | starpu_mpi_init (int *argc, char ***argv, int initialize_mpi) |
int | starpu_mpi_initialize (void) |
int | starpu_mpi_initialize_extended (int *rank, int *world_size) |
int | starpu_mpi_shutdown (void) |
void | starpu_mpi_comm_amounts_retrieve (size_t *comm_amounts) |
Communication | |
int | starpu_mpi_send (starpu_data_handle_t data_handle, int dest, int mpi_tag, MPI_Comm comm) |
int | starpu_mpi_recv (starpu_data_handle_t data_handle, int source, int mpi_tag, MPI_Comm comm, MPI_Status *status) |
int | starpu_mpi_isend (starpu_data_handle_t data_handle, starpu_mpi_req *req, int dest, int mpi_tag, MPI_Comm comm) |
int | starpu_mpi_irecv (starpu_data_handle_t data_handle, starpu_mpi_req *req, int source, int mpi_tag, MPI_Comm comm) |
int | starpu_mpi_isend_detached (starpu_data_handle_t data_handle, int dest, int mpi_tag, MPI_Comm comm, void(*callback)(void *), void *arg) |
int | starpu_mpi_irecv_detached (starpu_data_handle_t data_handle, int source, int mpi_tag, MPI_Comm comm, void(*callback)(void *), void *arg) |
int | starpu_mpi_issend (starpu_data_handle_t data_handle, starpu_mpi_req *req, int dest, int mpi_tag, MPI_Comm comm) |
int | starpu_mpi_wait (starpu_mpi_req *req, MPI_Status *status) |
int | starpu_mpi_test (starpu_mpi_req *req, int *flag, MPI_Status *status) |
int | starpu_mpi_barrier (MPI_Comm comm) |
int | starpu_mpi_isend_detached_unlock_tag (starpu_data_handle_t data_handle, int dest, int mpi_tag, MPI_Comm comm, starpu_tag_t tag) |
int | starpu_mpi_irecv_detached_unlock_tag (starpu_data_handle_t data_handle, int source, int mpi_tag, MPI_Comm comm, starpu_tag_t tag) |
int | starpu_mpi_isend_array_detached_unlock_tag (unsigned array_size, starpu_data_handle_t *data_handle, int *dest, int *mpi_tag, MPI_Comm *comm, starpu_tag_t tag) |
int | starpu_mpi_irecv_array_detached_unlock_tag (unsigned array_size, starpu_data_handle_t *data_handle, int *source, int *mpi_tag, MPI_Comm *comm, starpu_tag_t tag) |
Communication Cache | |
int | starpu_mpi_cache_is_enabled () |
int | starpu_mpi_cache_set (int enabled) |
void | starpu_mpi_cache_flush (MPI_Comm comm, starpu_data_handle_t data_handle) |
void | starpu_mpi_cache_flush_all_data (MPI_Comm comm) |
MPI Insert Task | |
#define | STARPU_EXECUTE_ON_NODE |
#define | STARPU_EXECUTE_ON_DATA |
void | starpu_mpi_data_register_comm (starpu_data_handle_t data_handle, int tag, int rank, MPI_Comm comm) |
int | starpu_mpi_data_get_rank (starpu_data_handle_t handle) |
int | starpu_mpi_data_get_tag (starpu_data_handle_t handle) |
int | starpu_mpi_insert_task (MPI_Comm comm, struct starpu_codelet *codelet,...) |
void | starpu_mpi_get_data_on_node (MPI_Comm comm, starpu_data_handle_t data_handle, int node) |
void | starpu_mpi_get_data_on_node_detached (MPI_Comm comm, starpu_data_handle_t data_handle, int node, void(*callback)(void *), void *arg) |
Collective Operations | |
void | starpu_mpi_redux_data (MPI_Comm comm, starpu_data_handle_t data_handle) |
int | starpu_mpi_scatter_detached (starpu_data_handle_t *data_handles, int count, int root, MPI_Comm comm, void(*scallback)(void *), void *sarg, void(*rcallback)(void *), void *rarg) |
int | starpu_mpi_gather_detached (starpu_data_handle_t *data_handles, int count, int root, MPI_Comm comm, void(*scallback)(void *), void *sarg, void(*rcallback)(void *), void *rarg) |
#define STARPU_USE_MPI |
This macro is defined when StarPU has been installed with MPI support. It should be used in your code to detect the availability of MPI.
#define STARPU_EXECUTE_ON_NODE |
this macro is used when calling starpu_mpi_insert_task(), and must be followed by a integer value which specified the node on which to execute the codelet.
#define STARPU_EXECUTE_ON_DATA |
this macro is used when calling starpu_mpi_insert_task(), and must be followed by a data handle to specify that the node owning the given data will execute the codelet.
int starpu_mpi_init | ( | int * | argc, |
char *** | argv, | ||
int | initialize_mpi | ||
) |
Initializes the starpumpi library. initialize_mpi
indicates if MPI should be initialized or not by StarPU. If the value is not 0, MPI will be initialized by calling MPI_Init_Thread(argc, argv, MPI_THREAD_SERIALIZED, ...)
.
int starpu_mpi_initialize | ( | void | ) |
int starpu_mpi_initialize_extended | ( | int * | rank, |
int * | world_size | ||
) |
MPI_Init_Thread(argc, argv, MPI_THREAD_SERIALIZED, ...)
.int starpu_mpi_shutdown | ( | void | ) |
Cleans the starpumpi library. This must be called between calling starpu_mpi functions and starpu_shutdown(). MPI_Finalize() will be called if StarPU-MPI has been initialized by starpu_mpi_init().
void starpu_mpi_comm_amounts_retrieve | ( | size_t * | comm_amounts | ) |
Retrieve the current amount of communications from the current node in the array comm_amounts
which must have a size greater or equal to the world size. Communications statistics must be enabled (see STARPU_COMM_STATS).
int starpu_mpi_send | ( | starpu_data_handle_t | data_handle, |
int | dest, | ||
int | mpi_tag, | ||
MPI_Comm | comm | ||
) |
Performs a standard-mode, blocking send of data_handle
to the node dest
using the message tag mpi_tag
within the communicator comm
.
int starpu_mpi_recv | ( | starpu_data_handle_t | data_handle, |
int | source, | ||
int | mpi_tag, | ||
MPI_Comm | comm, | ||
MPI_Status * | status | ||
) |
Performs a standard-mode, blocking receive in data_handle
from the node source
using the message tag mpi_tag
within the communicator comm
.
int starpu_mpi_isend | ( | starpu_data_handle_t | data_handle, |
starpu_mpi_req * | req, | ||
int | dest, | ||
int | mpi_tag, | ||
MPI_Comm | comm | ||
) |
Posts a standard-mode, non blocking send of data_handle
to the node dest
using the message tag mpi_tag
within the communicator comm
. After the call, the pointer to the request req
can be used to test or to wait for the completion of the communication.
int starpu_mpi_irecv | ( | starpu_data_handle_t | data_handle, |
starpu_mpi_req * | req, | ||
int | source, | ||
int | mpi_tag, | ||
MPI_Comm | comm | ||
) |
Posts a nonblocking receive in data_handle
from the node source
using the message tag mpi_tag
within the communicator comm
. After the call, the pointer to the request req
can be used to test or to wait for the completion of the communication.
int starpu_mpi_isend_detached | ( | starpu_data_handle_t | data_handle, |
int | dest, | ||
int | mpi_tag, | ||
MPI_Comm | comm, | ||
void(*)(void *) | callback, | ||
void * | arg | ||
) |
Posts a standard-mode, non blocking send of data_handle
to the node dest
using the message tag mpi_tag
within the communicator comm
. On completion, the callback
function is called with the argument arg
. Similarly to the pthread detached functionality, when a detached communication completes, its resources are automatically released back to the system, there is no need to test or to wait for the completion of the request.
int starpu_mpi_irecv_detached | ( | starpu_data_handle_t | data_handle, |
int | source, | ||
int | mpi_tag, | ||
MPI_Comm | comm, | ||
void(*)(void *) | callback, | ||
void * | arg | ||
) |
Posts a nonblocking receive in data_handle
from the node source
using the message tag mpi_tag
within the communicator comm
. On completion, the callback
function is called with the argument arg
. Similarly to the pthread detached functionality, when a detached communication completes, its resources are automatically released back to the system, there is no need to test or to wait for the completion of the request.
int starpu_mpi_issend | ( | starpu_data_handle_t | data_handle, |
starpu_mpi_req * | req, | ||
int | dest, | ||
int | mpi_tag, | ||
MPI_Comm | comm | ||
) |
Performs a synchronous-mode, non-blocking send of data_handle
to the node dest
using the message tag mpi_tag
within the communicator comm
.
int starpu_mpi_issend_detached(starpu_data_handle_t data_handle, int dest, int mpi_tag, MPI_Comm comm, void (*callback)(void *), void *arg)
Performs a synchronous-mode, non-blocking send of data_handle
to the node dest
using the message tag mpi_tag
within the communicator comm
. On completion, the callback
function is called with the argument arg
. Similarly to the pthread detached functionality, when a detached communication completes, its resources are automatically released back to the system, there is no need to test or to wait for the completion of the request.
int starpu_mpi_wait | ( | starpu_mpi_req * | req, |
MPI_Status * | status | ||
) |
Returns when the operation identified by request req
is complete.
int starpu_mpi_test | ( | starpu_mpi_req * | req, |
int * | flag, | ||
MPI_Status * | status | ||
) |
If the operation identified by req
is complete, set flag
to 1. The status
object is set to contain information on the completed operation.
int starpu_mpi_barrier | ( | MPI_Comm | comm | ) |
Blocks the caller until all group members of the communicator comm
have called it.
int starpu_mpi_isend_detached_unlock_tag | ( | starpu_data_handle_t | data_handle, |
int | dest, | ||
int | mpi_tag, | ||
MPI_Comm | comm, | ||
starpu_tag_t | tag | ||
) |
Posts a standard-mode, non blocking send of data_handle
to the node dest
using the message tag mpi_tag
within the communicator comm
. On completion, tag
is unlocked.
int starpu_mpi_irecv_detached_unlock_tag | ( | starpu_data_handle_t | data_handle, |
int | source, | ||
int | mpi_tag, | ||
MPI_Comm | comm, | ||
starpu_tag_t | tag | ||
) |
Posts a nonblocking receive in data_handle
from the node source
using the message tag mpi_tag
within the communicator comm
. On completion, tag
is unlocked.
int starpu_mpi_isend_array_detached_unlock_tag | ( | unsigned | array_size, |
starpu_data_handle_t * | data_handle, | ||
int * | dest, | ||
int * | mpi_tag, | ||
MPI_Comm * | comm, | ||
starpu_tag_t | tag | ||
) |
Posts array_size
standard-mode, non blocking send. Each post sends the n-th data of the array data_handle
to the n-th node of the array dest
using the n-th message tag of the array mpi_tag
within the n-th communicator of the array comm
. On completion of the all the requests, tag
is unlocked.
int starpu_mpi_irecv_array_detached_unlock_tag | ( | unsigned | array_size, |
starpu_data_handle_t * | data_handle, | ||
int * | source, | ||
int * | mpi_tag, | ||
MPI_Comm * | comm, | ||
starpu_tag_t | tag | ||
) |
Posts array_size
nonblocking receive. Each post receives in the n-th data of the array data_handle
from the n-th node of the array source
using the n-th message tag of the array mpi_tag
within the n-th communicator of the array comm
. On completion of the all the requests, tag
is unlocked.
int starpu_mpi_cache_is_enabled | ( | ) |
Return 1 if the communication cache is enabled, 0 otherwise
int starpu_mpi_cache_set | ( | int | enabled | ) |
If enabled
is 1, enable the communication cache. Otherwise, clean the cache if it was enabled and disable it.
void starpu_mpi_cache_flush | ( | MPI_Comm | comm, |
starpu_data_handle_t | data_handle | ||
) |
Clear the send and receive communication cache for the data data_handle
and invalidate the value. The function has to be called synchronously by all the MPI nodes. The function does nothing if the cache mechanism is disabled (see STARPU_MPI_CACHE).
void starpu_mpi_cache_flush_all_data | ( | MPI_Comm | comm | ) |
Clear the send and receive communication cache for all data and invalidate their values. The function has to be called synchronously by all the MPI nodes. The function does nothing if the cache mechanism is disabled (see STARPU_MPI_CACHE).
int starpu_mpi_data_register_comm | ( | starpu_data_handle_t | handle, |
int | tag, | ||
int | rank, | ||
MPI_Comm | comm | ||
) |
Register to MPI a StarPU data handle with the given tag, rank and MPI communicator. It also automatically clears the MPI communication cache when unregistering the data.
int starpu_mpi_data_get_rank | ( | starpu_data_handle_t | handle | ) |
Return the rank of the given data.
int starpu_mpi_data_get_tag | ( | starpu_data_handle_t | handle | ) |
Return the tag of the given data.
int starpu_mpi_insert_task | ( | MPI_Comm | comm, |
struct starpu_codelet * | codelet, | ||
... | |||
) |
Create and submit a task corresponding to codelet with the following arguments. The argument list must be zero-terminated.
The arguments following the codelet are the same types as for the function starpu_insert_task(). Access modes for data can also be set with STARPU_SSEND to specify the data has to be sent using a synchronous and non-blocking mode (see starpu_mpi_issend()) The extra argument STARPU_EXECUTE_ON_NODE followed by an integer allows to specify the MPI node to execute the codelet. It is also possible to specify that the node owning a specific data will execute the codelet, by using STARPU_EXECUTE_ON_DATA followed by a data handle.
The internal algorithm is as follows:
The algorithm also includes a communication cache mechanism that allows not to send data twice to the same MPI node, unless the data has been modified. The cache can be disabled (see STARPU_MPI_CACHE).
void starpu_mpi_get_data_on_node | ( | MPI_Comm | comm, |
starpu_data_handle_t | data_handle, | ||
int | node | ||
) |
Transfer data data_handle
to MPI node node
, sending it from its owner if needed. At least the target node and the owner have to call the function.
void starpu_mpi_get_data_on_node_detached | ( | MPI_Comm | comm, |
starpu_data_handle_t | data_handle, | ||
int | node, | ||
void(*)(void *) | callback, | ||
void * | arg | ||
) |
Transfer data data_handle
to MPI node node
, sending it from its owner if needed. At least the target node and the owner have to call the function. On reception, the callback
function is called with the argument arg
.
void starpu_mpi_redux_data | ( | MPI_Comm | comm, |
starpu_data_handle_t | data_handle | ||
) |
Perform a reduction on the given data. All nodes send the data to its owner node which will perform a reduction.
int starpu_mpi_scatter_detached | ( | starpu_data_handle_t * | data_handles, |
int | count, | ||
int | root, | ||
MPI_Comm | comm, | ||
void(*)(void *) | scallback, | ||
void * | sarg, | ||
void(*)(void *) | rcallback, | ||
void * | rarg | ||
) |
Scatter data among processes of the communicator based on the ownership of the data. For each data of the array data_handles
, the process root
sends the data to the process owning this data. Processes receiving data must have valid data handles to receive them. On completion of the collective communication, the scallback
function is called with the argument sarg
on the process root
, the rcallback
function is called with the argument rarg
on any other process.
int starpu_mpi_gather_detached | ( | starpu_data_handle_t * | data_handles, |
int | count, | ||
int | root, | ||
MPI_Comm | comm, | ||
void(*)(void *) | scallback, | ||
void * | sarg, | ||
void(*)(void *) | rcallback, | ||
void * | rarg | ||
) |
Gather data from the different processes of the communicator onto the process root
. Each process owning data handle in the array data_handles
will send them to the process root
. The process root
must have valid data handles to receive the data. On completion of the collective communication, the rcallback
function is called with the argument rarg
on the process root, the scallback
function is called with the argument sarg
on any other process.