Skip to content

Refactor the request completion #1422

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 24, 2016
Merged

Refactor the request completion #1422

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b72337d
Remodel the request.
Oct 7, 2015
fab6683
Fix the condition to release the request.
bosilca Mar 2, 2016
875c69b
Removed ompi_request_lock from pml/ob1 and fix compilation error in UCX.
May 20, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions ompi/communicator/comm_request.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
* reseved.
* Copyright (c) 2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2004-2016 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -114,7 +117,7 @@ static int ompi_comm_request_progress (void)
/* don't call ompi_request_test_all as it causes a recursive call into opal_progress */
while (request_item->subreq_count) {
ompi_request_t *subreq = request_item->subreqs[request_item->subreq_count-1];
if (true == subreq->req_complete) {
if( REQUEST_COMPLETE(subreq) ) {
ompi_request_free (&subreq);
request_item->subreq_count--;
} else {
Expand Down Expand Up @@ -204,7 +207,7 @@ static int ompi_comm_request_free (struct ompi_request_t **ompi_req)
{
ompi_comm_request_t *request = (ompi_comm_request_t *) *ompi_req;

if (!(*ompi_req)->req_complete) {
if( !REQUEST_COMPLETE(*ompi_req) ) {
return MPI_ERR_REQUEST;
}

Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/libnbc/coll_libnbc_component.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* Copyright (c) 2004-2016 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
Expand Down Expand Up @@ -300,7 +300,7 @@ request_free(struct ompi_request_t **ompi_req)
ompi_coll_libnbc_request_t *request =
(ompi_coll_libnbc_request_t*) *ompi_req;

if (true != request->super.req_complete) {
if( !REQUEST_COMPLETE(&request->super) ) {
return MPI_ERR_REQUEST;
}

Expand Down
16 changes: 8 additions & 8 deletions ompi/mca/io/ompio/io_ompio_request.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2007 The University of Tennessee and The University
* Copyright (c) 2004-2016 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
Expand All @@ -29,7 +29,7 @@ static int mca_io_ompio_request_free ( struct ompi_request_t **req)
{
mca_ompio_request_t *ompio_req = ( mca_ompio_request_t *)*req;
if ( NULL != ompio_req->req_free_fn ) {
ompio_req->req_free_fn (ompio_req );
ompio_req->req_free_fn (ompio_req );
}
opal_list_remove_item (&mca_io_ompio_pending_requests, &ompio_req->req_item);

Expand Down Expand Up @@ -65,7 +65,7 @@ void mca_io_ompio_request_destruct(mca_ompio_request_t* req)
OMPI_REQUEST_FINI ( &(req->req_ompi));
OBJ_DESTRUCT (&req->req_item);
if ( NULL != req->req_data ) {
free (req->req_data);
free (req->req_data);
}

return;
Expand All @@ -79,16 +79,16 @@ int mca_io_ompio_component_progress ( void )

OPAL_LIST_FOREACH(litem, &mca_io_ompio_pending_requests, opal_list_item_t) {
req = GET_OMPIO_REQ_FROM_ITEM(litem);
if ( true == req->req_ompi.req_complete ) {
continue;
}
if( REQUEST_COMPLETE(&req->req_ompi) ) {
continue;
}
if ( NULL != req->req_progress_fn ) {
if ( req->req_progress_fn(req) ) {
completed++;
ompi_request_complete (&req->req_ompi, 1);
/* The fbtl progress function is expected to set the
** status elements
*/
* status elements
*/
}
}

Expand Down
5 changes: 4 additions & 1 deletion ompi/mca/osc/pt2pt/osc_pt2pt_request.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
* Copyright (c) 2011-2012 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2016 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -32,7 +35,7 @@ request_free(struct ompi_request_t **ompi_req)
ompi_osc_pt2pt_request_t *request =
(ompi_osc_pt2pt_request_t*) *ompi_req;

if (true != request->super.req_complete) {
if (REQUEST_COMPLETED != request->super.req_complete) {
return MPI_ERR_REQUEST;
}

Expand Down
5 changes: 4 additions & 1 deletion ompi/mca/osc/rdma/osc_rdma_request.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
* Copyright (c) 2011-2012 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2016 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -30,7 +33,7 @@ static int request_free(struct ompi_request_t **ompi_req)
ompi_osc_rdma_request_t *request =
(ompi_osc_rdma_request_t*) *ompi_req;

if (true != request->super.req_complete) {
if( REQUEST_COMPLETE(&request->super) ) {
return MPI_ERR_REQUEST;
}

Expand Down
6 changes: 3 additions & 3 deletions ompi/mca/pml/base/pml_base_recvreq.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2010 The University of Tennessee and The University
* Copyright (c) 2004-2016 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
Expand Down Expand Up @@ -78,7 +78,7 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_pml_base_recv_request_t);
(request)->req_base.req_sequence = 0; \
(request)->req_base.req_datatype = datatype; \
/* What about req_type ? */ \
(request)->req_base.req_pml_complete = OPAL_INT_TO_BOOL(persistent); \
(request)->req_base.req_pml_complete = false; \
(request)->req_base.req_free_called = false; \
}
/**
Expand All @@ -99,7 +99,7 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_pml_base_recv_request_t);
(request)->req_ompi.req_status._ucount = 0; \
(request)->req_ompi.req_status._cancelled = 0; \
\
(request)->req_ompi.req_complete = false; \
(request)->req_ompi.req_complete = REQUEST_PENDING; \
(request)->req_ompi.req_state = OMPI_REQUEST_ACTIVE; \
} while (0)

Expand Down
6 changes: 3 additions & 3 deletions ompi/mca/pml/base/pml_base_request.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2007 The University of Tennessee and The University
* Copyright (c) 2004-2016 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
Expand Down Expand Up @@ -62,11 +62,11 @@ struct mca_pml_base_request_t {

/* START: These fields have to match the definition of the mca_pml_cm_request_t */
ompi_request_t req_ompi; /**< base request */
volatile bool req_pml_complete; /**< flag indicating if the pt-2-pt layer is done with this request */
volatile int32_t req_pml_complete; /**< flag indicating if the pt-2-pt layer is done with this request */
volatile int32_t req_free_called; /**< flag indicating if the user has freed this request */
mca_pml_base_request_type_t req_type; /**< MPI request type - used for test */
struct ompi_communicator_t *req_comm; /**< communicator pointer */
struct ompi_datatype_t *req_datatype; /**< pointer to data type */
volatile bool req_free_called; /**< flag indicating if the user has freed this request */
opal_convertor_t req_convertor; /**< always need the convertor */
/* END: These field have to match the definition of the mca_pml_cm_request_t */

Expand Down
6 changes: 3 additions & 3 deletions ompi/mca/pml/base/pml_base_sendreq.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2007 The University of Tennessee and The University
* Copyright (c) 2004-2016 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
Expand Down Expand Up @@ -88,7 +88,7 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION( mca_pml_base_send_request_t );
(request)->req_base.req_tag = (int32_t)tag; \
(request)->req_base.req_comm = comm; \
/* (request)->req_base.req_proc is set on request allocation */ \
(request)->req_base.req_pml_complete = OPAL_INT_TO_BOOL(persistent); \
(request)->req_base.req_pml_complete = false; \
(request)->req_base.req_free_called = false; \
(request)->req_base.req_ompi.req_status._cancelled = 0; \
(request)->req_bytes_packed = 0; \
Expand Down Expand Up @@ -119,7 +119,7 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION( mca_pml_base_send_request_t );
#define MCA_PML_BASE_SEND_START( request ) \
do { \
(request)->req_pml_complete = false; \
(request)->req_ompi.req_complete = false; \
(request)->req_ompi.req_complete = REQUEST_PENDING; \
(request)->req_ompi.req_state = OMPI_REQUEST_ACTIVE; \
(request)->req_ompi.req_status._cancelled = 0; \
} while (0)
Expand Down
6 changes: 3 additions & 3 deletions ompi/mca/pml/cm/pml_cm_request.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2007 The University of Tennessee and The University
* Copyright (c) 2004-2016 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
Expand Down Expand Up @@ -42,11 +42,11 @@ struct mca_pml_cm_request_t {

/* START: These fields have to match the definition of the mca_pml_base_request_t */
ompi_request_t req_ompi; /**< base request */
volatile bool req_pml_complete; /**< flag indicating if the pt-2-pt layer is done with this request */
volatile int32_t req_pml_complete; /**< flag indicating if the pt-2-pt layer is done with this request */
volatile int32_t req_free_called; /**< flag indicating if the user has freed this request */
mca_pml_cm_request_type_t req_pml_type;
struct ompi_communicator_t *req_comm; /**< communicator pointer */
struct ompi_datatype_t *req_datatype; /**< pointer to data type */
volatile bool req_free_called; /**< flag indicating if the user has freed this request */
opal_convertor_t req_convertor; /**< convertor that describes the memory layout */
/* END: These fields have to match the definition of the mca_pml_base_request_t */
};
Expand Down
6 changes: 3 additions & 3 deletions ompi/mca/pml/ob1/pml_ob1_iprobe.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2013 The University of Tennessee and The University
* Copyright (c) 2004-2016 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
Expand Down Expand Up @@ -39,7 +39,7 @@ int mca_pml_ob1_iprobe(int src,
MCA_PML_OB1_RECV_REQUEST_INIT(&recvreq, NULL, 0, &ompi_mpi_char.dt, src, tag, comm, true);
MCA_PML_OB1_RECV_REQUEST_START(&recvreq);

if( recvreq.req_recv.req_base.req_ompi.req_complete == true ) {
if( REQUEST_COMPLETE( &(recvreq.req_recv.req_base.req_ompi)) ) {
if( NULL != status ) {
*status = recvreq.req_recv.req_base.req_ompi.req_status;
}
Expand Down Expand Up @@ -106,7 +106,7 @@ mca_pml_ob1_improbe(int src,
src, tag, comm, false);
MCA_PML_OB1_RECV_REQUEST_START(recvreq);

if( recvreq->req_recv.req_base.req_ompi.req_complete == true ) {
if( REQUEST_COMPLETE( &(recvreq->req_recv.req_base.req_ompi)) ) {
if( NULL != status ) {
*status = recvreq->req_recv.req_base.req_ompi.req_status;
}
Expand Down
40 changes: 18 additions & 22 deletions ompi/mca/pml/ob1/pml_ob1_recvreq.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,28 +71,26 @@ static int mca_pml_ob1_recv_request_free(struct ompi_request_t** request)
{
mca_pml_ob1_recv_request_t* recvreq = *(mca_pml_ob1_recv_request_t**)request;

assert( false == recvreq->req_recv.req_base.req_free_called );

OPAL_THREAD_LOCK(&ompi_request_lock);
recvreq->req_recv.req_base.req_free_called = true;

PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_NOTIFY,
&(recvreq->req_recv.req_base), PERUSE_RECV );

if( true == recvreq->req_recv.req_base.req_pml_complete ) {
/* make buffer defined when the request is compeleted,
and before releasing the objects. */
MEMCHECKER(
memchecker_call(&opal_memchecker_base_mem_defined,
recvreq->req_recv.req_base.req_addr,
recvreq->req_recv.req_base.req_count,
recvreq->req_recv.req_base.req_datatype);
);
if(false == recvreq->req_recv.req_base.req_free_called){

recvreq->req_recv.req_base.req_free_called = true;
PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_NOTIFY,
&(recvreq->req_recv.req_base), PERUSE_RECV );

if( true == recvreq->req_recv.req_base.req_pml_complete ) {
/* make buffer defined when the request is compeleted,
and before releasing the objects. */
MEMCHECKER(
memchecker_call(&opal_memchecker_base_mem_defined,
recvreq->req_recv.req_base.req_addr,
recvreq->req_recv.req_base.req_count,
recvreq->req_recv.req_base.req_datatype);
);

MCA_PML_OB1_RECV_REQUEST_RETURN( recvreq );
}

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why are the calls to OPAL_THREAD_LOCK removed?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because there is no locking anymore. Everything related to the synchronization is now handled down in the sync_wait, via atomic operations.

MCA_PML_OB1_RECV_REQUEST_RETURN( recvreq );
}

OPAL_THREAD_UNLOCK(&ompi_request_lock);
*request = MPI_REQUEST_NULL;
return OMPI_SUCCESS;
}
Expand Down Expand Up @@ -126,14 +124,12 @@ static int mca_pml_ob1_recv_request_cancel(struct ompi_request_t* ompi_request,
request->req_recv.req_base.req_pml_complete = true;
OB1_MATCHING_UNLOCK(&ob1_comm->matching_lock);

OPAL_THREAD_LOCK(&ompi_request_lock);
ompi_request->req_status._cancelled = true;
/* This macro will set the req_complete to true so the MPI Test/Wait* functions
* on this request will be able to complete. As the status is marked as
* cancelled the cancel state will be detected.
*/
MCA_PML_OB1_RECV_REQUEST_MPI_COMPLETE(request);
OPAL_THREAD_UNLOCK(&ompi_request_lock);
/*
* Receive request cancelled, make user buffer accessible.
*/
Expand Down
Loading