[prev in list] [next in list] [prev in thread] [next in thread] 

List:       linux-rdma
Subject:    [PATCH 10/16] librdmacm/rsocket: Add option to specify size of inline data
From:       "Hefty, Sean" <sean.hefty () intel ! com>
Date:       2012-05-30 17:21:44
Message-ID: 1828884A29C6694DAF28B7E6B8A8237346A24C0D () ORSMSX101 ! amr ! corp ! intel ! com
[Download RAW message or body]

Allow the user to override the default inline data size.
We still require a minimum size in order to transfer receive
buffer update message.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
---
We can eliminate the need for inline entirely by reserving some of the
send buffer space for control messages, but that work is deferred for
a later patch.

 include/rdma/rsocket.h |    3 ++-
 src/rsocket.c          |   21 +++++++++++++++++----
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/include/rdma/rsocket.h b/include/rdma/rsocket.h
index 9fd9929..65feda9 100644
--- a/include/rdma/rsocket.h
+++ b/include/rdma/rsocket.h
@@ -75,7 +75,8 @@ int rgetsockname(int socket, struct sockaddr *addr, socklen_t *addrlen);
 #define SOL_RDMA 0x10000
 enum {
 	RDMA_SQSIZE,
-	RDMA_RQSIZE
+	RDMA_RQSIZE,
+	RDMA_INLINE
 };
 
 int rsetsockopt(int socket, int level, int optname,
diff --git a/src/rsocket.c b/src/rsocket.c
index f94e48a..ef070a8 100644
--- a/src/rsocket.c
+++ b/src/rsocket.c
@@ -105,6 +105,7 @@ struct rs_sge {
 	uint32_t length;
 };
 
+#define RS_MIN_INLINE    (sizeof(struct rs_sge))
 #define rs_host_is_net() (1 == htonl(1))
 #define RS_CONN_FLAG_NET 1
 
@@ -163,6 +164,7 @@ struct rsocket {
 	uint16_t	  sseq_no;
 	uint16_t	  sseq_comp;
 	uint16_t	  sq_size;
+	uint16_t	  sq_inline;
 
 	uint16_t	  rq_size;
 	uint16_t	  rseq_no;
@@ -229,11 +231,13 @@ static struct rsocket *rs_alloc(struct rsocket *inherited_rs)
 	if (inherited_rs) {
 		rs->sbuf_size = inherited_rs->sbuf_size;
 		rs->rbuf_size = inherited_rs->rbuf_size;
+		rs->sq_inline = inherited_rs->sq_inline;
 		rs->sq_size = inherited_rs->sq_size;
 		rs->rq_size = inherited_rs->rq_size;
 		rs->ctrl_avail = inherited_rs->ctrl_avail;
 	} else {
 		rs->sbuf_size = rs->rbuf_size = RS_BUF_SIZE;
+		rs->sq_inline = RS_INLINE;
 		rs->sq_size = rs->rq_size = RS_QP_SIZE;
 		rs->ctrl_avail = RS_QP_CTRL_SIZE;
 	}
@@ -367,7 +371,7 @@ static int rs_create_ep(struct rsocket *rs)
 	qp_attr.cap.max_recv_wr = rs->rq_size;
 	qp_attr.cap.max_send_sge = 2;
 	qp_attr.cap.max_recv_sge = 1;
-	qp_attr.cap.max_inline_data = RS_INLINE;
+	qp_attr.cap.max_inline_data = rs->sq_inline;
 
 	ret = rdma_create_qp(rs->cm_id, NULL, &qp_attr);
 	if (ret)
@@ -1156,7 +1160,7 @@ ssize_t rsend(int socket, const void *buf, size_t len, int flags)
 		if (xfer_size > rs->target_sgl[rs->target_sge].length)
 			xfer_size = rs->target_sgl[rs->target_sge].length;
 
-		if (xfer_size <= RS_INLINE) {
+		if (xfer_size <= rs->sq_inline) {
 			sge.addr = (uintptr_t) buf;
 			sge.length = xfer_size;
 			sge.lkey = 0;
@@ -1277,7 +1281,7 @@ static ssize_t rsendv(int socket, const struct iovec *iov, int iovcnt, int flags
 			ret = rs_write_data(rs, rs_wrid(1, xfer_size),
 					    rs->ssgl, 1,
 					    rs_msg_set(RS_OP_DATA, xfer_size),
-					    xfer_size <= RS_INLINE ? IBV_SEND_INLINE : 0);
+					    xfer_size <= rs->sq_inline ? IBV_SEND_INLINE : 0);
 			if (xfer_size < rs_sbuf_left(rs))
 				rs->ssgl[0].addr += xfer_size;
 			else
@@ -1291,7 +1295,7 @@ static ssize_t rsendv(int socket, const struct iovec *iov, int iovcnt, int flags
 			ret = rs_write_data(rs, rs_wrid(1, xfer_size),
 					    rs->ssgl, 2,
 					    rs_msg_set(RS_OP_DATA, xfer_size),
-					    xfer_size <= RS_INLINE ? IBV_SEND_INLINE : 0);
+					    xfer_size <= rs->sq_inline ? IBV_SEND_INLINE : 0);
 			rs->ssgl[0].addr = (uintptr_t) rs->sbuf + rs->ssgl[1].length;
 		}
 		if (ret)
@@ -1730,6 +1734,11 @@ int rsetsockopt(int socket, int level, int optname,
 		case RDMA_RQSIZE:
 			rs->rq_size = min((*(uint32_t *) optval), RS_QP_MAX_SIZE);
 			break;
+		case RDMA_INLINE:
+			rs->sq_inline = min(*(uint32_t *) optval, RS_QP_MAX_SIZE);
+			if (rs->sq_inline < RS_MIN_INLINE)
+				rs->sq_inline = RS_MIN_INLINE;
+			break;
 		default:
 			break;
 		}
@@ -1809,6 +1818,10 @@ int rgetsockopt(int socket, int level, int optname,
 			*((int *) optval) = rs->rq_size;
 			*optlen = sizeof(int);
 			break;
+		case RDMA_INLINE:
+			*((int *) optval) = rs->sq_inline;
+			*optlen = sizeof(int);
+			break;
 		default:
 			ret = ENOTSUP;
 			break;


--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
[prev in list] [next in list] [prev in thread] [next in thread]