aboutsummaryrefslogtreecommitdiff
path: root/sys/dev/ntb/ntb_transport.c
diff options
context:
space:
mode:
authorAlexander Motin <mav@FreeBSD.org>2019-11-10 03:37:45 +0000
committerAlexander Motin <mav@FreeBSD.org>2019-11-10 03:37:45 +0000
commit028d96899b0d23573a882da496970366e6ef50b1 (patch)
tree218807df3f9c27bcb5b437081592d47f325ec0b4 /sys/dev/ntb/ntb_transport.c
parent7aafa7c368e7d545a9f0a74408fa244cd04d0f59 (diff)
downloadsrc-028d96899b0d23573a882da496970366e6ef50b1.tar.gz
src-028d96899b0d23573a882da496970366e6ef50b1.zip
Add compact scraptchpad protocol for ntb_transport(4).
Previously ntb_transport(4) required at least 6 scratchpad registers, plus 2 more for each additional memory window. That is too much for some configurations, where several drivers have to share resources of the same NTB hardware. This patch introduces new compact version of the protocol, requiring only 3 scratchpad registers, plus one more for each additional memory window. The optimization is based on fact that neither of version, number of windows or number of queue pairs really need more then one byte each, and window sizes of 4GB are not very useful now. The new protocol is activated automatically when the configuration is low on scratchpad registers, or it can be activated explicitly with loader tunable. MFC after: 2 weeks Sponsored by: iXsystems, Inc.
Notes
Notes: svn path=/head/; revision=354581
Diffstat (limited to 'sys/dev/ntb/ntb_transport.c')
-rw-r--r--sys/dev/ntb/ntb_transport.c120
1 files changed, 88 insertions, 32 deletions
diff --git a/sys/dev/ntb/ntb_transport.c b/sys/dev/ntb/ntb_transport.c
index bc1091b940e8..a9cd4f202ed1 100644
--- a/sys/dev/ntb/ntb_transport.c
+++ b/sys/dev/ntb/ntb_transport.c
@@ -203,6 +203,7 @@ struct ntb_transport_ctx {
struct ntb_transport_child *child;
struct ntb_transport_mw *mw_vec;
struct ntb_transport_qp *qp_vec;
+ int compact;
unsigned mw_count;
unsigned qp_count;
uint64_t qp_bitmap;
@@ -252,6 +253,15 @@ enum {
NTBT_WATCHDOG_SPAD = 15
};
+/*
+ * Compart version of sratchpad protocol, using twice less registers.
+ */
+enum {
+ NTBTC_PARAMS = 0, /* NUM_QPS << 24 + NUM_MWS << 16 + VERSION */
+ NTBTC_QP_LINKS, /* QP links status */
+ NTBTC_MW0_SZ, /* MW size limited to 32 bits. */
+};
+
#define QP_TO_MW(nt, qp) ((qp) % nt->mw_count)
#define NTB_QP_DEF_NUM_ENTRIES 100
#define NTB_LINK_DOWN_TIMEOUT 100
@@ -352,14 +362,30 @@ ntb_transport_attach(device_t dev)
device_printf(dev, "At least 1 memory window required.\n");
return (ENXIO);
}
- if (spad_count < 6) {
- device_printf(dev, "At least 6 scratchpads required.\n");
- return (ENXIO);
- }
- if (spad_count < 4 + 2 * nt->mw_count) {
- nt->mw_count = (spad_count - 4) / 2;
- device_printf(dev, "Scratchpads enough only for %d "
- "memory windows.\n", nt->mw_count);
+ nt->compact = (spad_count < 4 + 2 * nt->mw_count);
+ snprintf(buf, sizeof(buf), "hint.%s.%d.compact", device_get_name(dev),
+ device_get_unit(dev));
+ TUNABLE_INT_FETCH(buf, &nt->compact);
+ if (nt->compact) {
+ if (spad_count < 3) {
+ device_printf(dev, "At least 3 scratchpads required.\n");
+ return (ENXIO);
+ }
+ if (spad_count < 2 + nt->mw_count) {
+ nt->mw_count = spad_count - 2;
+ device_printf(dev, "Scratchpads enough only for %d "
+ "memory windows.\n", nt->mw_count);
+ }
+ } else {
+ if (spad_count < 6) {
+ device_printf(dev, "At least 6 scratchpads required.\n");
+ return (ENXIO);
+ }
+ if (spad_count < 4 + 2 * nt->mw_count) {
+ nt->mw_count = (spad_count - 4) / 2;
+ device_printf(dev, "Scratchpads enough only for %d "
+ "memory windows.\n", nt->mw_count);
+ }
}
if (db_bitmap == 0) {
device_printf(dev, "At least one doorbell required.\n");
@@ -380,10 +406,16 @@ ntb_transport_attach(device_t dev)
mw->tx_size = mw->phys_size;
if (max_mw_size != 0 && mw->tx_size > max_mw_size) {
device_printf(dev, "Memory window %d limited from "
- "%ju to %ju\n", i, (uintmax_t)mw->phys_size,
+ "%ju to %ju\n", i, (uintmax_t)mw->tx_size,
max_mw_size);
mw->tx_size = max_mw_size;
}
+ if (nt->compact && mw->tx_size > UINT32_MAX) {
+ device_printf(dev, "Memory window %d is too big "
+ "(%ju)\n", i, (uintmax_t)mw->tx_size);
+ rc = ENXIO;
+ goto err;
+ }
mw->rx_size = 0;
mw->buff_size = 0;
@@ -1110,37 +1142,61 @@ ntb_transport_link_work(void *arg)
int rc;
/* send the local info, in the opposite order of the way we read it */
- for (i = 0; i < nt->mw_count; i++) {
- size = nt->mw_vec[i].tx_size;
- ntb_peer_spad_write(dev, NTBT_MW0_SZ_HIGH + (i * 2),
- size >> 32);
- ntb_peer_spad_write(dev, NTBT_MW0_SZ_LOW + (i * 2), size);
+ if (nt->compact) {
+ for (i = 0; i < nt->mw_count; i++) {
+ size = nt->mw_vec[i].tx_size;
+ KASSERT(size <= UINT32_MAX, ("size too big (%jx)", size));
+ ntb_peer_spad_write(dev, NTBTC_MW0_SZ + i, size);
+ }
+ ntb_peer_spad_write(dev, NTBTC_QP_LINKS, 0);
+ ntb_peer_spad_write(dev, NTBTC_PARAMS,
+ (nt->qp_count << 24) | (nt->mw_count << 16) |
+ NTB_TRANSPORT_VERSION);
+ } else {
+ for (i = 0; i < nt->mw_count; i++) {
+ size = nt->mw_vec[i].tx_size;
+ ntb_peer_spad_write(dev, NTBT_MW0_SZ_HIGH + (i * 2),
+ size >> 32);
+ ntb_peer_spad_write(dev, NTBT_MW0_SZ_LOW + (i * 2), size);
+ }
+ ntb_peer_spad_write(dev, NTBT_NUM_MWS, nt->mw_count);
+ ntb_peer_spad_write(dev, NTBT_NUM_QPS, nt->qp_count);
+ ntb_peer_spad_write(dev, NTBT_QP_LINKS, 0);
+ ntb_peer_spad_write(dev, NTBT_VERSION, NTB_TRANSPORT_VERSION);
}
- ntb_peer_spad_write(dev, NTBT_NUM_MWS, nt->mw_count);
- ntb_peer_spad_write(dev, NTBT_NUM_QPS, nt->qp_count);
- ntb_peer_spad_write(dev, NTBT_QP_LINKS, 0);
- ntb_peer_spad_write(dev, NTBT_VERSION, NTB_TRANSPORT_VERSION);
/* Query the remote side for its info */
val = 0;
- ntb_spad_read(dev, NTBT_VERSION, &val);
- if (val != NTB_TRANSPORT_VERSION)
- goto out;
+ if (nt->compact) {
+ ntb_spad_read(dev, NTBTC_PARAMS, &val);
+ if (val != ((nt->qp_count << 24) | (nt->mw_count << 16) |
+ NTB_TRANSPORT_VERSION))
+ goto out;
+ } else {
+ ntb_spad_read(dev, NTBT_VERSION, &val);
+ if (val != NTB_TRANSPORT_VERSION)
+ goto out;
- ntb_spad_read(dev, NTBT_NUM_QPS, &val);
- if (val != nt->qp_count)
- goto out;
+ ntb_spad_read(dev, NTBT_NUM_QPS, &val);
+ if (val != nt->qp_count)
+ goto out;
- ntb_spad_read(dev, NTBT_NUM_MWS, &val);
- if (val != nt->mw_count)
- goto out;
+ ntb_spad_read(dev, NTBT_NUM_MWS, &val);
+ if (val != nt->mw_count)
+ goto out;
+ }
for (i = 0; i < nt->mw_count; i++) {
- ntb_spad_read(dev, NTBT_MW0_SZ_HIGH + (i * 2), &val);
- val64 = (uint64_t)val << 32;
-
- ntb_spad_read(dev, NTBT_MW0_SZ_LOW + (i * 2), &val);
- val64 |= val;
+ if (nt->compact) {
+ ntb_spad_read(dev, NTBTC_MW0_SZ + i, &val);
+ val64 = val;
+ } else {
+ ntb_spad_read(dev, NTBT_MW0_SZ_HIGH + (i * 2), &val);
+ val64 = (uint64_t)val << 32;
+
+ ntb_spad_read(dev, NTBT_MW0_SZ_LOW + (i * 2), &val);
+ val64 |= val;
+ }
mw = &nt->mw_vec[i];
mw->rx_size = val64;