aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorWarner Losh <imp@FreeBSD.org>2019-09-04 20:08:36 +0000
committerWarner Losh <imp@FreeBSD.org>2019-09-04 20:08:36 +0000
commitf93b7f954e96d5162007561ea3ddc1443468d912 (patch)
tree8f672da7f1e16731d81d3bd8aa22902e9389b21b
parentc07d4a0a681c7db4fd30dd27a3b949f1820e63c3 (diff)
downloadsrc-f93b7f954e96d5162007561ea3ddc1443468d912.tar.gz
src-f93b7f954e96d5162007561ea3ddc1443468d912.zip
Support doorbell strides != 0.
The NVMe standard (1.4) states >>> 8.6 Doorbell Stride for Software Emulation >>> The doorbell stride,...is useful in software emulation of an NVM >>> Express controller. ... For hardware implementations of the NVM >>> Express interface, the expected doorbell stride value is 0h. However, hardware in the wild exists with a doorbell stride of 1 (meaning 8 byte separation). This change supports that hardware, as well as software emulators as envisioned in Section 8.6. Since this is the fast path, care has been taken to make this computation efficient. The bit of math to compute an offset for each is replaced by a memory load from cache of a pre-computed value. MFC After: 3 days Reviewed by: scottl@ Differential Revision: https://reviews.freebsd.org/D21514
Notes
Notes: svn path=/head/; revision=351828
-rw-r--r--sys/dev/nvme/nvme_ctrlr.c35
-rw-r--r--sys/dev/nvme/nvme_private.h3
-rw-r--r--sys/dev/nvme/nvme_qpair.c20
3 files changed, 33 insertions, 25 deletions
diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c
index f71e57671bef..a206b5efc6cf 100644
--- a/sys/dev/nvme/nvme_ctrlr.c
+++ b/sys/dev/nvme/nvme_ctrlr.c
@@ -90,19 +90,25 @@ nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)
struct nvme_qpair *qpair;
uint32_t cap_lo;
uint16_t mqes;
- int i, error, num_entries, num_trackers;
-
- num_entries = NVME_IO_ENTRIES;
- TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries);
+ int i, error, num_entries, num_trackers, max_entries;
/*
- * NVMe spec sets a hard limit of 64K max entries, but
- * devices may specify a smaller limit, so we need to check
- * the MQES field in the capabilities register.
+ * NVMe spec sets a hard limit of 64K max entries, but devices may
+ * specify a smaller limit, so we need to check the MQES field in the
+ * capabilities register. We have to cap the number of entries to the
+ * current stride allows for in BAR 0/1, otherwise the remainder entries
+ * are inaccessable. MQES should reflect this, and this is just a
+ * fail-safe.
*/
+ max_entries =
+ (rman_get_size(ctrlr->resource) - nvme_mmio_offsetof(doorbell[0])) /
+ (1 << (ctrlr->dstrd + 1));
+ num_entries = NVME_IO_ENTRIES;
+ TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries);
cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);
mqes = NVME_CAP_LO_MQES(cap_lo);
num_entries = min(num_entries, mqes + 1);
+ num_entries = min(num_entries, max_entries);
num_trackers = NVME_IO_TRACKERS;
TUNABLE_INT_FETCH("hw.nvme.io_trackers", &num_trackers);
@@ -110,9 +116,9 @@ nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)
num_trackers = max(num_trackers, NVME_MIN_IO_TRACKERS);
num_trackers = min(num_trackers, NVME_MAX_IO_TRACKERS);
/*
- * No need to have more trackers than entries in the submit queue.
- * Note also that for a queue size of N, we can only have (N-1)
- * commands outstanding, hence the "-1" here.
+ * No need to have more trackers than entries in the submit queue. Note
+ * also that for a queue size of N, we can only have (N-1) commands
+ * outstanding, hence the "-1" here.
*/
num_trackers = min(num_trackers, (num_entries-1));
@@ -1120,7 +1126,6 @@ nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev)
uint32_t cap_lo;
uint32_t cap_hi;
uint32_t to;
- uint8_t dstrd;
uint8_t mpsmin;
int status, timeout_period;
@@ -1128,14 +1133,8 @@ nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev)
mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF);
- /*
- * Software emulators may set the doorbell stride to something
- * other than zero, but this driver is not set up to handle that.
- */
cap_hi = nvme_mmio_read_4(ctrlr, cap_hi);
- dstrd = NVME_CAP_HI_DSTRD(cap_hi);
- if (dstrd != 0)
- return (ENXIO);
+ ctrlr->dstrd = NVME_CAP_HI_DSTRD(cap_hi) + 2;
mpsmin = NVME_CAP_HI_MPSMIN(cap_hi);
ctrlr->min_page_size = 1 << (12 + mpsmin);
diff --git a/sys/dev/nvme/nvme_private.h b/sys/dev/nvme/nvme_private.h
index d26d376ce6cc..e62a78a222f1 100644
--- a/sys/dev/nvme/nvme_private.h
+++ b/sys/dev/nvme/nvme_private.h
@@ -297,6 +297,9 @@ struct nvme_controller {
/** timeout period in seconds */
uint32_t timeout_period;
+ /** doorbell stride */
+ uint32_t dstrd;
+
struct nvme_qpair adminq;
struct nvme_qpair *ioq;
diff --git a/sys/dev/nvme/nvme_qpair.c b/sys/dev/nvme/nvme_qpair.c
index 80c69012bf4f..744ca3492eda 100644
--- a/sys/dev/nvme/nvme_qpair.c
+++ b/sys/dev/nvme/nvme_qpair.c
@@ -622,8 +622,8 @@ nvme_qpair_process_completions(struct nvme_qpair *qpair)
qpair->phase = !qpair->phase; /* 3 */
}
- nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl,
- qpair->cq_head);
+ bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle,
+ qpair->cq_hdbl_off, qpair->cq_head);
}
return (done != 0);
}
@@ -731,8 +731,15 @@ nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id,
qpair->cpl_bus_addr = queuemem_phys + cmdsz;
prpmem_phys = queuemem_phys + cmdsz + cplsz;
- qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl);
- qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl);
+ /*
+ * Calcuate the stride of the doorbell register. Many emulators set this
+ * value to correspond to a cache line. However, some hardware has set
+ * it to various small values.
+ */
+ qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[0]) +
+ (id << (ctrlr->dstrd + 1));
+ qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[0]) +
+ (id << (ctrlr->dstrd + 1)) + (1 << ctrlr->dstrd);
TAILQ_INIT(&qpair->free_tr);
TAILQ_INIT(&qpair->outstanding_tr);
@@ -950,9 +957,8 @@ nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr)
wmb();
#endif
- nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl,
- qpair->sq_tail);
-
+ bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle,
+ qpair->sq_tdbl_off, qpair->sq_tail);
qpair->num_cmds++;
}