5 files changed, 141 insertions, 77 deletions

diff --git a/sys/dev/nvme/nvme.h b/sys/dev/nvme/nvme.h
index ed1098a3cab8..bb9e945a332f 100644
--- a/sys/dev/nvme/nvme.h
+++ b/sys/dev/nvme/nvme.h
@@ -360,7 +360,7 @@ enum nvme_feature {
 	NVME_FEAT_INTERRUPT_COALESCING		= 0x08,
 	NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION = 0x09,
 	NVME_FEAT_WRITE_ATOMICITY		= 0x0A,
-	NVME_FEAT_ASYNCHRONOUS_EVENT_CONFIGURATION = 0x0B,
+	NVME_FEAT_ASYNC_EVENT_CONFIGURATION	= 0x0B,
 	/* 0x0C-0x7F - reserved */
 	NVME_FEAT_SOFTWARE_PROGRESS_MARKER	= 0x80,
 	/* 0x81-0xBF - command set specific (reserved) */
diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c
index 7eafd1f9d6bb..74cacd9a6f5d 100644
--- a/sys/dev/nvme/nvme_ctrlr.c
+++ b/sys/dev/nvme/nvme_ctrlr.c
@@ -38,6 +38,9 @@ __FBSDID("$FreeBSD$");
 
 #include "nvme_private.h"
 
+static void nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
+						struct nvme_async_event_request *aer);
+
 static void
 nvme_ctrlr_cb(void *arg, const struct nvme_completion *status)
 {
@@ -409,30 +412,6 @@ nvme_ctrlr_reset(struct nvme_controller *ctrlr)
 	return (nvme_ctrlr_enable(ctrlr));
 }
 
-/*
- * Disable this code for now, since Chatham doesn't support
- *  AERs so I have no good way to test them.
- */
-#if 0
-static void
-nvme_async_event_cb(void *arg, const struct nvme_completion *status)
-{
-	struct nvme_controller *ctrlr = arg;
-
-	printf("Asynchronous event occurred.\n");
-
-	/* TODO: decode async event type based on status */
-	/* TODO: check status for any error bits */
-
-	/*
-	 * Repost an asynchronous event request so that it can be
-	 *  used again by the controller.
-	 */
-	nvme_ctrlr_cmd_asynchronous_event_request(ctrlr, nvme_async_event_cb,
-	    ctrlr);
-}
-#endif
-
 static int
 nvme_ctrlr_identify(struct nvme_controller *ctrlr)
 {
@@ -558,27 +537,71 @@ nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr)
 }
 
 static void
+nvme_ctrlr_async_event_cb(void *arg, const struct nvme_completion *cpl)
+{
+	struct nvme_async_event_request *aer = arg;
+
+	if (cpl->sf_sc == NVME_SC_ABORTED_SQ_DELETION) {
+		/*
+		 *  This is simulated when controller is being shut down, to
+		 *  effectively abort outstanding asynchronous event requests
+		 *  and make sure all memory is freed.  Do not repost the
+		 *  request in this case.
+		 */
+		return;
+	}
+
+	/* TODO: decode async event type based on status */
+
+	/*
+	 * Repost another asynchronous event request to replace the one that
+	 *  just completed.
+	 */
+	printf("Asynchronous event occurred.\n");
+	nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);
+}
+
+static void
+nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
+    struct nvme_async_event_request *aer)
+{
+	struct nvme_request *req;
+
+	aer->ctrlr = ctrlr;
+	req = nvme_allocate_request(NULL, 0, nvme_ctrlr_async_event_cb, aer);
+	aer->req = req;
+
+	/*
+	 * Override default timeout value here, since asynchronous event
+	 *  requests should by nature never be timed out.
+	 */
+	req->timeout = 0;
+	req->cmd.opc = NVME_OPC_ASYNC_EVENT_REQUEST;
+	nvme_ctrlr_submit_admin_request(ctrlr, req);
+}
+
+static void
 nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr)
 {
 	union nvme_critical_warning_state	state;
-	uint8_t					num_async_events;
+	struct nvme_async_event_request		*aer;
+	uint32_t				i;
 
 	state.raw = 0xFF;
 	state.bits.reserved = 0;
-	nvme_ctrlr_cmd_set_asynchronous_event_config(ctrlr, state, NULL, NULL);
+	nvme_ctrlr_cmd_set_async_event_config(ctrlr, state, NULL, NULL);
 
 	/* aerl is a zero-based value, so we need to add 1 here. */
-	num_async_events = min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl+1));
+	ctrlr->num_aers = min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl+1));
 
-	/*
-	 * Disable this code for now, since Chatham doesn't support
-	 *  AERs so I have no good way to test them.
-	 */
-#if 0
-	for (int i = 0; i < num_async_events; i++)
-		nvme_ctrlr_cmd_asynchronous_event_request(ctrlr,
-		    nvme_async_event_cb, ctrlr);
-#endif
+	/* Chatham doesn't support AERs. */
+	if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID)
+		ctrlr->num_aers = 0;
+
+	for (i = 0; i < ctrlr->num_aers; i++) {
+		aer = &ctrlr->aer[i];
+		nvme_ctrlr_construct_and_submit_aer(ctrlr, aer);
+	}
 }
 
 static void
@@ -810,8 +833,8 @@ intx:
 void
 nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev)
 {
-	struct nvme_namespace	*ns;
-	int			i;
+	struct nvme_namespace		*ns;
+	int				i;
 
 	for (i = 0; i < NVME_MAX_NAMESPACES; i++) {
 		ns = &ctrlr->ns[i];
@@ -828,6 +851,13 @@ nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev)
 
 	free(ctrlr->ioq, M_NVME);
 
+	/* Manually abort outstanding async event requests. */
+	for (i = 0; i < ctrlr->num_aers; i++) {
+		nvme_qpair_manual_abort_request(&ctrlr->adminq,
+		    ctrlr->aer[i].req, NVME_SCT_GENERIC,
+		    NVME_SC_ABORTED_SQ_DELETION, FALSE);
+	}
+
 	nvme_admin_qpair_destroy(&ctrlr->adminq);
 
 	if (ctrlr->resource != NULL) {
diff --git a/sys/dev/nvme/nvme_ctrlr_cmd.c b/sys/dev/nvme/nvme_ctrlr_cmd.c
index 203c4bd4e077..03124b6c50a4 100644
--- a/sys/dev/nvme/nvme_ctrlr_cmd.c
+++ b/sys/dev/nvme/nvme_ctrlr_cmd.c
@@ -211,7 +211,7 @@ nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr,
 }
 
 void
-nvme_ctrlr_cmd_set_asynchronous_event_config(struct nvme_controller *ctrlr,
+nvme_ctrlr_cmd_set_async_event_config(struct nvme_controller *ctrlr,
     union nvme_critical_warning_state state, nvme_cb_fn_t cb_fn,
     void *cb_arg)
 {
@@ -219,7 +219,7 @@ nvme_ctrlr_cmd_set_asynchronous_event_config(struct nvme_controller *ctrlr,
 
 	cdw11 = state.raw;
 	nvme_ctrlr_cmd_set_feature(ctrlr,
-	    NVME_FEAT_ASYNCHRONOUS_EVENT_CONFIGURATION, cdw11, NULL, 0, cb_fn,
+	    NVME_FEAT_ASYNC_EVENT_CONFIGURATION, cdw11, NULL, 0, cb_fn,
 	    cb_arg);
 }
 
@@ -249,27 +249,6 @@ nvme_ctrlr_cmd_set_interrupt_coalescing(struct nvme_controller *ctrlr,
 }
 
 void
-nvme_ctrlr_cmd_asynchronous_event_request(struct nvme_controller *ctrlr,
-    nvme_cb_fn_t cb_fn, void *cb_arg)
-{
-	struct nvme_request *req;
-	struct nvme_command *cmd;
-
-	req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg);
-
-	/*
-	 * Override default timeout value here, since asynchronous event
-	 *  requests should by nature never be timed out.
-	 */
-	req->timeout = 0;
-
-	cmd = &req->cmd;
-	cmd->opc = NVME_OPC_ASYNC_EVENT_REQUEST;
-
-	nvme_ctrlr_submit_admin_request(ctrlr, req);
-}
-
-void
 nvme_ctrlr_cmd_get_health_information_page(struct nvme_controller *ctrlr,
     uint32_t nsid, struct nvme_health_information_page *payload,
     nvme_cb_fn_t cb_fn, void *cb_arg)
diff --git a/sys/dev/nvme/nvme_private.h b/sys/dev/nvme/nvme_private.h
index 4ea3a067c41e..6874449329b3 100644
--- a/sys/dev/nvme/nvme_private.h
+++ b/sys/dev/nvme/nvme_private.h
@@ -97,7 +97,7 @@ MALLOC_DECLARE(M_NVME);
 
 #define NVME_MAX_NAMESPACES	(16)
 #define NVME_MAX_CONSUMERS	(2)
-#define NVME_MAX_ASYNC_EVENTS	(4)
+#define NVME_MAX_ASYNC_EVENTS	(8)
 
 #define NVME_TIMEOUT_IN_SEC	(30)
 
@@ -119,6 +119,12 @@ struct nvme_request {
 	STAILQ_ENTRY(nvme_request)	stailq;
 };
 
+struct nvme_async_event_request {
+
+	struct nvme_controller		*ctrlr;
+	struct nvme_request		*req;
+};
+
 struct nvme_tracker {
 
 	SLIST_ENTRY(nvme_tracker)	slist;
@@ -255,6 +261,9 @@ struct nvme_controller {
 
 	boolean_t			is_started;
 
+	uint32_t			num_aers;
+	struct nvme_async_event_request	aer[NVME_MAX_ASYNC_EVENTS];
+
 #ifdef CHATHAM2
 	uint64_t		chatham_size;
 	uint64_t		chatham_lbas;
@@ -343,12 +352,9 @@ void	nvme_ctrlr_cmd_delete_io_sq(struct nvme_controller *ctrlr,
 void	nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr,
 				      uint32_t num_queues, nvme_cb_fn_t cb_fn,
 				      void *cb_arg);
-void	nvme_ctrlr_cmd_set_asynchronous_event_config(struct nvme_controller *ctrlr,
-					   union nvme_critical_warning_state state,
-					   nvme_cb_fn_t cb_fn, void *cb_arg);
-void	nvme_ctrlr_cmd_asynchronous_event_request(struct nvme_controller *ctrlr,
-						  nvme_cb_fn_t cb_fn,
-						  void *cb_arg);
+void	nvme_ctrlr_cmd_set_async_event_config(struct nvme_controller *ctrlr,
+					      union nvme_critical_warning_state state,
+					      nvme_cb_fn_t cb_fn, void *cb_arg);
 void	nvme_ctrlr_cmd_abort(struct nvme_controller *ctrlr, uint16_t cid,
 			     uint16_t sqid, nvme_cb_fn_t cb_fn, void *cb_arg);
 
@@ -376,6 +382,9 @@ void	nvme_qpair_submit_cmd(struct nvme_qpair *qpair,
 void	nvme_qpair_process_completions(struct nvme_qpair *qpair);
 void	nvme_qpair_submit_request(struct nvme_qpair *qpair,
 				  struct nvme_request *req);
+void	nvme_qpair_manual_abort_request(struct nvme_qpair *qpair,
+					struct nvme_request *req, uint32_t sct,
+					uint32_t sc, boolean_t print_on_error);
 
 void	nvme_admin_qpair_destroy(struct nvme_qpair *qpair);
 
diff --git a/sys/dev/nvme/nvme_qpair.c b/sys/dev/nvme/nvme_qpair.c
index b7eb0323619d..365ae680ae4f 100644
--- a/sys/dev/nvme/nvme_qpair.c
+++ b/sys/dev/nvme/nvme_qpair.c
@@ -87,6 +87,23 @@ nvme_completion_is_retry(const struct nvme_completion *cpl)
 	}
 }
 
+static struct nvme_tracker *
+nvme_qpair_find_tracker(struct nvme_qpair *qpair, struct nvme_request *req)
+{
+	struct nvme_tracker	*tr;
+	uint32_t		i;
+
+	KASSERT(req != NULL, ("%s: called with NULL req\n", __func__));
+
+	for (i = 0; i < qpair->num_entries; ++i) {
+		tr = qpair->act_tr[i];
+		if (tr != NULL && tr->req == req)
+			return (tr);
+	}
+
+	return (NULL);
+}
+
 static void
 nvme_qpair_construct_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr,
     uint16_t cid)
@@ -137,6 +154,7 @@ nvme_qpair_complete_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr,
 			    tr->payload_dma_map);
 
 		nvme_free_request(req);
+		tr->req = NULL;
 
 		SLIST_INSERT_HEAD(&qpair->free_tr, tr, slist);
 
@@ -297,7 +315,7 @@ nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id,
 static void
 nvme_qpair_destroy(struct nvme_qpair *qpair)
 {
-	struct nvme_tracker *tr;
+	struct nvme_tracker	*tr;
 
 	if (qpair->tag)
 		bus_teardown_intr(qpair->ctrlr->dev, qpair->res, qpair->tag);
@@ -393,9 +411,41 @@ nvme_io_qpair_destroy(struct nvme_qpair *qpair)
 }
 
 static void
-nvme_abort_complete(void *arg, const struct nvme_completion *status)
+nvme_qpair_manual_abort_tracker(struct nvme_qpair *qpair,
+    struct nvme_tracker *tr, uint32_t sct, uint32_t sc,
+    boolean_t print_on_error)
 {
 	struct nvme_completion	cpl;
+
+	memset(&cpl, 0, sizeof(cpl));
+	cpl.sqid = qpair->id;
+	cpl.cid = tr->cid;
+	cpl.sf_sct = sct;
+	cpl.sf_sc = sc;
+	nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error);
+}
+
+void
+nvme_qpair_manual_abort_request(struct nvme_qpair *qpair,
+    struct nvme_request *req, uint32_t sct, uint32_t sc,
+    boolean_t print_on_error)
+{
+	struct nvme_tracker	*tr;
+
+	tr = nvme_qpair_find_tracker(qpair, req);
+
+	if (tr == NULL) {
+		printf("%s: request not found\n", __func__);
+		nvme_dump_command(&req->cmd);
+		return;
+	}
+
+	nvme_qpair_manual_abort_tracker(qpair, tr, sct, sc, print_on_error);
+}
+
+static void
+nvme_abort_complete(void *arg, const struct nvme_completion *status)
+{
 	struct nvme_tracker	*tr = arg;
 
 	/*
@@ -411,12 +461,8 @@ nvme_abort_complete(void *arg, const struct nvme_completion *status)
 		 *  status, and then complete the I/O's tracker manually.
 		 */
 		printf("abort command failed, aborting command manually\n");
-		memset(&cpl, 0, sizeof(cpl));
-		cpl.sqid = tr->qpair->id;
-		cpl.cid = tr->cid;
-		cpl.sf_sct = NVME_SCT_GENERIC;
-		cpl.sf_sc = NVME_SC_ABORTED_BY_REQUEST;
-		nvme_qpair_complete_tracker(tr->qpair, tr, &cpl, TRUE);
+		nvme_qpair_manual_abort_tracker(tr->qpair, tr,
+		    NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST, TRUE);
 	}
 }