diff options
Diffstat (limited to 'lib/libnvmf/nvmf_tcp.c')
| -rw-r--r-- | lib/libnvmf/nvmf_tcp.c | 1501 |
1 files changed, 1501 insertions, 0 deletions
diff --git a/lib/libnvmf/nvmf_tcp.c b/lib/libnvmf/nvmf_tcp.c new file mode 100644 index 000000000000..6f41ca7ff502 --- /dev/null +++ b/lib/libnvmf/nvmf_tcp.c @@ -0,0 +1,1501 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2022-2024 Chelsio Communications, Inc. + * Written by: John Baldwin <jhb@FreeBSD.org> + */ + +#include <sys/endian.h> +#include <sys/gsb_crc32.h> +#include <sys/queue.h> +#include <sys/socket.h> +#include <sys/uio.h> +#include <assert.h> +#include <errno.h> +#include <netdb.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include "libnvmf.h" +#include "internal.h" +#include "nvmf_tcp.h" + +struct nvmf_tcp_qpair; + +struct nvmf_tcp_command_buffer { + struct nvmf_tcp_qpair *qp; + + void *data; + size_t data_len; + size_t data_xfered; + uint32_t data_offset; + + uint16_t cid; + uint16_t ttag; + + LIST_ENTRY(nvmf_tcp_command_buffer) link; +}; + +LIST_HEAD(nvmf_tcp_command_buffer_list, nvmf_tcp_command_buffer); + +struct nvmf_tcp_association { + struct nvmf_association na; + + uint32_t ioccsz; +}; + +struct nvmf_tcp_rxpdu { + struct nvme_tcp_common_pdu_hdr *hdr; + uint32_t data_len; +}; + +struct nvmf_tcp_capsule { + struct nvmf_capsule nc; + + struct nvmf_tcp_rxpdu rx_pdu; + struct nvmf_tcp_command_buffer *cb; + + TAILQ_ENTRY(nvmf_tcp_capsule) link; +}; + +struct nvmf_tcp_qpair { + struct nvmf_qpair qp; + int s; + + uint8_t txpda; + uint8_t rxpda; + bool header_digests; + bool data_digests; + uint32_t maxr2t; + uint32_t maxh2cdata; + uint32_t max_icd; /* Host only */ + uint16_t next_ttag; /* Controller only */ + + struct nvmf_tcp_command_buffer_list tx_buffers; + struct nvmf_tcp_command_buffer_list rx_buffers; + TAILQ_HEAD(, nvmf_tcp_capsule) rx_capsules; +}; + +#define TASSOC(nc) ((struct nvmf_tcp_association *)(na)) +#define TCAP(nc) ((struct nvmf_tcp_capsule *)(nc)) +#define CTCAP(nc) ((const struct nvmf_tcp_capsule *)(nc)) +#define TQP(qp) ((struct nvmf_tcp_qpair *)(qp)) + +static const char zero_padding[NVME_TCP_PDU_PDO_MAX_OFFSET]; + +static uint32_t +compute_digest(const void *buf, size_t len) +{ + return (calculate_crc32c(0xffffffff, buf, len) ^ 0xffffffff); +} + +static struct nvmf_tcp_command_buffer * +tcp_alloc_command_buffer(struct nvmf_tcp_qpair *qp, void *data, + uint32_t data_offset, size_t data_len, uint16_t cid, uint16_t ttag, + bool receive) +{ + struct nvmf_tcp_command_buffer *cb; + + cb = malloc(sizeof(*cb)); + cb->qp = qp; + cb->data = data; + cb->data_offset = data_offset; + cb->data_len = data_len; + cb->data_xfered = 0; + cb->cid = cid; + cb->ttag = ttag; + + if (receive) + LIST_INSERT_HEAD(&qp->rx_buffers, cb, link); + else + LIST_INSERT_HEAD(&qp->tx_buffers, cb, link); + return (cb); +} + +static struct nvmf_tcp_command_buffer * +tcp_find_command_buffer(struct nvmf_tcp_qpair *qp, uint16_t cid, uint16_t ttag, + bool receive) +{ + struct nvmf_tcp_command_buffer_list *list; + struct nvmf_tcp_command_buffer *cb; + + list = receive ? &qp->rx_buffers : &qp->tx_buffers; + LIST_FOREACH(cb, list, link) { + if (cb->cid == cid && cb->ttag == ttag) + return (cb); + } + return (NULL); +} + +static void +tcp_purge_command_buffer(struct nvmf_tcp_qpair *qp, uint16_t cid, uint16_t ttag, + bool receive) +{ + struct nvmf_tcp_command_buffer *cb; + + cb = tcp_find_command_buffer(qp, cid, ttag, receive); + if (cb != NULL) + LIST_REMOVE(cb, link); +} + +static void +tcp_free_command_buffer(struct nvmf_tcp_command_buffer *cb) +{ + LIST_REMOVE(cb, link); + free(cb); +} + +static int +nvmf_tcp_write_pdu(struct nvmf_tcp_qpair *qp, const void *pdu, size_t len) +{ + ssize_t nwritten; + const char *cp; + + cp = pdu; + while (len != 0) { + nwritten = write(qp->s, cp, len); + if (nwritten < 0) + return (errno); + len -= nwritten; + cp += nwritten; + } + return (0); +} + +static int +nvmf_tcp_write_pdu_iov(struct nvmf_tcp_qpair *qp, struct iovec *iov, + u_int iovcnt, size_t len) +{ + ssize_t nwritten; + + for (;;) { + nwritten = writev(qp->s, iov, iovcnt); + if (nwritten < 0) + return (errno); + + len -= nwritten; + if (len == 0) + return (0); + + while (iov->iov_len <= (size_t)nwritten) { + nwritten -= iov->iov_len; + iovcnt--; + iov++; + } + + iov->iov_base = (char *)iov->iov_base + nwritten; + iov->iov_len -= nwritten; + } +} + +static void +nvmf_tcp_report_error(struct nvmf_association *na, struct nvmf_tcp_qpair *qp, + uint16_t fes, uint32_t fei, const void *rx_pdu, size_t pdu_len, u_int hlen) +{ + struct nvme_tcp_term_req_hdr hdr; + struct iovec iov[2]; + + if (hlen != 0) { + if (hlen > NVME_TCP_TERM_REQ_ERROR_DATA_MAX_SIZE) + hlen = NVME_TCP_TERM_REQ_ERROR_DATA_MAX_SIZE; + if (hlen > pdu_len) + hlen = pdu_len; + } + + memset(&hdr, 0, sizeof(hdr)); + hdr.common.pdu_type = na->na_controller ? + NVME_TCP_PDU_TYPE_C2H_TERM_REQ : NVME_TCP_PDU_TYPE_H2C_TERM_REQ; + hdr.common.hlen = sizeof(hdr); + hdr.common.plen = sizeof(hdr) + hlen; + hdr.fes = htole16(fes); + le32enc(hdr.fei, fei); + iov[0].iov_base = &hdr; + iov[0].iov_len = sizeof(hdr); + iov[1].iov_base = __DECONST(void *, rx_pdu); + iov[1].iov_len = hlen; + + (void)nvmf_tcp_write_pdu_iov(qp, iov, nitems(iov), sizeof(hdr) + hlen); + close(qp->s); + qp->s = -1; +} + +static int +nvmf_tcp_validate_pdu(struct nvmf_tcp_qpair *qp, struct nvmf_tcp_rxpdu *pdu, + size_t pdu_len) +{ + const struct nvme_tcp_common_pdu_hdr *ch; + uint32_t data_len, fei, plen; + uint32_t digest, rx_digest; + u_int hlen; + int error; + uint16_t fes; + + /* Determine how large of a PDU header to return for errors. */ + ch = pdu->hdr; + hlen = ch->hlen; + plen = le32toh(ch->plen); + if (hlen < sizeof(*ch) || hlen > plen) + hlen = sizeof(*ch); + + error = nvmf_tcp_validate_pdu_header(ch, + qp->qp.nq_association->na_controller, qp->header_digests, + qp->data_digests, qp->rxpda, &data_len, &fes, &fei); + if (error != 0) { + if (error == ECONNRESET) { + close(qp->s); + qp->s = -1; + } else { + nvmf_tcp_report_error(qp->qp.nq_association, qp, + fes, fei, ch, pdu_len, hlen); + } + return (error); + } + + /* Check header digest if present. */ + if ((ch->flags & NVME_TCP_CH_FLAGS_HDGSTF) != 0) { + digest = compute_digest(ch, ch->hlen); + memcpy(&rx_digest, (const char *)ch + ch->hlen, + sizeof(rx_digest)); + if (digest != rx_digest) { + printf("NVMe/TCP: Header digest mismatch\n"); + nvmf_tcp_report_error(qp->qp.nq_association, qp, + NVME_TCP_TERM_REQ_FES_HDGST_ERROR, rx_digest, ch, + pdu_len, hlen); + return (EBADMSG); + } + } + + /* Check data digest if present. */ + if ((ch->flags & NVME_TCP_CH_FLAGS_DDGSTF) != 0) { + digest = compute_digest((const char *)ch + ch->pdo, data_len); + memcpy(&rx_digest, (const char *)ch + plen - sizeof(rx_digest), + sizeof(rx_digest)); + if (digest != rx_digest) { + printf("NVMe/TCP: Data digest mismatch\n"); + return (EBADMSG); + } + } + + pdu->data_len = data_len; + return (0); +} + +/* + * Read data from a socket, retrying until the data has been fully + * read or an error occurs. + */ +static int +nvmf_tcp_read_buffer(int s, void *buf, size_t len) +{ + ssize_t nread; + char *cp; + + cp = buf; + while (len != 0) { + nread = read(s, cp, len); + if (nread < 0) + return (errno); + if (nread == 0) + return (ECONNRESET); + len -= nread; + cp += nread; + } + return (0); +} + +static int +nvmf_tcp_read_pdu(struct nvmf_tcp_qpair *qp, struct nvmf_tcp_rxpdu *pdu) +{ + struct nvme_tcp_common_pdu_hdr ch; + uint32_t plen; + int error; + + memset(pdu, 0, sizeof(*pdu)); + error = nvmf_tcp_read_buffer(qp->s, &ch, sizeof(ch)); + if (error != 0) + return (error); + + plen = le32toh(ch.plen); + + /* + * Validate a header with garbage lengths to trigger + * an error message without reading more. + */ + if (plen < sizeof(ch) || ch.hlen > plen) { + pdu->hdr = &ch; + error = nvmf_tcp_validate_pdu(qp, pdu, sizeof(ch)); + pdu->hdr = NULL; + assert(error != 0); + return (error); + } + + /* Read the rest of the PDU. */ + pdu->hdr = malloc(plen); + memcpy(pdu->hdr, &ch, sizeof(ch)); + error = nvmf_tcp_read_buffer(qp->s, pdu->hdr + 1, plen - sizeof(ch)); + if (error != 0) + return (error); + error = nvmf_tcp_validate_pdu(qp, pdu, plen); + if (error != 0) { + free(pdu->hdr); + pdu->hdr = NULL; + } + return (error); +} + +static void +nvmf_tcp_free_pdu(struct nvmf_tcp_rxpdu *pdu) +{ + free(pdu->hdr); + pdu->hdr = NULL; +} + +static int +nvmf_tcp_handle_term_req(struct nvmf_tcp_rxpdu *pdu) +{ + struct nvme_tcp_term_req_hdr *hdr; + + hdr = (void *)pdu->hdr; + + printf("NVMe/TCP: Received termination request: fes %#x fei %#x\n", + le16toh(hdr->fes), le32dec(hdr->fei)); + nvmf_tcp_free_pdu(pdu); + return (ECONNRESET); +} + +static int +nvmf_tcp_save_command_capsule(struct nvmf_tcp_qpair *qp, + struct nvmf_tcp_rxpdu *pdu) +{ + struct nvme_tcp_cmd *cmd; + struct nvmf_capsule *nc; + struct nvmf_tcp_capsule *tc; + + cmd = (void *)pdu->hdr; + + nc = nvmf_allocate_command(&qp->qp, &cmd->ccsqe); + if (nc == NULL) + return (ENOMEM); + + tc = TCAP(nc); + tc->rx_pdu = *pdu; + + TAILQ_INSERT_TAIL(&qp->rx_capsules, tc, link); + return (0); +} + +static int +nvmf_tcp_save_response_capsule(struct nvmf_tcp_qpair *qp, + struct nvmf_tcp_rxpdu *pdu) +{ + struct nvme_tcp_rsp *rsp; + struct nvmf_capsule *nc; + struct nvmf_tcp_capsule *tc; + + rsp = (void *)pdu->hdr; + + nc = nvmf_allocate_response(&qp->qp, &rsp->rccqe); + if (nc == NULL) + return (ENOMEM); + + nc->nc_sqhd_valid = true; + tc = TCAP(nc); + tc->rx_pdu = *pdu; + + TAILQ_INSERT_TAIL(&qp->rx_capsules, tc, link); + + /* + * Once the CQE has been received, no further transfers to the + * command buffer for the associated CID can occur. + */ + tcp_purge_command_buffer(qp, rsp->rccqe.cid, 0, true); + tcp_purge_command_buffer(qp, rsp->rccqe.cid, 0, false); + + return (0); +} + +/* + * Construct and send a PDU that contains an optional data payload. + * This includes dealing with digests and the length fields in the + * common header. + */ +static int +nvmf_tcp_construct_pdu(struct nvmf_tcp_qpair *qp, void *hdr, size_t hlen, + void *data, uint32_t data_len) +{ + struct nvme_tcp_common_pdu_hdr *ch; + struct iovec iov[5]; + u_int iovcnt; + uint32_t header_digest, data_digest, pad, pdo, plen; + + plen = hlen; + if (qp->header_digests) + plen += sizeof(header_digest); + if (data_len != 0) { + pdo = roundup(plen, qp->txpda); + pad = pdo - plen; + plen = pdo + data_len; + if (qp->data_digests) + plen += sizeof(data_digest); + } else { + assert(data == NULL); + pdo = 0; + pad = 0; + } + + ch = hdr; + ch->hlen = hlen; + if (qp->header_digests) + ch->flags |= NVME_TCP_CH_FLAGS_HDGSTF; + if (qp->data_digests && data_len != 0) + ch->flags |= NVME_TCP_CH_FLAGS_DDGSTF; + ch->pdo = pdo; + ch->plen = htole32(plen); + + /* CH + PSH */ + iov[0].iov_base = hdr; + iov[0].iov_len = hlen; + iovcnt = 1; + + /* HDGST */ + if (qp->header_digests) { + header_digest = compute_digest(hdr, hlen); + iov[iovcnt].iov_base = &header_digest; + iov[iovcnt].iov_len = sizeof(header_digest); + iovcnt++; + } + + if (pad != 0) { + /* PAD */ + iov[iovcnt].iov_base = __DECONST(char *, zero_padding); + iov[iovcnt].iov_len = pad; + iovcnt++; + } + + if (data_len != 0) { + /* DATA */ + iov[iovcnt].iov_base = data; + iov[iovcnt].iov_len = data_len; + iovcnt++; + + /* DDGST */ + if (qp->data_digests) { + data_digest = compute_digest(data, data_len); + iov[iovcnt].iov_base = &data_digest; + iov[iovcnt].iov_len = sizeof(data_digest); + iovcnt++; + } + } + + return (nvmf_tcp_write_pdu_iov(qp, iov, iovcnt, plen)); +} + +static int +nvmf_tcp_handle_h2c_data(struct nvmf_tcp_qpair *qp, struct nvmf_tcp_rxpdu *pdu) +{ + struct nvme_tcp_h2c_data_hdr *h2c; + struct nvmf_tcp_command_buffer *cb; + uint32_t data_len, data_offset; + const char *icd; + + h2c = (void *)pdu->hdr; + if (le32toh(h2c->datal) > qp->maxh2cdata) { + nvmf_tcp_report_error(qp->qp.nq_association, qp, + NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_LIMIT_EXCEEDED, 0, + pdu->hdr, le32toh(pdu->hdr->plen), pdu->hdr->hlen); + nvmf_tcp_free_pdu(pdu); + return (EBADMSG); + } + + cb = tcp_find_command_buffer(qp, h2c->cccid, h2c->ttag, true); + if (cb == NULL) { + nvmf_tcp_report_error(qp->qp.nq_association, qp, + NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD, + offsetof(struct nvme_tcp_h2c_data_hdr, ttag), pdu->hdr, + le32toh(pdu->hdr->plen), pdu->hdr->hlen); + nvmf_tcp_free_pdu(pdu); + return (EBADMSG); + } + + data_len = le32toh(h2c->datal); + if (data_len != pdu->data_len) { + nvmf_tcp_report_error(qp->qp.nq_association, qp, + NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD, + offsetof(struct nvme_tcp_h2c_data_hdr, datal), pdu->hdr, + le32toh(pdu->hdr->plen), pdu->hdr->hlen); + nvmf_tcp_free_pdu(pdu); + return (EBADMSG); + } + + data_offset = le32toh(h2c->datao); + if (data_offset < cb->data_offset || + data_offset + data_len > cb->data_offset + cb->data_len) { + nvmf_tcp_report_error(qp->qp.nq_association, qp, + NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE, 0, + pdu->hdr, le32toh(pdu->hdr->plen), pdu->hdr->hlen); + nvmf_tcp_free_pdu(pdu); + return (EBADMSG); + } + + if (data_offset != cb->data_offset + cb->data_xfered) { + nvmf_tcp_report_error(qp->qp.nq_association, qp, + NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, pdu->hdr, + le32toh(pdu->hdr->plen), pdu->hdr->hlen); + nvmf_tcp_free_pdu(pdu); + return (EBADMSG); + } + + if ((cb->data_xfered + data_len == cb->data_len) != + ((pdu->hdr->flags & NVME_TCP_H2C_DATA_FLAGS_LAST_PDU) != 0)) { + nvmf_tcp_report_error(qp->qp.nq_association, qp, + NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, pdu->hdr, + le32toh(pdu->hdr->plen), pdu->hdr->hlen); + nvmf_tcp_free_pdu(pdu); + return (EBADMSG); + } + + cb->data_xfered += data_len; + data_offset -= cb->data_offset; + icd = (const char *)pdu->hdr + pdu->hdr->pdo; + memcpy((char *)cb->data + data_offset, icd, data_len); + + nvmf_tcp_free_pdu(pdu); + return (0); +} + +static int +nvmf_tcp_handle_c2h_data(struct nvmf_tcp_qpair *qp, struct nvmf_tcp_rxpdu *pdu) +{ + struct nvme_tcp_c2h_data_hdr *c2h; + struct nvmf_tcp_command_buffer *cb; + uint32_t data_len, data_offset; + const char *icd; + + c2h = (void *)pdu->hdr; + + cb = tcp_find_command_buffer(qp, c2h->cccid, 0, true); + if (cb == NULL) { + /* + * XXX: Could be PDU sequence error if cccid is for a + * command that doesn't use a command buffer. + */ + nvmf_tcp_report_error(qp->qp.nq_association, qp, + NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD, + offsetof(struct nvme_tcp_c2h_data_hdr, cccid), pdu->hdr, + le32toh(pdu->hdr->plen), pdu->hdr->hlen); + nvmf_tcp_free_pdu(pdu); + return (EBADMSG); + } + + data_len = le32toh(c2h->datal); + if (data_len != pdu->data_len) { + nvmf_tcp_report_error(qp->qp.nq_association, qp, + NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD, + offsetof(struct nvme_tcp_c2h_data_hdr, datal), pdu->hdr, + le32toh(pdu->hdr->plen), pdu->hdr->hlen); + nvmf_tcp_free_pdu(pdu); + return (EBADMSG); + } + + data_offset = le32toh(c2h->datao); + if (data_offset < cb->data_offset || + data_offset + data_len > cb->data_offset + cb->data_len) { + nvmf_tcp_report_error(qp->qp.nq_association, qp, + NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE, 0, + pdu->hdr, le32toh(pdu->hdr->plen), pdu->hdr->hlen); + nvmf_tcp_free_pdu(pdu); + return (EBADMSG); + } + + if (data_offset != cb->data_offset + cb->data_xfered) { + nvmf_tcp_report_error(qp->qp.nq_association, qp, + NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, pdu->hdr, + le32toh(pdu->hdr->plen), pdu->hdr->hlen); + nvmf_tcp_free_pdu(pdu); + return (EBADMSG); + } + + if ((cb->data_xfered + data_len == cb->data_len) != + ((pdu->hdr->flags & NVME_TCP_C2H_DATA_FLAGS_LAST_PDU) != 0)) { + nvmf_tcp_report_error(qp->qp.nq_association, qp, + NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, pdu->hdr, + le32toh(pdu->hdr->plen), pdu->hdr->hlen); + nvmf_tcp_free_pdu(pdu); + return (EBADMSG); + } + + cb->data_xfered += data_len; + data_offset -= cb->data_offset; + icd = (const char *)pdu->hdr + pdu->hdr->pdo; + memcpy((char *)cb->data + data_offset, icd, data_len); + + if ((pdu->hdr->flags & NVME_TCP_C2H_DATA_FLAGS_SUCCESS) != 0) { + struct nvme_completion cqe; + struct nvmf_tcp_capsule *tc; + struct nvmf_capsule *nc; + + memset(&cqe, 0, sizeof(cqe)); + cqe.cid = cb->cid; + + nc = nvmf_allocate_response(&qp->qp, &cqe); + if (nc == NULL) { + nvmf_tcp_free_pdu(pdu); + return (ENOMEM); + } + nc->nc_sqhd_valid = false; + + tc = TCAP(nc); + TAILQ_INSERT_TAIL(&qp->rx_capsules, tc, link); + } + + nvmf_tcp_free_pdu(pdu); + return (0); +} + +/* NB: cid and ttag and little-endian already. */ +static int +tcp_send_h2c_pdu(struct nvmf_tcp_qpair *qp, uint16_t cid, uint16_t ttag, + uint32_t data_offset, void *buf, size_t len, bool last_pdu) +{ + struct nvme_tcp_h2c_data_hdr h2c; + + memset(&h2c, 0, sizeof(h2c)); + h2c.common.pdu_type = NVME_TCP_PDU_TYPE_H2C_DATA; + if (last_pdu) + h2c.common.flags |= NVME_TCP_H2C_DATA_FLAGS_LAST_PDU; + h2c.cccid = cid; + h2c.ttag = ttag; + h2c.datao = htole32(data_offset); + h2c.datal = htole32(len); + + return (nvmf_tcp_construct_pdu(qp, &h2c, sizeof(h2c), buf, len)); +} + +/* Sends one or more H2C_DATA PDUs, subject to MAXH2CDATA. */ +static int +tcp_send_h2c_pdus(struct nvmf_tcp_qpair *qp, uint16_t cid, uint16_t ttag, + uint32_t data_offset, void *buf, size_t len, bool last_pdu) +{ + char *p; + + p = buf; + while (len != 0) { + size_t todo; + int error; + + todo = len; + if (todo > qp->maxh2cdata) + todo = qp->maxh2cdata; + error = tcp_send_h2c_pdu(qp, cid, ttag, data_offset, p, todo, + last_pdu && todo == len); + if (error != 0) + return (error); + p += todo; + len -= todo; + } + return (0); +} + +static int +nvmf_tcp_handle_r2t(struct nvmf_tcp_qpair *qp, struct nvmf_tcp_rxpdu *pdu) +{ + struct nvmf_tcp_command_buffer *cb; + struct nvme_tcp_r2t_hdr *r2t; + uint32_t data_len, data_offset; + int error; + + r2t = (void *)pdu->hdr; + + cb = tcp_find_command_buffer(qp, r2t->cccid, 0, false); + if (cb == NULL) { + nvmf_tcp_report_error(qp->qp.nq_association, qp, + NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD, + offsetof(struct nvme_tcp_r2t_hdr, cccid), pdu->hdr, + le32toh(pdu->hdr->plen), pdu->hdr->hlen); + nvmf_tcp_free_pdu(pdu); + return (EBADMSG); + } + + data_offset = le32toh(r2t->r2to); + if (data_offset != cb->data_xfered) { + nvmf_tcp_report_error(qp->qp.nq_association, qp, + NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, pdu->hdr, + le32toh(pdu->hdr->plen), pdu->hdr->hlen); + nvmf_tcp_free_pdu(pdu); + return (EBADMSG); + } + + /* + * XXX: The spec does not specify how to handle R2T tranfers + * out of range of the original command. + */ + data_len = le32toh(r2t->r2tl); + if (data_offset + data_len > cb->data_len) { + nvmf_tcp_report_error(qp->qp.nq_association, qp, + NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE, 0, + pdu->hdr, le32toh(pdu->hdr->plen), pdu->hdr->hlen); + nvmf_tcp_free_pdu(pdu); + return (EBADMSG); + } + + cb->data_xfered += data_len; + + /* + * Write out one or more H2C_DATA PDUs containing the + * requested data. + */ + error = tcp_send_h2c_pdus(qp, r2t->cccid, r2t->ttag, + data_offset, (char *)cb->data + data_offset, data_len, true); + + nvmf_tcp_free_pdu(pdu); + return (error); +} + +static int +nvmf_tcp_receive_pdu(struct nvmf_tcp_qpair *qp) +{ + struct nvmf_tcp_rxpdu pdu; + int error; + + error = nvmf_tcp_read_pdu(qp, &pdu); + if (error != 0) + return (error); + + switch (pdu.hdr->pdu_type) { + default: + __unreachable(); + break; + case NVME_TCP_PDU_TYPE_H2C_TERM_REQ: + case NVME_TCP_PDU_TYPE_C2H_TERM_REQ: + return (nvmf_tcp_handle_term_req(&pdu)); + case NVME_TCP_PDU_TYPE_CAPSULE_CMD: + return (nvmf_tcp_save_command_capsule(qp, &pdu)); + case NVME_TCP_PDU_TYPE_CAPSULE_RESP: + return (nvmf_tcp_save_response_capsule(qp, &pdu)); + case NVME_TCP_PDU_TYPE_H2C_DATA: + return (nvmf_tcp_handle_h2c_data(qp, &pdu)); + case NVME_TCP_PDU_TYPE_C2H_DATA: + return (nvmf_tcp_handle_c2h_data(qp, &pdu)); + case NVME_TCP_PDU_TYPE_R2T: + return (nvmf_tcp_handle_r2t(qp, &pdu)); + } +} + +static bool +nvmf_tcp_validate_ic_pdu(struct nvmf_association *na, struct nvmf_tcp_qpair *qp, + const struct nvme_tcp_common_pdu_hdr *ch, size_t pdu_len) +{ + const struct nvme_tcp_ic_req *pdu; + uint32_t plen; + u_int hlen; + + /* Determine how large of a PDU header to return for errors. */ + hlen = ch->hlen; + plen = le32toh(ch->plen); + if (hlen < sizeof(*ch) || hlen > plen) + hlen = sizeof(*ch); + + /* + * Errors must be reported for the lowest incorrect field + * first, so validate fields in order. + */ + + /* Validate pdu_type. */ + + /* Controllers only receive PDUs with a PDU direction of 0. */ + if (na->na_controller != ((ch->pdu_type & 0x01) == 0)) { + na_error(na, "NVMe/TCP: Invalid PDU type %u", ch->pdu_type); + nvmf_tcp_report_error(na, qp, + NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD, 0, ch, pdu_len, + hlen); + return (false); + } + + switch (ch->pdu_type) { + case NVME_TCP_PDU_TYPE_IC_REQ: + case NVME_TCP_PDU_TYPE_IC_RESP: + break; + default: + na_error(na, "NVMe/TCP: Invalid PDU type %u", ch->pdu_type); + nvmf_tcp_report_error(na, qp, + NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD, 0, ch, pdu_len, + hlen); + return (false); + } + + /* Validate flags. */ + if (ch->flags != 0) { + na_error(na, "NVMe/TCP: Invalid PDU header flags %#x", + ch->flags); + nvmf_tcp_report_error(na, qp, + NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD, 1, ch, pdu_len, + hlen); + return (false); + } + + /* Validate hlen. */ + if (ch->hlen != 128) { + na_error(na, "NVMe/TCP: Invalid PDU header length %u", + ch->hlen); + nvmf_tcp_report_error(na, qp, + NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD, 2, ch, pdu_len, + hlen); + return (false); + } + + /* Validate pdo. */ + if (ch->pdo != 0) { + na_error(na, "NVMe/TCP: Invalid PDU data offset %u", ch->pdo); + nvmf_tcp_report_error(na, qp, + NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD, 3, ch, pdu_len, + hlen); + return (false); + } + + /* Validate plen. */ + if (plen != 128) { + na_error(na, "NVMe/TCP: Invalid PDU length %u", plen); + nvmf_tcp_report_error(na, qp, + NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD, 4, ch, pdu_len, + hlen); + return (false); + } + + /* Validate fields common to both ICReq and ICResp. */ + pdu = (const struct nvme_tcp_ic_req *)ch; + if (le16toh(pdu->pfv) != 0) { + na_error(na, "NVMe/TCP: Unsupported PDU version %u", + le16toh(pdu->pfv)); + nvmf_tcp_report_error(na, qp, + NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER, + 8, ch, pdu_len, hlen); + return (false); + } + + if (pdu->hpda > NVME_TCP_HPDA_MAX) { + na_error(na, "NVMe/TCP: Unsupported PDA %u", pdu->hpda); + nvmf_tcp_report_error(na, qp, + NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD, 10, ch, pdu_len, + hlen); + return (false); + } + + if (pdu->dgst.bits.reserved != 0) { + na_error(na, "NVMe/TCP: Invalid digest settings"); + nvmf_tcp_report_error(na, qp, + NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD, 11, ch, pdu_len, + hlen); + return (false); + } + + return (true); +} + +static bool +nvmf_tcp_read_ic_req(struct nvmf_association *na, struct nvmf_tcp_qpair *qp, + struct nvme_tcp_ic_req *pdu) +{ + int error; + + error = nvmf_tcp_read_buffer(qp->s, pdu, sizeof(*pdu)); + if (error != 0) { + na_error(na, "NVMe/TCP: Failed to read IC request: %s", + strerror(error)); + return (false); + } + + return (nvmf_tcp_validate_ic_pdu(na, qp, &pdu->common, sizeof(*pdu))); +} + +static bool +nvmf_tcp_read_ic_resp(struct nvmf_association *na, struct nvmf_tcp_qpair *qp, + struct nvme_tcp_ic_resp *pdu) +{ + int error; + + error = nvmf_tcp_read_buffer(qp->s, pdu, sizeof(*pdu)); + if (error != 0) { + na_error(na, "NVMe/TCP: Failed to read IC response: %s", + strerror(error)); + return (false); + } + + return (nvmf_tcp_validate_ic_pdu(na, qp, &pdu->common, sizeof(*pdu))); +} + +static struct nvmf_association * +tcp_allocate_association(bool controller, + const struct nvmf_association_params *params) +{ + struct nvmf_tcp_association *ta; + + if (controller) { + /* 7.4.10.3 */ + if (params->tcp.maxh2cdata < 4096 || + params->tcp.maxh2cdata % 4 != 0) + return (NULL); + } + + ta = calloc(1, sizeof(*ta)); + + return (&ta->na); +} + +static void +tcp_update_association(struct nvmf_association *na, + const struct nvme_controller_data *cdata) +{ + struct nvmf_tcp_association *ta = TASSOC(na); + + ta->ioccsz = le32toh(cdata->ioccsz); +} + +static void +tcp_free_association(struct nvmf_association *na) +{ + free(na); +} + +static bool +tcp_connect(struct nvmf_tcp_qpair *qp, struct nvmf_association *na, bool admin) +{ + const struct nvmf_association_params *params = &na->na_params; + struct nvmf_tcp_association *ta = TASSOC(na); + struct nvme_tcp_ic_req ic_req; + struct nvme_tcp_ic_resp ic_resp; + uint32_t maxh2cdata; + int error; + + if (!admin) { + if (ta->ioccsz == 0) { + na_error(na, "TCP I/O queues require cdata"); + return (false); + } + if (ta->ioccsz < 4) { + na_error(na, "Invalid IOCCSZ %u", ta->ioccsz); + return (false); + } + } + + memset(&ic_req, 0, sizeof(ic_req)); + ic_req.common.pdu_type = NVME_TCP_PDU_TYPE_IC_REQ; + ic_req.common.hlen = sizeof(ic_req); + ic_req.common.plen = htole32(sizeof(ic_req)); + ic_req.pfv = htole16(0); + ic_req.hpda = params->tcp.pda; + if (params->tcp.header_digests) + ic_req.dgst.bits.hdgst_enable = 1; + if (params->tcp.data_digests) + ic_req.dgst.bits.ddgst_enable = 1; + ic_req.maxr2t = htole32(params->tcp.maxr2t); + + error = nvmf_tcp_write_pdu(qp, &ic_req, sizeof(ic_req)); + if (error != 0) { + na_error(na, "Failed to write IC request: %s", strerror(error)); + return (false); + } + + if (!nvmf_tcp_read_ic_resp(na, qp, &ic_resp)) + return (false); + + /* Ensure the controller didn't enable digests we didn't request. */ + if ((!params->tcp.header_digests && + ic_resp.dgst.bits.hdgst_enable != 0) || + (!params->tcp.data_digests && + ic_resp.dgst.bits.ddgst_enable != 0)) { + na_error(na, "Controller enabled unrequested digests"); + nvmf_tcp_report_error(na, qp, + NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER, + 11, &ic_resp, sizeof(ic_resp), sizeof(ic_resp)); + return (false); + } + + /* + * XXX: Is there an upper-bound to enforce here? Perhaps pick + * some large value and report larger values as an unsupported + * parameter? + */ + maxh2cdata = le32toh(ic_resp.maxh2cdata); + if (maxh2cdata < 4096 || maxh2cdata % 4 != 0) { + na_error(na, "Invalid MAXH2CDATA %u", maxh2cdata); + nvmf_tcp_report_error(na, qp, + NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD, 12, &ic_resp, + sizeof(ic_resp), sizeof(ic_resp)); + return (false); + } + + qp->rxpda = (params->tcp.pda + 1) * 4; + qp->txpda = (ic_resp.cpda + 1) * 4; + qp->header_digests = ic_resp.dgst.bits.hdgst_enable != 0; + qp->data_digests = ic_resp.dgst.bits.ddgst_enable != 0; + qp->maxr2t = params->tcp.maxr2t; + qp->maxh2cdata = maxh2cdata; + if (admin) + /* 7.4.3 */ + qp->max_icd = 8192; + else + qp->max_icd = (ta->ioccsz - 4) * 16; + + return (0); +} + +static bool +tcp_accept(struct nvmf_tcp_qpair *qp, struct nvmf_association *na) +{ + const struct nvmf_association_params *params = &na->na_params; + struct nvme_tcp_ic_req ic_req; + struct nvme_tcp_ic_resp ic_resp; + int error; + + if (!nvmf_tcp_read_ic_req(na, qp, &ic_req)) + return (false); + + memset(&ic_resp, 0, sizeof(ic_resp)); + ic_resp.common.pdu_type = NVME_TCP_PDU_TYPE_IC_RESP; + ic_resp.common.hlen = sizeof(ic_req); + ic_resp.common.plen = htole32(sizeof(ic_req)); + ic_resp.pfv = htole16(0); + ic_resp.cpda = params->tcp.pda; + if (params->tcp.header_digests && ic_req.dgst.bits.hdgst_enable != 0) + ic_resp.dgst.bits.hdgst_enable = 1; + if (params->tcp.data_digests && ic_req.dgst.bits.ddgst_enable != 0) + ic_resp.dgst.bits.ddgst_enable = 1; + ic_resp.maxh2cdata = htole32(params->tcp.maxh2cdata); + + error = nvmf_tcp_write_pdu(qp, &ic_resp, sizeof(ic_resp)); + if (error != 0) { + na_error(na, "Failed to write IC response: %s", + strerror(error)); + return (false); + } + + qp->rxpda = (params->tcp.pda + 1) * 4; + qp->txpda = (ic_req.hpda + 1) * 4; + qp->header_digests = ic_resp.dgst.bits.hdgst_enable != 0; + qp->data_digests = ic_resp.dgst.bits.ddgst_enable != 0; + qp->maxr2t = le32toh(ic_req.maxr2t); + qp->maxh2cdata = params->tcp.maxh2cdata; + qp->max_icd = 0; /* XXX */ + return (0); +} + +static struct nvmf_qpair * +tcp_allocate_qpair(struct nvmf_association *na, + const struct nvmf_qpair_params *qparams) +{ + const struct nvmf_association_params *aparams = &na->na_params; + struct nvmf_tcp_qpair *qp; + int error; + + if (aparams->tcp.pda > NVME_TCP_CPDA_MAX) { + na_error(na, "Invalid PDA"); + return (NULL); + } + + qp = calloc(1, sizeof(*qp)); + qp->s = qparams->tcp.fd; + LIST_INIT(&qp->rx_buffers); + LIST_INIT(&qp->tx_buffers); + TAILQ_INIT(&qp->rx_capsules); + if (na->na_controller) + error = tcp_accept(qp, na); + else + error = tcp_connect(qp, na, qparams->admin); + if (error != 0) { + free(qp); + return (NULL); + } + + return (&qp->qp); +} + +static void +tcp_free_qpair(struct nvmf_qpair *nq) +{ + struct nvmf_tcp_qpair *qp = TQP(nq); + struct nvmf_tcp_capsule *ntc, *tc; + struct nvmf_tcp_command_buffer *ncb, *cb; + + TAILQ_FOREACH_SAFE(tc, &qp->rx_capsules, link, ntc) { + TAILQ_REMOVE(&qp->rx_capsules, tc, link); + nvmf_free_capsule(&tc->nc); + } + LIST_FOREACH_SAFE(cb, &qp->rx_buffers, link, ncb) { + tcp_free_command_buffer(cb); + } + LIST_FOREACH_SAFE(cb, &qp->tx_buffers, link, ncb) { + tcp_free_command_buffer(cb); + } + free(qp); +} + +static void +tcp_kernel_handoff_params(struct nvmf_qpair *nq, nvlist_t *nvl) +{ + struct nvmf_tcp_qpair *qp = TQP(nq); + + nvlist_add_number(nvl, "fd", qp->s); + nvlist_add_number(nvl, "rxpda", qp->rxpda); + nvlist_add_number(nvl, "txpda", qp->txpda); + nvlist_add_bool(nvl, "header_digests", qp->header_digests); + nvlist_add_bool(nvl, "data_digests", qp->data_digests); + nvlist_add_number(nvl, "maxr2t", qp->maxr2t); + nvlist_add_number(nvl, "maxh2cdata", qp->maxh2cdata); + nvlist_add_number(nvl, "max_icd", qp->max_icd); +} + +static int +tcp_populate_dle(struct nvmf_qpair *nq, struct nvme_discovery_log_entry *dle) +{ + struct nvmf_tcp_qpair *qp = TQP(nq); + struct sockaddr_storage ss; + socklen_t ss_len; + + ss_len = sizeof(ss); + if (getpeername(qp->s, (struct sockaddr *)&ss, &ss_len) == -1) + return (errno); + + if (getnameinfo((struct sockaddr *)&ss, ss_len, dle->traddr, + sizeof(dle->traddr), dle->trsvcid, sizeof(dle->trsvcid), + NI_NUMERICHOST | NI_NUMERICSERV) != 0) + return (EINVAL); + + return (0); +} + +static struct nvmf_capsule * +tcp_allocate_capsule(struct nvmf_qpair *qp __unused) +{ + struct nvmf_tcp_capsule *nc; + + nc = calloc(1, sizeof(*nc)); + return (&nc->nc); +} + +static void +tcp_free_capsule(struct nvmf_capsule *nc) +{ + struct nvmf_tcp_capsule *tc = TCAP(nc); + + nvmf_tcp_free_pdu(&tc->rx_pdu); + if (tc->cb != NULL) + tcp_free_command_buffer(tc->cb); + free(tc); +} + +static int +tcp_transmit_command(struct nvmf_capsule *nc) +{ + struct nvmf_tcp_qpair *qp = TQP(nc->nc_qpair); + struct nvmf_tcp_capsule *tc = TCAP(nc); + struct nvme_tcp_cmd cmd; + struct nvme_sgl_descriptor *sgl; + int error; + bool use_icd; + + use_icd = false; + if (nc->nc_data_len != 0 && nc->nc_send_data && + nc->nc_data_len <= qp->max_icd) + use_icd = true; + + memset(&cmd, 0, sizeof(cmd)); + cmd.common.pdu_type = NVME_TCP_PDU_TYPE_CAPSULE_CMD; + cmd.ccsqe = nc->nc_sqe; + + /* Populate SGL in SQE. */ + sgl = &cmd.ccsqe.sgl; + memset(sgl, 0, sizeof(*sgl)); + sgl->address = 0; + sgl->length = htole32(nc->nc_data_len); + if (use_icd) { + /* Use in-capsule data. */ + sgl->type = NVME_SGL_TYPE_ICD; + } else { + /* Use a command buffer. */ + sgl->type = NVME_SGL_TYPE_COMMAND_BUFFER; + } + + /* Send command capsule. */ + error = nvmf_tcp_construct_pdu(qp, &cmd, sizeof(cmd), use_icd ? + nc->nc_data : NULL, use_icd ? nc->nc_data_len : 0); + if (error != 0) + return (error); + + /* + * If data will be transferred using a command buffer, allocate a + * buffer structure and queue it. + */ + if (nc->nc_data_len != 0 && !use_icd) + tc->cb = tcp_alloc_command_buffer(qp, nc->nc_data, 0, + nc->nc_data_len, cmd.ccsqe.cid, 0, !nc->nc_send_data); + + return (0); +} + +static int +tcp_transmit_response(struct nvmf_capsule *nc) +{ + struct nvmf_tcp_qpair *qp = TQP(nc->nc_qpair); + struct nvme_tcp_rsp rsp; + + memset(&rsp, 0, sizeof(rsp)); + rsp.common.pdu_type = NVME_TCP_PDU_TYPE_CAPSULE_RESP; + rsp.rccqe = nc->nc_cqe; + + return (nvmf_tcp_construct_pdu(qp, &rsp, sizeof(rsp), NULL, 0)); +} + +static int +tcp_transmit_capsule(struct nvmf_capsule *nc) +{ + if (nc->nc_qe_len == sizeof(struct nvme_command)) + return (tcp_transmit_command(nc)); + else + return (tcp_transmit_response(nc)); +} + +static int +tcp_receive_capsule(struct nvmf_qpair *nq, struct nvmf_capsule **ncp) +{ + struct nvmf_tcp_qpair *qp = TQP(nq); + struct nvmf_tcp_capsule *tc; + int error; + + while (TAILQ_EMPTY(&qp->rx_capsules)) { + error = nvmf_tcp_receive_pdu(qp); + if (error != 0) + return (error); + } + tc = TAILQ_FIRST(&qp->rx_capsules); + TAILQ_REMOVE(&qp->rx_capsules, tc, link); + *ncp = &tc->nc; + return (0); +} + +static uint8_t +tcp_validate_command_capsule(const struct nvmf_capsule *nc) +{ + const struct nvmf_tcp_capsule *tc = CTCAP(nc); + const struct nvme_sgl_descriptor *sgl; + + assert(tc->rx_pdu.hdr != NULL); + + sgl = &nc->nc_sqe.sgl; + switch (sgl->type) { + case NVME_SGL_TYPE_ICD: + if (tc->rx_pdu.data_len != le32toh(sgl->length)) { + printf("NVMe/TCP: Command Capsule with mismatched ICD length\n"); + return (NVME_SC_DATA_SGL_LENGTH_INVALID); + } + break; + case NVME_SGL_TYPE_COMMAND_BUFFER: + if (tc->rx_pdu.data_len != 0) { + printf("NVMe/TCP: Command Buffer SGL with ICD\n"); + return (NVME_SC_INVALID_FIELD); + } + break; + default: + printf("NVMe/TCP: Invalid SGL type in Command Capsule\n"); + return (NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID); + } + + if (sgl->address != 0) { + printf("NVMe/TCP: Invalid SGL offset in Command Capsule\n"); + return (NVME_SC_SGL_OFFSET_INVALID); + } + + return (NVME_SC_SUCCESS); +} + +static size_t +tcp_capsule_data_len(const struct nvmf_capsule *nc) +{ + assert(nc->nc_qe_len == sizeof(struct nvme_command)); + return (le32toh(nc->nc_sqe.sgl.length)); +} + +/* NB: cid and ttag are both little-endian already. */ +static int +tcp_send_r2t(struct nvmf_tcp_qpair *qp, uint16_t cid, uint16_t ttag, + uint32_t data_offset, uint32_t data_len) +{ + struct nvme_tcp_r2t_hdr r2t; + + memset(&r2t, 0, sizeof(r2t)); + r2t.common.pdu_type = NVME_TCP_PDU_TYPE_R2T; + r2t.cccid = cid; + r2t.ttag = ttag; + r2t.r2to = htole32(data_offset); + r2t.r2tl = htole32(data_len); + + return (nvmf_tcp_construct_pdu(qp, &r2t, sizeof(r2t), NULL, 0)); +} + +static int +tcp_receive_r2t_data(const struct nvmf_capsule *nc, uint32_t data_offset, + void *buf, size_t len) +{ + struct nvmf_tcp_qpair *qp = TQP(nc->nc_qpair); + struct nvmf_tcp_command_buffer *cb; + int error; + uint16_t ttag; + + /* + * Don't bother byte-swapping ttag as it is just a cookie + * value returned by the other end as-is. + */ + ttag = qp->next_ttag++; + + error = tcp_send_r2t(qp, nc->nc_sqe.cid, ttag, data_offset, len); + if (error != 0) + return (error); + + cb = tcp_alloc_command_buffer(qp, buf, data_offset, len, + nc->nc_sqe.cid, ttag, true); + + /* Parse received PDUs until the data transfer is complete. */ + while (cb->data_xfered < cb->data_len) { + error = nvmf_tcp_receive_pdu(qp); + if (error != 0) + break; + } + tcp_free_command_buffer(cb); + return (error); +} + +static int +tcp_receive_icd_data(const struct nvmf_capsule *nc, uint32_t data_offset, + void *buf, size_t len) +{ + const struct nvmf_tcp_capsule *tc = CTCAP(nc); + const char *icd; + + icd = (const char *)tc->rx_pdu.hdr + tc->rx_pdu.hdr->pdo + data_offset; + memcpy(buf, icd, len); + return (0); +} + +static int +tcp_receive_controller_data(const struct nvmf_capsule *nc, uint32_t data_offset, + void *buf, size_t len) +{ + struct nvmf_association *na = nc->nc_qpair->nq_association; + const struct nvme_sgl_descriptor *sgl; + size_t data_len; + + if (nc->nc_qe_len != sizeof(struct nvme_command) || !na->na_controller) + return (EINVAL); + + sgl = &nc->nc_sqe.sgl; + data_len = le32toh(sgl->length); + if (data_offset + len > data_len) + return (EFBIG); + + if (sgl->type == NVME_SGL_TYPE_ICD) + return (tcp_receive_icd_data(nc, data_offset, buf, len)); + else + return (tcp_receive_r2t_data(nc, data_offset, buf, len)); +} + +/* NB: cid is little-endian already. */ +static int +tcp_send_c2h_pdu(struct nvmf_tcp_qpair *qp, uint16_t cid, + uint32_t data_offset, const void *buf, size_t len, bool last_pdu, + bool success) +{ + struct nvme_tcp_c2h_data_hdr c2h; + + memset(&c2h, 0, sizeof(c2h)); + c2h.common.pdu_type = NVME_TCP_PDU_TYPE_C2H_DATA; + if (last_pdu) + c2h.common.flags |= NVME_TCP_C2H_DATA_FLAGS_LAST_PDU; + if (success) + c2h.common.flags |= NVME_TCP_C2H_DATA_FLAGS_SUCCESS; + c2h.cccid = cid; + c2h.datao = htole32(data_offset); + c2h.datal = htole32(len); + + return (nvmf_tcp_construct_pdu(qp, &c2h, sizeof(c2h), + __DECONST(void *, buf), len)); +} + +static int +tcp_send_controller_data(const struct nvmf_capsule *nc, const void *buf, + size_t len) +{ + struct nvmf_association *na = nc->nc_qpair->nq_association; + struct nvmf_tcp_qpair *qp = TQP(nc->nc_qpair); + const struct nvme_sgl_descriptor *sgl; + const char *src; + size_t todo; + uint32_t data_len, data_offset; + int error; + bool last_pdu, send_success_flag; + + if (nc->nc_qe_len != sizeof(struct nvme_command) || !na->na_controller) + return (EINVAL); + + sgl = &nc->nc_sqe.sgl; + data_len = le32toh(sgl->length); + if (len != data_len) { + nvmf_send_generic_error(nc, NVME_SC_INVALID_FIELD); + return (EFBIG); + } + + if (sgl->type != NVME_SGL_TYPE_COMMAND_BUFFER) { + nvmf_send_generic_error(nc, NVME_SC_INVALID_FIELD); + return (EINVAL); + } + + /* Use the SUCCESS flag if SQ flow control is disabled. */ + send_success_flag = !qp->qp.nq_flow_control; + + /* + * Write out one or more C2H_DATA PDUs containing the data. + * Each PDU is arbitrarily capped at 256k. + */ + data_offset = 0; + src = buf; + while (len > 0) { + if (len > 256 * 1024) { + todo = 256 * 1024; + last_pdu = false; + } else { + todo = len; + last_pdu = true; + } + error = tcp_send_c2h_pdu(qp, nc->nc_sqe.cid, data_offset, + src, todo, last_pdu, last_pdu && send_success_flag); + if (error != 0) { + nvmf_send_generic_error(nc, + NVME_SC_TRANSIENT_TRANSPORT_ERROR); + return (error); + } + data_offset += todo; + src += todo; + len -= todo; + } + if (!send_success_flag) + nvmf_send_success(nc); + return (0); +} + +struct nvmf_transport_ops tcp_ops = { + .allocate_association = tcp_allocate_association, + .update_association = tcp_update_association, + .free_association = tcp_free_association, + .allocate_qpair = tcp_allocate_qpair, + .free_qpair = tcp_free_qpair, + .kernel_handoff_params = tcp_kernel_handoff_params, + .populate_dle = tcp_populate_dle, + .allocate_capsule = tcp_allocate_capsule, + .free_capsule = tcp_free_capsule, + .transmit_capsule = tcp_transmit_capsule, + .receive_capsule = tcp_receive_capsule, + .validate_command_capsule = tcp_validate_command_capsule, + .capsule_data_len = tcp_capsule_data_len, + .receive_controller_data = tcp_receive_controller_data, + .send_controller_data = tcp_send_controller_data, +}; |